diff --git a/.editorconfig b/.editorconfig index d0cac42e..25e75413 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,8 +14,9 @@ indent_size = 2 indent_style = space indent_size = 4 -[*.go] -indent_style = tab +[*.{rs,toml}] +indent_style = space +indent_size = 4 [*.md] trim_trailing_whitespace = false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c1486114..b7fa18c0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -22,7 +22,7 @@ - [ ] `dart format --output=none --set-exit-if-changed lib test` - [ ] `flutter analyze` - [ ] `flutter test` -- [ ] `gofmt`, `go vet ./...`, and `go test ./...` in `go_backend/` +- [ ] `bash scripts/check_rust_backend.sh` - [ ] Android or iOS native checks - [ ] Manual reproduction or device test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e46ae208..e21ced5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,8 +18,9 @@ jobs: pull-requests: read outputs: dart: ${{ steps.filter.outputs.dart }} - go: ${{ steps.filter.outputs.go }} android: ${{ steps.filter.outputs.android }} + ios: ${{ steps.filter.outputs.ios }} + rust: ${{ steps.filter.outputs.rust }} steps: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 @@ -40,20 +41,34 @@ jobs: - 'l10n.yaml' - '.fvmrc' - '.github/workflows/ci.yml' - go: - - 'go_backend/**' - - 'android/app/src/test/resources/**' + rust: + - 'rust_backend/**' + - 'scripts/*rust*.sh' + - 'android/gradlew' + - 'android/gradle/wrapper/**' + - '.github/workflows/ci.yml' + ios: + - 'ios/**' + - 'scripts/build_ios.sh' + - 'scripts/check_backend_ios.py' + - 'lib/**' + - 'assets/**' + - 'pubspec.yaml' + - 'pubspec.lock' + - 'l10n.yaml' + - '.fvmrc' - '.github/workflows/ci.yml' android: - 'android/**' - - 'go_backend/**' - - 'lib/models/settings.dart' - - 'lib/providers/download_queue_provider*.dart' - - 'lib/services/download_request_payload.dart' - - 'lib/services/history_database.dart' - - 'lib/services/platform_bridge.dart' + - 'rust_backend/**' + - 'scripts/build_android.sh' + - 'scripts/build_rust_backend.sh' + - 'scripts/check_backend_apk.py' + - 'lib/**' + - 'assets/**' - 'pubspec.yaml' - 'pubspec.lock' + - 'l10n.yaml' - '.fvmrc' - '.github/workflows/ci.yml' @@ -87,40 +102,88 @@ jobs: - name: Analyze run: flutter analyze + - name: Install media test tools + run: | + sudo apt-get update + sudo apt-get install -y ffmpeg + ffmpeg -version + ffprobe -version + - name: Run tests run: flutter test - go: - name: Go vet & test + rust: + name: Rust checks & Android libraries runs-on: ubuntu-latest needs: changes - if: needs.changes.outputs.go == 'true' - - defaults: - run: - working-directory: go_backend + if: needs.changes.outputs.rust == 'true' steps: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - - name: Setup Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + - name: Setup Java for Android SDK tools + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 with: - go-version-file: go_backend/go.mod - cache-dependency-path: go_backend/go.sum + distribution: "temurin" + java-version: "25" - - name: Check formatting - run: test -z "$(gofmt -l .)" + - name: Activate pinned Rust toolchain + working-directory: rust_backend + run: rustup show - - name: Vet - run: go vet ./... + - name: Install C binding generator dependencies + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev - - name: Run tests - run: go test ./... + - name: Rust formatting, Clippy and unit tests + run: bash scripts/check_rust_backend.sh + + - name: Install Android NDK + run: $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;29.0.14206865" + + - name: Build Android ARM32 and ARM64 libraries + run: ANDROID_NDK_HOME="$ANDROID_HOME/ndk/29.0.14206865" bash scripts/build_rust_backend.sh android + + rust-ios: + name: Rust iOS libraries & release application + runs-on: macos-latest + needs: changes + if: needs.changes.outputs.rust == 'true' || needs.changes.outputs.ios == 'true' + + steps: + - name: Checkout repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + + - name: Activate pinned Rust toolchain + working-directory: rust_backend + run: rustup show + + - name: Build iOS device and universal simulator XCFramework + run: bash scripts/build_rust_backend.sh ios + + - name: Setup Flutter + uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2 + with: + channel: "stable" + flutter-version-file: .fvmrc + cache: true + + - name: Configure Rust application pods + run: | + flutter pub get + cd ios + pod install + + - name: Build and audit Rust iOS release application + run: | + flutter build ios --release --no-codesign --target lib/main.dart + python3 scripts/check_backend_ios.py build/ios/iphoneos/Runner.app \ + --backend rust --archs arm64 --platform ios --release android: - name: Android compile & native tests + name: Android Rust builds & native tests runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.android == 'true' @@ -135,12 +198,6 @@ jobs: distribution: "temurin" java-version: "25" - - name: Setup Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 - with: - go-version-file: go_backend/go.mod - cache-dependency-path: go_backend/go.sum - - name: Setup Flutter uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2 with: @@ -162,25 +219,30 @@ jobs: "build-tools;37.0.0" echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/29.0.14206865" >> "$GITHUB_ENV" - - name: Build Go backend for Android - working-directory: go_backend - run: | - go install golang.org/x/mobile/cmd/gomobile - gomobile init - mkdir -p ../android/app/libs - gomobile bind \ - -target=android/arm,android/arm64 \ - -androidapi 24 \ - -o ../android/app/libs/gobackend.aar \ - . - env: - CGO_ENABLED: 1 - - name: Get Flutter dependencies run: flutter pub get - name: Configure Flutter SDK for Gradle run: echo "flutter.sdk=$FLUTTER_ROOT" > android/local.properties - - name: Compile Kotlin and run native unit tests - run: gradle -p android :app:compileDebugKotlin :app:testDebugUnitTest + - name: Install Rust application build prerequisites + run: | + cd rust_backend + rustup show + sudo apt-get update + sudo apt-get install -y libclang-dev + + - name: Package Rust application and run native unit tests + run: gradle -p android :app:assembleDebug :app:testDebugUnitTest + + - name: Build Rust release application + run: bash scripts/build_android.sh --target lib/main.dart + + - name: Verify Rust release APK payloads + run: | + python3 scripts/check_backend_apk.py \ + build/app/outputs/flutter-apk/app-arm64-v8a-release.apk \ + --backend rust --abis arm64-v8a + python3 scripts/check_backend_apk.py \ + build/app/outputs/flutter-apk/app-armeabi-v7a-release.apk \ + --backend rust --abis armeabi-v7a diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fb87f060..b2a52246 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,11 +74,14 @@ jobs: distribution: "temurin" java-version: "25" - - name: Setup Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 - with: - go-version-file: go_backend/go.mod - cache-dependency-path: go_backend/go.sum + - name: Activate pinned Rust toolchain + working-directory: rust_backend + run: rustup show + + - name: Install Rust binding generator dependencies + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev # Cache Gradle for faster builds - name: Cache Gradle @@ -112,26 +115,6 @@ jobs: # Set NDK path echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/29.0.14206865" >> $GITHUB_ENV - - name: Install gomobile - working-directory: go_backend - run: | - # Installed from inside the module so the go.mod-pinned x/mobile - # version is used (reproducible + covered by the setup-go cache). - go install golang.org/x/mobile/cmd/gomobile - gomobile init - - - name: Build Go backend for Android - working-directory: go_backend - run: | - mkdir -p ../android/app/libs - # arm/arm64 only: ndk.abiFilters in app/build.gradle.kts strips - # every other ABI from all outputs (universal APK included), so an - # amd64 slice would never reach a shipped APK — it only bloats the - # aar and slows this step. - gomobile bind -target=android/arm,android/arm64 -androidapi 24 -o ../android/app/libs/gobackend.aar . - env: - CGO_ENABLED: 1 - - name: Setup Flutter uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2 with: @@ -154,7 +137,7 @@ jobs: - name: Build APK (Release - unsigned) run: | - bash scripts/build_android.sh + bash scripts/build_android.sh --target lib/main.dart ls -la build/app/outputs/flutter-apk/ - name: Sign APKs @@ -167,7 +150,7 @@ jobs: keyStorePassword: ${{ secrets.KEYSTORE_PASSWORD }} keyPassword: ${{ secrets.KEY_PASSWORD }} env: - BUILD_TOOLS_VERSION: "36.0.0" + BUILD_TOOLS_VERSION: "37.0.0" - name: Rename APKs env: @@ -209,6 +192,20 @@ jobs: done ls -la + - name: Audit signed Rust APKs before upload + env: + VERSION: ${{ needs.get-version.outputs.version }} + run: | + apk_dir=build/app/outputs/flutter-apk + python3 scripts/check_backend_apk.py "$apk_dir/SpotiFLAC-${VERSION}-arm64.apk" \ + --backend rust --abis arm64-v8a + python3 scripts/check_backend_apk.py "$apk_dir/SpotiFLAC-${VERSION}-arm32.apk" \ + --backend rust --abis armeabi-v7a + for abi in arm64 arm32; do + "$ANDROID_HOME/build-tools/37.0.0/zipalign" -c -P 16 4 \ + "$apk_dir/SpotiFLAC-${VERSION}-${abi}.apk" + done + - name: Upload APK artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: @@ -228,24 +225,16 @@ jobs: sudo xcode-select -s /Applications/Xcode_26.1.1.app xcodebuild -version - - name: Setup Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 - with: - go-version-file: go_backend/go.mod - cache-dependency-path: go_backend/go.sum + - name: Build Rust backend for iOS + run: bash scripts/build_ios.sh # Cache CocoaPods - name: Cache CocoaPods uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: path: ios/Pods - key: pods-${{ runner.os }}-${{ hashFiles('ios/Podfile.lock') }} - restore-keys: pods-${{ runner.os }}- - - - name: Build Go backend for iOS - run: bash scripts/build_ios.sh - env: - CGO_ENABLED: 1 + key: pods-rust-${{ runner.os }}-${{ hashFiles('ios/Podfile', 'ios/Podfile.lock', 'rust_backend/SpotiFLACBackend.podspec', 'rust_backend/Cargo.lock') }} + restore-keys: pods-rust-${{ runner.os }}- - name: Setup Flutter uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2 @@ -276,10 +265,14 @@ jobs: - name: Generate app icons run: dart run flutter_launcher_icons + - name: Configure Rust application pods + working-directory: ios + run: pod install + - name: Build iOS (unsigned) run: | # Build Flutter iOS without codesigning - flutter build ios --release --no-codesign --config-only \ + flutter build ios --release --no-codesign --config-only --target lib/main.dart \ --dart-define="GIT_COMMIT=$(git rev-parse --short=8 HEAD)" # Use xcodebuild with code signing disabled @@ -296,6 +289,12 @@ jobs: CODE_SIGN_IDENTITY="" \ DEVELOPMENT_TEAM="" + - name: Audit Rust iOS archive before packaging + run: | + python3 scripts/check_backend_ios.py \ + ios/build/Runner.xcarchive/Products/Applications/Runner.app \ + --backend rust --archs arm64 --platform ios --release + - name: Create IPA env: VERSION: ${{ needs.get-version.outputs.version }} diff --git a/.gitignore b/.gitignore index a790be83..d9e68432 100644 --- a/.gitignore +++ b/.gitignore @@ -37,11 +37,8 @@ AGENTS.md /bun.lock /package-lock.json -# Go backend build outputs -/go_backend/*.aar -/go_backend/*.jar -/go_backend/*.exe -/go_backend/*.xcframework/ +# Rust backend build outputs and generated native bindings +/rust_backend/target # Android build state and signing material /android/.gradle/ @@ -57,6 +54,7 @@ AGENTS.md /android/app/*.jks # iOS generated state +/ios/build/ /ios/Frameworks/ /ios/Pods/ /ios/.symlinks/ @@ -73,6 +71,8 @@ DerivedData/ *.mobileprovision *.p12 *.pem +# Public ISRG roots embedded by the Rust network library. +!/rust_backend/crates/network/src/roots.pem # Local research and design material /design/ @@ -81,10 +81,8 @@ DerivedData/ /tool/ /spotiflac_android/ -# Documentation published separately. Keep the in-repo extension contract -# available because contribution templates link to it. -/docs/* -!/docs/EXTENSION_DEVELOPMENT.md +# Local documentation, migration archives, fixtures, and benchmark evidence. +/docs/ # Temporary files and logs /.tmp/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b83915d0..7b708158 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,7 +25,7 @@ The repository is the source of truth for tool versions: - Flutter: `.fvmrc` - Dart: bundled with the pinned Flutter SDK -- Go: `go_backend/go.mod` +- Rust: `rust_backend/rust-toolchain.toml` and `rust_backend/Cargo.lock` - Android SDK, NDK, and Java: `.github/workflows/ci.yml` - Xcode: required only for iOS builds @@ -50,25 +50,11 @@ exact Flutter version declared in `.fvmrc` and replace `fvm flutter` with fvm flutter pub get ``` -3. Build the Go backend for Android. `ANDROID_NDK_HOME` must point to the NDK - version used by CI and `CGO_ENABLED` must be enabled. - - ```bash - cd go_backend - go mod download - go install golang.org/x/mobile/cmd/gomobile - gomobile init - mkdir -p ../android/app/libs - gomobile bind \ - -target=android/arm,android/arm64 \ - -androidapi 24 \ - -o ../android/app/libs/gobackend.aar \ - . - cd .. - ``` - - Running `go install` from `go_backend/` uses the `x/mobile` version pinned by - `go.mod`. Do not replace it with `@latest` in project scripts. +3. Install Rust with rustup, then run `(cd rust_backend && rustup show)` to + activate the pinned toolchain. Install Clang/libclang for the native bindings + (`libclang-dev` on Ubuntu, Xcode command-line tools on macOS). Android Gradle + builds the Rust libraries and bindings automatically. For a manual host build, + run `bash scripts/build_rust_backend.sh host`. 4. Run the app: @@ -76,8 +62,8 @@ exact Flutter version declared in `.fvmrc` and replace `fvm flutter` with fvm flutter run --dart-define="GIT_COMMIT=$(git rev-parse --short=8 HEAD)" ``` -For iOS, run `scripts/build_ios.sh` on macOS before opening -`ios/Runner.xcworkspace`. +For iOS, run `bash scripts/build_ios.sh` on macOS, then `(cd ios && pod install)` +before opening `ios/Runner.xcworkspace`. The application uses the Rust backend. The About footer shows the short commit supplied through `GIT_COMMIT` at compile time. The Android build script and iOS release workflow supply it @@ -88,20 +74,20 @@ commands directly; without it, the footer shows only the copyright. ```text lib/ Flutter UI, state, models, and platform orchestration -go_backend/ Download pipeline, extension runtime, and shared backend logic +rust_backend/ Production backend, native bindings, and unit tests android/ Android platform bridge and foreground worker ios/ iOS platform bridge and application project test/ Flutter unit and widget tests assets/ Images, fonts, and bundled resources -docs/ Contributor-facing technical contracts +docs/ Local documentation and migration archives (gitignored) scripts/ Reproducible project build helpers ``` SpotiFLAC Mobile is extension-driven. Extension-specific behavior must be declared through a generic manifest field, capability, or reusable app API. Do not add provider-name checks such as `if source == 'provider-name'` to the -main app. The Go backend should parse and expose the generic declaration, and -Dart should consume that declaration without knowing which extension uses it. +main app. The backend should parse and expose the generic declaration, and Dart +should consume that declaration without knowing which extension uses it. ## Generated Files @@ -118,8 +104,9 @@ Run checks that cover the code you changed. Before opening a PR, the relevant commands should pass. Cross-language lyric usability cases live in -`android/app/src/test/resources/lyrics_usability_cases.tsv`. Dart, Go, and -Android tests read the same cases; add a case there when changing that policy. +`android/app/src/test/resources/lyrics_usability_cases.tsv`. Dart and Android +tests read the same cases; add a case there when changing +that policy. Flutter and Dart: @@ -129,16 +116,13 @@ fvm flutter analyze fvm flutter test ``` -Go backend: +Rust formatting, Clippy, and unit tests: ```bash -cd go_backend -gofmt -w . -go vet ./... -go test ./... +bash scripts/check_rust_backend.sh ``` -Android native code, after building `gobackend.aar`: +Android native code (Gradle builds the Rust artifacts automatically): ```bash cd android diff --git a/README.md b/README.md index c3715f3e..e085715c 100644 --- a/README.md +++ b/README.md @@ -51,19 +51,18 @@ Extensions let the community add new music sources and features without waiting ### Developing Extensions > [!NOTE] -> Want to build your own extension? Start with the -> [Extension Development Guide](docs/EXTENSION_DEVELOPMENT.md). The -> [documentation site](https://spotiflac.zarz.moe/docs) contains the expanded -> API reference. +> Want to build your own extension? The +> [documentation site](https://spotiflac.zarz.moe/docs) contains the API reference. --- ## Development -SpotiFLAC Mobile combines a Flutter/Dart UI, a Go backend compiled with -`gomobile`, and thin Android/iOS platform bridges. Toolchain versions are -pinned in [`.fvmrc`](.fvmrc), [`go_backend/go.mod`](go_backend/go.mod), and the -GitHub Actions workflows. +SpotiFLAC Mobile combines a Flutter/Dart UI, a Rust backend from +[`rust_backend/`](rust_backend/), and thin Android/iOS platform bridges. Android +Gradle builds the Rust native artifacts automatically. Tool versions are pinned in +[`rust_backend/rust-toolchain.toml`](rust_backend/rust-toolchain.toml) and +[`rust_backend/Cargo.lock`](rust_backend/Cargo.lock). Start with the [Contributing Guide](CONTRIBUTING.md) for the development setup, project boundaries, validation commands, and pull request checklist. diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 6780afe6..81a1de00 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -14,6 +14,19 @@ if (keystorePropertiesFile.exists()) { keystoreProperties.load(FileInputStream(keystorePropertiesFile)) } +val rustBackendDir = rootProject.file("../rust_backend") +val rustAndroidAbis = providers.environmentVariable("SPOTIFLAC_RUST_ANDROID_ABIS") + .orElse("arm64-v8a,armeabi-v7a") + .get() + .split(",") +val supportedRustAndroidAbis = setOf("arm64-v8a", "armeabi-v7a") +require(rustAndroidAbis.size == rustAndroidAbis.toSet().size) { + "SPOTIFLAC_RUST_ANDROID_ABIS must not contain duplicate ABIs" +} +require(rustAndroidAbis.all { it in supportedRustAndroidAbis }) { + "SPOTIFLAC_RUST_ANDROID_ABIS must contain only arm64-v8a and/or armeabi-v7a" +} + android { namespace = "com.zarz.spotiflac" compileSdk = 37 @@ -23,6 +36,12 @@ android { buildConfig = true } + sourceSets.getByName("main") { + java.srcDir("src/rust/kotlin") + java.srcDir(rustBackendDir.resolve("target/bindings/kotlin")) + jniLibs.srcDir(rustBackendDir.resolve("target/android/jniLibs")) + } + compileOptions { isCoreLibraryDesugaringEnabled = true sourceCompatibility = JavaVersion.VERSION_25 @@ -55,7 +74,8 @@ android { multiDexEnabled = true ndk { - abiFilters += listOf("arm64-v8a", "armeabi-v7a") + abiFilters.clear() + abiFilters += rustAndroidAbis } } @@ -99,26 +119,39 @@ android { abi { isEnable = true reset() - include("arm64-v8a", "armeabi-v7a") + include(*rustAndroidAbis.toTypedArray()) isUniversalApk = true // Also generate universal APK } } } +val buildRustBackend = tasks.register("buildRustBackend") { + workingDir(rootProject.projectDir.parentFile) + commandLine("bash", "scripts/build_rust_backend.sh", "android") + environment("SPOTIFLAC_RUST_ANDROID_ABIS", rustAndroidAbis.joinToString(",")) + environment( + "ANDROID_NDK_HOME", + System.getenv("ANDROID_NDK_HOME") + ?: android.sdkDirectory.resolve("ndk/29.0.14206865").absolutePath, + ) + inputs.files(fileTree(rustBackendDir) { + include("**/*.rs", "**/*.toml", "**/*.lock", "**/*.js", "**/*.tsv", "**/*.pem") + exclude("target/**", "smoke/**") + }) + inputs.property("SPOTIFLAC_RUST_ANDROID_ABIS", rustAndroidAbis.joinToString(",")) + inputs.file(rootProject.file("../scripts/build_rust_backend.sh")) + outputs.dir(rustBackendDir.resolve("target/bindings/kotlin")) + outputs.dir(rustBackendDir.resolve("target/android/jniLibs")) +} +tasks.named("preBuild").configure { dependsOn(buildRustBackend) } + flutter { source = "../.." } -repositories { - flatDir { - dirs("libs") - } -} - dependencies { coreLibraryDesugaring("com.android.tools:desugar_jdk_libs:2.1.5") - - implementation(fileTree(mapOf("dir" to "libs", "include" to listOf("*.jar", "*.aar")))) + implementation("net.java.dev.jna:jna:5.17.0@aar") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.11.0") implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.11.0") diff --git a/android/app/proguard-rules.pro b/android/app/proguard-rules.pro index caf24df5..5f65be57 100644 --- a/android/app/proguard-rules.pro +++ b/android/app/proguard-rules.pro @@ -6,7 +6,18 @@ # Ignore missing javax.xml.stream (not used on Android) -dontwarn javax.xml.stream.** -# The Go backend and FFmpeg plugin ship their JNI keep rules in their AARs. +# The FFmpeg plugin ships its JNI keep rules in its AAR. + +# UniFFI/JNA resolves FFI methods and structure fields by their generated names. +# Preserve the generated Rust bindings used by JNA. +-keep class com.sun.jna.** { *; } +-keep class com.spotiflac.backend.** { *; } +# JNA's desktop AWT helpers are unavailable and unused on Android. +# https://github.com/java-native-access/jna/blob/master/www/FrequentlyAskedQuestions.md#jna-on-android +-dontwarn java.awt.Component +-dontwarn java.awt.GraphicsEnvironment +-dontwarn java.awt.HeadlessException +-dontwarn java.awt.Window # Keep native methods -keepclasseswithmembernames class * { diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/CoreBackend.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/CoreBackend.kt new file mode 100644 index 00000000..b75da870 --- /dev/null +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/CoreBackend.kt @@ -0,0 +1,243 @@ +package com.zarz.spotiflac + +import android.content.Context +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.nio.file.Files +import java.nio.file.LinkOption +import java.util.UUID +import java.util.concurrent.CancellationException +import java.util.concurrent.atomic.AtomicBoolean + +internal interface CoreDownloadProgress : AutoCloseable { + fun waitDelta(since: Long, timeoutMs: Long): String +} + +internal data class CoreFFmpegCommand( + val id: String, + val arguments: Array, + val outputPath: String = "", +) + +internal interface CoreExtensionExecution : AutoCloseable { + fun download(requestJson: String): String + fun postProcess(inputJson: String, metadataJson: String): String + fun waitPending(timeoutMs: Long): List + fun commandIsActive(commandId: String): Boolean + fun complete(commandId: String, success: Boolean, output: String, error: String) +} + +internal fun parseCoreFFmpegCommands(raw: String): List { + val commands = JSONArray(raw) + return (0 until commands.length()).mapNotNull { index -> + val command = commands.optJSONObject(index) ?: return@mapNotNull null + val id = command.optString("command_id", "") + if (id.isBlank()) return@mapNotNull null + val arguments = command.optJSONArray("arguments") + CoreFFmpegCommand( + id, + Array(arguments?.length() ?: 0) { arguments!!.optString(it, "") }, + command.optString("output_path", ""), + ) + } +} + +private object CoreFFmpegStaging { + private val directories = mutableSetOf() + private val prefix = ".spotiflac-ffmpeg-${BuildConfig.APPLICATION_ID}-" + + @Synchronized + fun create(target: File): File { + val directory = requireNotNull(target.parentFile).canonicalFile + // Sweep only before this process starts using the directory. Later + // commands can share it with an FFmpeg operation that is still active. + if (directory.path !in directories) { + val entries = checkNotNull(directory.listFiles()) { "Cannot read FFmpeg output directory" } + for (file in entries) { + if (!file.name.startsWith(prefix)) continue + val suffix = file.name.removePrefix(prefix) + if (suffix.length <= 36 || suffix[36] != '.') continue + if (runCatching { UUID.fromString(suffix.take(36)) }.isFailure) continue + if (Files.isRegularFile(file.toPath(), LinkOption.NOFOLLOW_LINKS)) { + check(file.delete()) { "Failed to remove interrupted FFmpeg staging" } + } + } + directories.add(directory.path) + } + val staged = File(directory, "$prefix${UUID.randomUUID()}.${target.extension}") + check(staged.createNewFile()) { "Failed to create FFmpeg staging" } + return staged + } +} + +internal fun executeCoreFFmpegCommand( + command: CoreFFmpegCommand, + cancelled: () -> Boolean, + execute: (Array, () -> Boolean) -> Pair, +): Pair { + if (cancelled()) return false to "cancelled" + if (command.outputPath.isEmpty()) return execute(command.arguments, cancelled) + require(command.arguments.lastOrNull() == command.outputPath) { "FFmpeg output does not match command" } + val target = File(command.outputPath) + val staged = CoreFFmpegStaging.create(target) + try { + val arguments = command.arguments.copyOf() + arguments[arguments.lastIndex] = staged.absolutePath + val result = execute(arguments, cancelled) + if (cancelled()) return false to "cancelled" + if (!result.first) return result + // Rename within the same filesystem publishes only a completed file. + // An interrupted FFmpeg process can never become an existing-library hit. + check(staged.renameTo(target)) { "Failed to publish FFmpeg output" } + return result + } finally { + staged.delete() + } +} + +internal fun withCoreFFmpegExecution( + execution: CoreExtensionExecution, + execute: (Array, () -> Boolean) -> Pair = { arguments, cancelled -> + NativeDownloadFinalizer.runFFmpegArguments(arguments, cancelled, trackFinalizerSession = false) + }, + block: (CoreExtensionExecution) -> String, +): String { + val running = AtomicBoolean(true) + val pump = Thread { + try { + while (running.get()) { + val commands = try { + execution.waitPending(1_000L) + } catch (_: Exception) { + break + } + // Finish every claimed command, even if our caller finishes first. + // Another operation on this owner may be waiting for its result. + for (command in commands) { + val cancelled = { + try { !execution.commandIsActive(command.id) } catch (_: Exception) { true } + } + val result = try { + when { + cancelled() -> false to "cancelled" + command.arguments.isEmpty() -> false to "FFmpeg arguments are empty" + else -> executeCoreFFmpegCommand(command, cancelled, execute) + } + } catch (error: Exception) { + false to (error.message ?: "FFmpeg execution failed") + } + try { + execution.complete(command.id, result.first, result.second, if (result.first) "" else result.second) + } catch (_: Exception) { + // Owner shutdown removes commands; late results cannot revive them. + } + } + } + } finally { + execution.close() + } + } + pump.isDaemon = true + try { + pump.start() + } catch (error: Throwable) { + execution.close() + throw error + } + return try { + block(execution) + } catch (error: Exception) { + // Both bindings expose the backend's cancellation sentinel as an error. + // Preserve its meaning for the worker's pause/retry state machine. + if (error.message == "download cancelled") { + throw CancellationException("download cancelled").apply { initCause(error) } + } + throw error + } finally { + // Do not interrupt a claimed command belonging to another operation. + running.set(false) + } +} + +/** Native migration boundary; each process selects one stateful backend. */ +internal interface CoreBackend { + val implementation: String + val routesApplication: Boolean get() = false + val supportsOutputDescriptors: Boolean get() = false + fun invokeApplication(method: String, arguments: Any?): Any? = + error("Application routing is unavailable for $method") + fun cleanupExtensions() { + invokeApplication("cleanupExtensions", null) + } + fun setRuntimeState(dataDirectory: String, payload: String) { + invokeApplication( + "prepareRuntimeState", + mapOf("data_dir" to dataDirectory, "runtime_state" to payload), + ) + } + fun completeAuthCallback(state: String, code: String, sessionGrant: Boolean, onResolved: (String) -> Unit) + fun downloadByStrategy(requestJson: String): String = + withCoreFFmpegExecution(openExtensionExecution()) { it.download(requestJson) } + fun waitForDownloadProgressDelta(since: Long, timeoutMs: Long): String + fun openDownloadProgress(): CoreDownloadProgress = object : CoreDownloadProgress { + override fun waitDelta(since: Long, timeoutMs: Long) = waitForDownloadProgressDelta(since, timeoutMs) + override fun close() {} + } + fun initItemProgress(itemId: String) + fun clearItemProgress(itemId: String) + fun cancelDownload(itemId: String) + fun resetDownloadCancel(itemId: String) + fun openExtensionExecution(): CoreExtensionExecution + fun runPostProcessing(inputJson: String, metadataJson: String): String = + withCoreFFmpegExecution(openExtensionExecution()) { it.postProcess(inputJson, metadataJson) } + fun buildFilename(template: String, metadataJson: String): String + fun sanitizeFilename(filename: String): String + fun fileMetadataImplementation(path: String): String + fun readFileMetadata(path: String, hint: String): String + fun readAudioMetadata(path: String, hint: String, cacheKey: String): String + fun setLibraryCoverCacheDirectory(path: String) + fun scanLibraryFolder(folder: String): String + fun scanLibraryFolderToNdjsonFile(folder: String, output: String): Long + fun scanLibraryFolderIncremental(folder: String, existing: String): String + fun scanLibraryFolderIncrementalFromSnapshot(folder: String, snapshot: String): String + fun getLibraryScanProgress(): String + fun cancelLibraryScan() + fun parseCueSheet(path: String, audioDirectory: String): String + fun scanCueForLibrary(path: String, audioDirectory: String, virtualPrefix: String, modTime: Long, cacheKey: String): String + fun editFileMetadata(path: String, metadataJson: String): String + fun reEnrichFile(requestJson: String): String + fun rewriteSplitArtistTags(path: String, artist: String, albumArtist: String): String + fun extractCoverToFile(audioPath: String, outputPath: String) + fun writeM4aFreeformTags(path: String, metadataJson: String): String + fun ensureAc4Config(path: String, reference: String): String + fun writeAc4Metadata(path: String, metadataJson: String, coverPath: String): String + fun getLyricsLrc(spotifyId: String, trackName: String, artistName: String, filePath: String, durationMs: Long): String + fun downloadCoverToFileSized(url: String, outputPath: String, maxDimension: Long) + fun createTemporaryMediaFile(context: Context, prefix: String, suffix: String): File + fun openDownloadDirectory(path: String): AutoCloseable + fun openDownloadDirectoryForRequest(requestJson: String): AutoCloseable { + val request = JSONObject(requestJson) + return if (request.optString("storage_mode") == "saf") AutoCloseable {} + else openDownloadDirectory(request.getString("output_dir")) + } + fun releaseIdleResources() + fun releaseMemoryUnderPressure() +} + +internal fun requireSuccessfulExtensionAction(extensionId: String, actionName: String, response: String) { + val obj = try { + JSONObject(response) + } catch (e: Exception) { + throw IllegalStateException( + "Extension $actionName for $extensionId returned invalid JSON: ${response.take(240)}" + ) + } + if (obj.optBoolean("success", false)) { + return + } + val error = obj.optString("error") + .ifBlank { obj.optString("message") } + .ifBlank { response.take(240) } + throw IllegalStateException("Extension $actionName failed for $extensionId: $error") +} diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadService.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadService.kt index 778d5108..5dc02bda 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadService.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadService.kt @@ -18,7 +18,6 @@ import android.os.PowerManager import android.os.SystemClock import android.util.AtomicFile import androidx.core.app.NotificationCompat -import gobackend.Gobackend import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -39,6 +38,7 @@ import org.json.JSONObject import java.io.File import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.atomic.AtomicReference /** * Foreground service to keep downloads running when app is in background. @@ -48,6 +48,7 @@ import java.util.concurrent.atomic.AtomicLong * The service will be stopped automatically after 6 hours of cumulative runtime in 24 hours. */ class DownloadService : Service() { + internal val coreBackend: CoreBackend by lazy { createCoreBackend(applicationContext) } companion object { private const val CHANNEL_ID = "download_channel" @@ -368,11 +369,14 @@ class DownloadService : Service() { internal var nativeWorkerProgressSeq = 0L internal val nativeWorkerProgressEpoch = AtomicLong(0L) @Volatile internal var nativeWorkerProgressJob: Job? = null + @Volatile internal var nativeWorkerProgressConnection: AtomicReference? = null internal val snapshotWriteLock = Any() internal val snapshotWriteSerial = AtomicLong(0L) internal var latestCommittedStateSnapshotSerial = 0L internal var latestCommittedProgressSnapshotSerial = 0L @Volatile private var nativeWorkerPaused = false + private val nativeWorkerPausePendingIds = mutableSetOf() + private var nativeWorkerResumePending = false @Volatile internal var nativeWorkerNetworkPaused = false @Volatile internal var nativeWorkerVerificationPaused = false @Volatile private var nativeWorkerCancelRequested = false @@ -383,7 +387,7 @@ class DownloadService : Service() { // Bumped every time a new native queue replaces the current one. A worker // coroutine that observes a different generation than its own must stop // without touching the snapshot or the service lifecycle: cancel() alone - // cannot interrupt the blocking gomobile call it may be sitting in, and + // cannot interrupt the blocking native call it may be sitting in, and // the shared pause/cancel flags get reset for the new run. @Volatile private var nativeWorkerGeneration = 0L @@ -458,7 +462,10 @@ class DownloadService : Service() { ) } ACTION_PAUSE_NATIVE_QUEUE -> { - nativeWorkerPaused = true + synchronized(nativeWorkerItems) { + nativeWorkerResumePending = false + nativeWorkerPaused = true + } cancelActiveNativeItemForPause() writeNativeWorkerSnapshotAsync( isRunning = nativeWorkerJob?.isActive == true, @@ -469,7 +476,10 @@ class DownloadService : Service() { ) } ACTION_RESUME_NATIVE_QUEUE -> { - nativeWorkerPaused = false + synchronized(nativeWorkerItems) { + nativeWorkerResumePending = nativeWorkerPausePendingIds.isNotEmpty() + nativeWorkerPaused = nativeWorkerResumePending + } val stillPaused = isNativeWorkerPaused() writeNativeWorkerSnapshotAsync( isRunning = nativeWorkerJob?.isActive == true, @@ -481,6 +491,10 @@ class DownloadService : Service() { } ACTION_CANCEL_NATIVE_QUEUE -> { nativeWorkerCancelRequested = true + synchronized(nativeWorkerItems) { + nativeWorkerResumePending = false + nativeWorkerPausePendingIds.clear() + } nativeWorkerVerificationPaused = false nativeWorkerPreparationComplete = true nativeWorkerRequestChannel?.close() @@ -494,7 +508,7 @@ class DownloadService : Service() { ) { item.status = "skipped" try { - Gobackend.cancelDownload(item.itemId) + coreBackend.cancelDownload(item.itemId) } catch (_: Exception) { } } @@ -589,7 +603,7 @@ class DownloadService : Service() { } for (itemId in activeItemIds) { try { - Gobackend.cancelDownload(itemId) + coreBackend.cancelDownload(itemId) } catch (_: Exception) { } } @@ -689,7 +703,7 @@ class DownloadService : Service() { cancelNativeVerificationNotification() // Abort the previous run's in-flight work before the shared flags are // reset for the new run: the coroutine cancel below cannot interrupt a - // blocking gomobile download by itself. + // blocking native download by itself. synchronized(nativeWorkerItems) { for (item in nativeWorkerItems) { if (item.status == "preparing" || @@ -697,7 +711,7 @@ class DownloadService : Service() { item.status == "finalizing" ) { try { - Gobackend.cancelDownload(item.itemId) + coreBackend.cancelDownload(item.itemId) } catch (_: Exception) { } } @@ -710,7 +724,11 @@ class DownloadService : Service() { nativeWorkerGeneration++ val generation = nativeWorkerGeneration nativeWorkerJob?.cancel(CancellationException("Native queue replaced")) - nativeWorkerPaused = false + synchronized(nativeWorkerItems) { + nativeWorkerPaused = false + nativeWorkerResumePending = false + nativeWorkerPausePendingIds.clear() + } nativeWorkerNetworkPaused = false nativeWorkerVerificationPaused = false nativeWorkerCancelRequested = false @@ -1016,10 +1034,13 @@ class DownloadService : Service() { nativeWorkerItems .filter { it.itemId != excludeItemId && - (it.status == "downloading" || + it.itemId !in nativeWorkerPausePendingIds && + (it.status == "preparing" || + it.status == "downloading" || it.status == "finalizing") } .map { item -> + nativeWorkerPausePendingIds.add(item.itemId) item.status = "queued" item.progress = 0.0 item.bytesReceived = 0L @@ -1030,7 +1051,7 @@ class DownloadService : Service() { } for (itemId in ids) { try { - Gobackend.cancelDownload(itemId) + coreBackend.cancelDownload(itemId) } catch (_: Exception) { } } @@ -1039,6 +1060,19 @@ class DownloadService : Service() { } } + private fun finishNativePauseCancellation(itemId: String, generation: Long) { + synchronized(nativeWorkerItems) { + if (generation != nativeWorkerGeneration) return + nativeWorkerPausePendingIds.remove(itemId) + // Resume may arrive before the blocking download unwinds. Keep the + // pause reason until every cancelled attempt has decided to retry. + if (nativeWorkerResumePending && nativeWorkerPausePendingIds.isEmpty()) { + nativeWorkerResumePending = false + nativeWorkerPaused = false + } + } + } + private suspend fun processConcurrentNativeRequest( request: NativeDownloadRequest, settingsJson: String, @@ -1059,7 +1093,9 @@ class DownloadService : Service() { var progressInitialized = false var retryCurrentRequest = false + var directoryScope: AutoCloseable? = null try { + directoryScope = coreBackend.openDownloadDirectoryForRequest(request.requestJson) // Acquire the provider permit first. If several requests from // one provider are queued, they must not occupy every global // network slot while waiting for that provider's lower limit. @@ -1094,20 +1130,18 @@ class DownloadService : Service() { settingsJson = settingsJson, includeItems = true, ) - Gobackend.initItemProgress(request.itemId) + coreBackend.initItemProgress(request.itemId) progressInitialized = true currentStatus = "downloading" updateNativeWorkerItem(request.itemId) { it.status = "downloading" } try { - SafDownloadHandler.handle(this, request.requestJson) { json -> - Gobackend.downloadByStrategy(json) - } + SafDownloadHandler.handle(this, request.requestJson, coreBackend) } finally { updateNativeWorkerItemProgress(request.itemId) try { - Gobackend.clearItemProgress(request.itemId) + coreBackend.clearItemProgress(request.itemId) } catch (_: Exception) { } progressInitialized = false @@ -1169,22 +1203,6 @@ class DownloadService : Service() { } else { val errorType = result.optString("error_type") val errorMessage = result.optString("error") - if (errorType == "cancelled" && - !isNativeWorkerPaused() && - !nativeWorkerCancelRequested && - generation == nativeWorkerGeneration - ) { - var waitedMs = 0L - while (waitedMs < 1500 && - !isNativeWorkerPaused() && - !nativeWorkerCancelRequested && - generation == nativeWorkerGeneration - ) { - delay(100) - waitedMs += 100 - } - } - if (errorType == "cancelled" && isNativeWorkerPaused() && !nativeWorkerCancelRequested @@ -1315,10 +1333,12 @@ class DownloadService : Service() { includeItems = true, ) } finally { + directoryScope?.close() + finishNativePauseCancellation(request.itemId, generation) if (progressInitialized) { updateNativeWorkerItemProgress(request.itemId) try { - Gobackend.clearItemProgress(request.itemId) + coreBackend.clearItemProgress(request.itemId) } catch (_: Exception) { } } @@ -1466,11 +1486,11 @@ class DownloadService : Service() { includeItems = true ) + var directoryScope: AutoCloseable? = null try { - Gobackend.initItemProgress(request.itemId) - val response = SafDownloadHandler.handle(this, request.requestJson) { json -> - Gobackend.downloadByStrategy(json) - } + directoryScope = coreBackend.openDownloadDirectoryForRequest(request.requestJson) + coreBackend.initItemProgress(request.itemId) + val response = SafDownloadHandler.handle(this, request.requestJson, coreBackend) if (generation != nativeWorkerGeneration) { // Superseded while blocked in the download call; the // new run owns the shared state now. @@ -1521,27 +1541,6 @@ class DownloadService : Service() { } else { val errorType = result.optString("error_type") val errorMessage = result.optString("error") - if (errorType == "cancelled" && - !isNativeWorkerPaused() && - !nativeWorkerCancelRequested && - generation == nativeWorkerGeneration - ) { - // A pause from Dart cancels the in-flight Go - // download directly but delivers the pause flag - // via a startService intent through the main - // looper; the download can unwind first. Give the - // flag a moment to settle before classifying this - // cancellation as a permanent skip. - var waitedMs = 0L - while (waitedMs < 1500 && - !isNativeWorkerPaused() && - !nativeWorkerCancelRequested && - generation == nativeWorkerGeneration - ) { - delay(100) - waitedMs += 100 - } - } if (errorType == "cancelled" && isNativeWorkerPaused() && !nativeWorkerCancelRequested @@ -1675,9 +1674,11 @@ class DownloadService : Service() { includeItems = true ) } finally { + directoryScope?.close() + finishNativePauseCancellation(request.itemId, generation) updateNativeWorkerItemProgress(request.itemId) try { - Gobackend.clearItemProgress(request.itemId) + coreBackend.clearItemProgress(request.itemId) } catch (_: Exception) { } } @@ -1722,9 +1723,9 @@ class DownloadService : Service() { private fun releaseIdleDownloadMemory() { try { - // All workers and album tagging have finished. Return unused Go - // heap without forcing a collection between individual tracks. - Gobackend.releaseMemory() + // All workers and album tagging have finished. Release idle backend + // resources once for the batch, preserving caches between tracks. + coreBackend.releaseIdleResources() } catch (e: Exception) { android.util.Log.w("SpotiFLAC", "Failed to release idle download memory: ${e.message}") } diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceNetwork.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceNetwork.kt index 887ba853..ff22f6f3 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceNetwork.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceNetwork.kt @@ -17,7 +17,6 @@ import android.os.IBinder import android.os.PowerManager import android.util.AtomicFile import androidx.core.app.NotificationCompat -import gobackend.Gobackend import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceReplayGain.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceReplayGain.kt index 98e69bd1..bcab79d2 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceReplayGain.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceReplayGain.kt @@ -17,7 +17,6 @@ import android.os.IBinder import android.os.PowerManager import android.util.AtomicFile import androidx.core.app.NotificationCompat -import gobackend.Gobackend import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceSnapshot.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceSnapshot.kt index 3925e1b2..751bdad2 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceSnapshot.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/DownloadServiceSnapshot.kt @@ -17,7 +17,6 @@ import android.os.IBinder import android.os.PowerManager import android.util.AtomicFile import androidx.core.app.NotificationCompat -import gobackend.Gobackend import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -31,6 +30,7 @@ import org.json.JSONArray import org.json.JSONObject import java.io.File import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.atomic.AtomicReference // Native-worker item state snapshots for the Flutter side. @@ -243,24 +243,26 @@ internal fun DownloadService.updateNativeWorkerItem(itemId: String, updater: (Do } /** - * Polls the exported Go delta API once for the whole native queue. Workers + * Polls the selected backend delta API once for the whole native queue. Workers * update their item state from [nativeWorkerProgressItems] instead of making * independent full-payload calls. The generation check prevents a delayed - * gomobile call from an old queue from contaminating a replacement queue. + * native call from an old queue from contaminating a replacement queue. */ internal fun DownloadService.startNativeWorkerProgressCoordinator(generation: Long): Job { - nativeWorkerProgressJob?.cancel() + stopNativeWorkerProgressCoordinator() val coordinatorEpoch = nativeWorkerProgressEpoch.incrementAndGet() synchronized(nativeWorkerProgressLock) { nativeWorkerProgressItems.clear() nativeWorkerProgressSeq = 0L } + val connection = AtomicReference(null) + nativeWorkerProgressConnection = connection val job = serviceScope.launch { val lastSignatures = mutableMapOf() while (isActive && isNativeWorkerProgressActive(generation)) { maintainNativeWorkerWakeLock() - val changedItemIds = pollNativeWorkerProgress(generation) + val changedItemIds = pollNativeWorkerProgress(generation, connection) val snapshotItemIds = mutableListOf() for (itemId in changedItemIds) { if (!updateNativeWorkerItemProgress(itemId, emitNotification = false)) { @@ -305,6 +307,7 @@ internal fun DownloadService.startNativeWorkerProgressCoordinator(generation: Lo delay(1000) } } + job.invokeOnCompletion { connection.getAndSet(null)?.close() } nativeWorkerProgressJob = job return job } @@ -313,6 +316,8 @@ internal fun DownloadService.stopNativeWorkerProgressCoordinator(job: Job? = nat if (job == null) return if (nativeWorkerProgressJob === job) { nativeWorkerProgressJob = null + nativeWorkerProgressConnection?.getAndSet(null)?.close() + nativeWorkerProgressConnection = null nativeWorkerProgressEpoch.incrementAndGet() } job.cancel() @@ -326,14 +331,29 @@ internal fun DownloadService.cancelNativeWorkerProgressCoordinator() { } } -private fun DownloadService.pollNativeWorkerProgress(generation: Long): Set { +private fun DownloadService.pollNativeWorkerProgress( + generation: Long, + connection: AtomicReference, +): Set { val sinceSeq = synchronized(nativeWorkerProgressLock) { nativeWorkerProgressSeq } val raw = try { - Gobackend.waitForAllDownloadProgressDelta(sinceSeq, 5_000L) + val reader = connection.get() ?: coreBackend.openDownloadProgress().also { connection.set(it) } + if (nativeWorkerProgressConnection !== connection || !isNativeWorkerProgressActive(generation)) { + connection.getAndSet(null)?.close() + return emptySet() + } + reader.waitDelta(sinceSeq, 5_000L) } catch (_: Exception) { + connection.getAndSet(null)?.close() + synchronized(nativeWorkerProgressLock) { + if (nativeWorkerProgressConnection === connection) { + nativeWorkerProgressItems.clear() + nativeWorkerProgressSeq = 0L + } + } return emptySet() } - if (raw.isBlank() || !isNativeWorkerProgressActive(generation)) return emptySet() + if (raw.isBlank() || nativeWorkerProgressConnection !== connection || !isNativeWorkerProgressActive(generation)) return emptySet() return try { val root = JSONObject(raw) @@ -366,11 +386,11 @@ private fun DownloadService.pollNativeWorkerProgress(generation: Long): Set nativeWorkerProgressSeq) { + if (reset || nextSeq > nativeWorkerProgressSeq) { nativeWorkerProgressSeq = nextSeq } } diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivity.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivity.kt index 6ef7f684..554cf8be 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivity.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivity.kt @@ -24,12 +24,12 @@ import io.flutter.embedding.engine.FlutterShellArgs import io.flutter.plugin.common.EventChannel import io.flutter.plugin.common.MethodChannel import com.ryanheise.audioservice.AudioServicePlugin -import gobackend.Gobackend import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -44,6 +44,7 @@ import java.security.SecureRandom import java.util.LinkedHashMap import java.util.Locale import java.util.UUID +import java.util.concurrent.atomic.AtomicReference class MainActivity: FlutterFragmentActivity() { // Mirrors audio_service's AudioServiceFragmentActivity: the shared engine @@ -76,6 +77,73 @@ class MainActivity: FlutterFragmentActivity() { private val LARGE_JSON_RESULT_FILE_THRESHOLD_BYTES = 256 * 1024 private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main) private var backendChannel: MethodChannel? = null + internal val coreBackend: CoreBackend by lazy { createCoreBackend(applicationContext) } + private val nativeBackendMethods = setOf( + "getBackendImplementations", + "ensureInstallMarker", + "prepareRuntimeState", + "downloadByStrategy", + "runPostProcessingV2", + "readAudioMetadata", + "readFileMetadata", + "editFileMetadata", + "reEnrichFile", + "setLibraryCoverCacheDir", + "scanLibraryFolder", + "scanLibraryFolderToNDJSONFile", + "scanLibraryFolderIncremental", + "scanLibraryFolderIncrementalFromSnapshot", + "scanSafTree", + "scanSafTreeToNDJSONFile", + "scanSafTreeIncremental", + "scanSafTreeIncrementalFromSnapshot", + "getLibraryScanProgress", + "cancelLibraryScan", + "parseCueSheet", + "pickSafTree", + "safExists", + "safExistsBatch", + "isSafTreeAccessible", + "safDelete", + "safStat", + "resolveSafFile", + "inspectSafFiles", + "safCopyToTemp", + "safOpenPlaybackLease", + "safClosePlaybackLease", + "openContentUri", + "shareContentUri", + "shareMultipleContentUris", + "getSafFileModTimes", + "safCreateFromPath", + "safCreateIfAbsentFromPath", + "safCreateUniqueFromPath", + "safCreateCollisionAwareFromPath", + "writeTempToSaf", + "writeSafSidecarLrc", + "downloadCoverToFile", + "extractCoverToFile", + "rewriteSplitArtistTags", + "writeM4AFreeformTags", + "ensureAC4Config", + "writeAC4Metadata", + "releaseMemory", + "releaseMemoryUnderPressure", + "startDownloadService", + "stopDownloadService", + "updateDownloadServiceProgress", + "isDownloadServiceRunning", + "startNativeDownloadWorker", + "appendNativeDownloadWorkerRequests", + "finishNativeDownloadWorkerPreparation", + "acknowledgeNativeDownloadWorkerItems", + "pauseNativeDownloadWorker", + "resumeNativeDownloadWorker", + "cancelNativeDownloadWorker", + "getNativeDownloadWorkerSnapshot", + "consumeVerificationNotification", + "exitApp", + ) private var libraryStorageReceiver: BroadcastReceiver? = null private val pendingSessionGrantEvents = mutableListOf>() private var pendingVerificationNotification: String? = null @@ -83,6 +151,7 @@ class MainActivity: FlutterFragmentActivity() { internal val safScanLock = Any() internal var safScanProgress = SafScanProgress() private var downloadProgressStreamJob: Job? = null + private var downloadProgressConnection: AtomicReference? = null private var downloadProgressEventSink: EventChannel.EventSink? = null private var lastDownloadProgressPayload: String? = null private var lastDownloadProgressSeq = 0L @@ -403,7 +472,7 @@ class MainActivity: FlutterFragmentActivity() { } } - private fun parseJsonPayload(payload: String): Any { + internal fun parseJsonPayload(payload: String): Any { return try { parseJsonValue(JSONTokener(payload).nextValue()) ?: payload } catch (_: Exception) { @@ -467,8 +536,9 @@ class MainActivity: FlutterFragmentActivity() { private fun updateDownloadProgressSeq(payload: String) { try { - val seq = JSONObject(payload).optLong("seq", lastDownloadProgressSeq) - if (seq > lastDownloadProgressSeq) { + val objectValue = JSONObject(payload) + val seq = objectValue.optLong("seq", lastDownloadProgressSeq) + if (objectValue.optBoolean("reset", false) || seq > lastDownloadProgressSeq) { lastDownloadProgressSeq = seq } } catch (_: Exception) {} @@ -479,29 +549,38 @@ class MainActivity: FlutterFragmentActivity() { downloadProgressEventSink = sink lastDownloadProgressPayload = null lastDownloadProgressSeq = 0L + val connection = AtomicReference(null) + downloadProgressConnection = connection downloadProgressStreamJob = scope.launch { - while (isActive && downloadProgressEventSink === sink) { - try { - val payload = withContext(Dispatchers.IO) { - Gobackend.waitForAllDownloadProgressDelta( - lastDownloadProgressSeq, - 15_000L, + try { + while (isActive && downloadProgressConnection === connection) { + try { + val payload = withContext(Dispatchers.IO) { + val reader = connection.get() ?: coreBackend.openDownloadProgress().also { connection.set(it) } + ensureActive() + reader.waitDelta(lastDownloadProgressSeq, 15_000L) + } + if (!isActive || downloadProgressConnection !== connection) break + if (payload.isNotEmpty() && payload != lastDownloadProgressPayload) { + updateDownloadProgressSeq(payload) + lastDownloadProgressPayload = payload + sink.success(parseJsonPayload(payload)) + delay(250L) + } + } catch (e: Exception) { + connection.getAndSet(null)?.close() + if (!isActive || downloadProgressConnection !== connection) break + lastDownloadProgressSeq = 0L + lastDownloadProgressPayload = null + android.util.Log.w( + "SpotiFLAC", + "Download progress stream poll failed: ${e.message}", ) - } - if (!isActive || downloadProgressEventSink !== sink) break - if (payload.isNotEmpty() && payload != lastDownloadProgressPayload) { - updateDownloadProgressSeq(payload) - lastDownloadProgressPayload = payload - sink.success(parseJsonPayload(payload)) delay(250L) } - } catch (e: Exception) { - android.util.Log.w( - "SpotiFLAC", - "Download progress stream poll failed: ${e.message}", - ) } - if (downloadProgressEventSink !== sink) break + } finally { + connection.getAndSet(null)?.close() } } } @@ -509,6 +588,8 @@ class MainActivity: FlutterFragmentActivity() { private fun stopDownloadProgressStream() { downloadProgressStreamJob?.cancel() downloadProgressStreamJob = null + downloadProgressConnection?.getAndSet(null)?.close() + downloadProgressConnection = null downloadProgressEventSink = null lastDownloadProgressPayload = null lastDownloadProgressSeq = 0L @@ -722,7 +803,7 @@ class MainActivity: FlutterFragmentActivity() { /** * Deliver Spotify (or other) OAuth authorization code to the extension runtime * and run its token exchange (e.g. completeSpotifyLogin). State is a one-time - * host nonce resolved to the owning extension by the Go backend. + * host nonce resolved to the owning extension by the backend. */ private fun handleExtensionOAuthIntent(intent: Intent?) { val uri = intent?.data ?: return @@ -759,26 +840,13 @@ class MainActivity: FlutterFragmentActivity() { var callbackExtensionId = "" scope.launch(Dispatchers.IO) { try { - val extId = if (isSessionGrant) { - Gobackend.resolveExtensionCallbackState(callbackState) - } else { - Gobackend.consumeExtensionCallbackState(callbackState) - } - callbackExtensionId = extId - val json = if (isSessionGrant) { - Gobackend.setExtensionSessionGrantByID(extId, code) - Gobackend.invokeExtensionActionJSON(extId, "completeGrant") - } else { - Gobackend.setExtensionAuthCodeByID(extId, code) - Gobackend.invokeExtensionActionJSON(extId, "completeSpotifyLogin") - } - if (isSessionGrant) { - requireSuccessfulExtensionAction(extId, "completeGrant", json) + coreBackend.completeAuthCallback(callbackState, code, isSessionGrant) { + callbackExtensionId = it } android.util.Log.i("SpotiFLAC", "Extension callback completed") if (isSessionGrant) { withContext(Dispatchers.Main) { - notifySessionGrantCompleted(extId, true) + notifySessionGrantCompleted(callbackExtensionId, true) } } } catch (e: Exception) { @@ -792,23 +860,6 @@ class MainActivity: FlutterFragmentActivity() { } } - private fun requireSuccessfulExtensionAction(extensionId: String, actionName: String, response: String) { - val obj = try { - JSONObject(response) - } catch (e: Exception) { - throw IllegalStateException( - "Extension $actionName for $extensionId returned invalid JSON: ${response.take(240)}" - ) - } - if (obj.optBoolean("success", false)) { - return - } - val error = obj.optString("error") - .ifBlank { obj.optString("message") } - .ifBlank { response.take(240) } - throw IllegalStateException("Extension $actionName failed for $extensionId: $error") - } - private fun notifySessionGrantCompleted(extensionId: String, success: Boolean) { val payload = mapOf( "extension_id" to extensionId, @@ -886,7 +937,7 @@ class MainActivity: FlutterFragmentActivity() { } libraryStorageReceiver = null try { - Gobackend.cleanupExtensions() + coreBackend.cleanupExtensions() } catch (e: Exception) { android.util.Log.w("SpotiFLAC", "Failed to cleanup extensions on destroy: ${e.message}") } @@ -898,7 +949,8 @@ class MainActivity: FlutterFragmentActivity() { override fun configureFlutterEngine(flutterEngine: FlutterEngine) { super.configureFlutterEngine(flutterEngine) - Gobackend.setAppVersion(BuildConfig.VERSION_NAME) + // Select and initialize the runtime before Flutter can dispatch a call. + val selectedBackend = coreBackend sweepStaleCacheFiles() // Always-enabled back callback to ensure back presses reach Flutter. @@ -963,6 +1015,9 @@ class MainActivity: FlutterFragmentActivity() { channel.setMethodCallHandler { call, result -> scope.launch { try { + if (call.method !in nativeBackendMethods && dispatchBackendApplication(call, result)) { + return@launch + } when (call.method) { "consumeVerificationNotification" -> { val payload = pendingVerificationNotification @@ -982,7 +1037,7 @@ class MainActivity: FlutterFragmentActivity() { "Extension data directory is required" } val payload = prepareRuntimeState(dataDir) - Gobackend.setRuntimeState(payload) + selectedBackend.setRuntimeState(dataDir, payload) mapOf("ready" to true) } result.success(runtimeState) @@ -999,95 +1054,41 @@ class MainActivity: FlutterFragmentActivity() { "downloadByStrategy" -> { val requestJson = call.arguments as String val response = withContext(Dispatchers.IO) { - SafDownloadHandler.handle(this@MainActivity, requestJson) { json -> - Gobackend.downloadByStrategy(json) - } + SafDownloadHandler.handle(this@MainActivity, requestJson, coreBackend) } result.success(response) } - "getAllDownloadProgress" -> { - val response = withContext(Dispatchers.IO) { - Gobackend.getAllDownloadProgress() - } - result.success(parseJsonPayload(response)) - } - "clearItemProgress" -> { - val itemId = call.argument("item_id") ?: "" - withContext(Dispatchers.IO) { - Gobackend.clearItemProgress(itemId) - } - result.success(null) - } - "cancelDownload" -> { - val itemId = call.argument("item_id") ?: "" - withContext(Dispatchers.IO) { - Gobackend.cancelDownload(itemId) - } - result.success(null) - } - "resetDownloadCancel" -> { - val itemId = call.argument("item_id") ?: "" - withContext(Dispatchers.IO) { - Gobackend.resetDownloadCancel(itemId) - } - result.success(null) - } - "setDownloadDirectory" -> { + "acquireDownloadDirectory" -> { val path = call.argument("path") ?: "" withContext(Dispatchers.IO) { - Gobackend.setDownloadDirectory(path) + coreBackend.openDownloadDirectory(path).close() } - result.success(null) + // Go grants have process lifetime; Rust owns scoped tokens in its dispatcher. + result.success("") } - "setNetworkCompatibilityOptions", "setSongLinkNetworkOptions" -> { - val allowHttp = call.argument("allow_http") ?: false - val insecureTls = call.argument("insecure_tls") ?: false - withContext(Dispatchers.IO) { - Gobackend.setNetworkCompatibilityOptions(allowHttp, insecureTls) - } - result.success(null) - } - "setAllowPrivateNetwork" -> { - val allowed = call.argument("allowed") ?: false - withContext(Dispatchers.IO) { - Gobackend.setAllowPrivateNetwork(allowed) - } - result.success(null) - } - "checkDuplicatesBatch" -> { - val outputDir = call.argument("output_dir") ?: "" - val tracksJson = call.argument("tracks") ?: "[]" - val response = withContext(Dispatchers.IO) { - Gobackend.checkDuplicatesBatch(outputDir, tracksJson) - } - result.success(response) - } - "preBuildDuplicateIndex" -> { - val outputDir = call.argument("output_dir") ?: "" - withContext(Dispatchers.IO) { - Gobackend.preBuildDuplicateIndex(outputDir) - } - result.success(null) - } - "invalidateDuplicateIndex" -> { - val outputDir = call.argument("output_dir") ?: "" - withContext(Dispatchers.IO) { - Gobackend.invalidateDuplicateIndex(outputDir) - } - result.success(null) + "releaseDownloadDirectory" -> result.success(null) + "getBackendImplementations" -> { + result.success( + mapOf( + "filename" to coreBackend.implementation, + "file_metadata" to coreBackend.fileMetadataImplementation(call.argument("file_path") ?: ""), + "extensions" to if (coreBackend.routesApplication) "rust" else "go", + "downloads" to coreBackend.implementation, + ) + ) } "buildFilename" -> { val template = call.argument("template") ?: "" val metadata = call.argument("metadata") ?: "{}" val response = withContext(Dispatchers.IO) { - Gobackend.buildFilename(template, metadata) + coreBackend.buildFilename(template, metadata) } result.success(response) } "sanitizeFilename" -> { val filename = call.argument("filename") ?: "" val response = withContext(Dispatchers.IO) { - Gobackend.sanitizeFilename(filename) + coreBackend.sanitizeFilename(filename) } result.success(response) } @@ -1385,92 +1386,6 @@ class MainActivity: FlutterFragmentActivity() { result.error("share_failed", e.message, null) } } - "getLyricsLRC" -> { - val spotifyId = call.argument("spotify_id") ?: "" - val trackName = call.argument("track_name") ?: "" - val artistName = call.argument("artist_name") ?: "" - val filePath = call.argument("file_path") ?: "" - val durationMs = call.argument("duration_ms")?.toLong() ?: 0L - val response = withContext(Dispatchers.IO) { - if (filePath.startsWith("content://")) { - val tempPath = copyUriToTemp(Uri.parse(filePath)) - if (tempPath == null) { - "" - } else { - try { - Gobackend.getLyricsLRC(spotifyId, trackName, artistName, tempPath, durationMs) - } finally { - try { - File(tempPath).delete() - } catch (_: Exception) {} - } - } - } else { - Gobackend.getLyricsLRC(spotifyId, trackName, artistName, filePath, durationMs) - } - } - result.success(response) - } - "getLyricsLRCWithSource" -> { - val spotifyId = call.argument("spotify_id") ?: "" - val trackName = call.argument("track_name") ?: "" - val artistName = call.argument("artist_name") ?: "" - val filePath = call.argument("file_path") ?: "" - val durationMs = call.argument("duration_ms")?.toLong() ?: 0L - val response = withContext(Dispatchers.IO) { - if (filePath.startsWith("content://")) { - val tempPath = copyUriToTemp(Uri.parse(filePath)) - if (tempPath == null) { - """{"lyrics":"","source":"","sync_type":"","instrumental":false}""" - } else { - try { - Gobackend.getLyricsLRCWithSource(spotifyId, trackName, artistName, tempPath, durationMs) - } finally { - try { - File(tempPath).delete() - } catch (_: Exception) {} - } - } - } else { - Gobackend.getLyricsLRCWithSource(spotifyId, trackName, artistName, filePath, durationMs) - } - } - result.success(response) - } - "embedLyricsToFile" -> { - val filePath = call.argument("file_path") ?: "" - val lyrics = call.argument("lyrics") ?: "" - val response = withContext(Dispatchers.IO) { - if (filePath.startsWith("content://")) { - val uri = Uri.parse(filePath) - val tempPath = copyUriToTemp(uri, ".flac") - ?: return@withContext errorJson("Failed to copy SAF file to temp") - try { - val raw = Gobackend.embedLyricsToFile(tempPath, lyrics) - val obj = JSONObject(raw) - if (!obj.optBoolean("success", false)) { - return@withContext raw - } - - if (!writeUriFromPath(uri, tempPath)) { - return@withContext errorJson("Failed to write embedded lyrics back to SAF file") - } - - obj.put("file_path", filePath) - obj.toString() - } catch (e: Exception) { - errorJson("Failed to embed lyrics to SAF file: ${e.message}") - } finally { - try { - File(tempPath).delete() - } catch (_: Exception) {} - } - } else { - Gobackend.embedLyricsToFile(filePath, lyrics) - } - } - result.success(response) - } "rewriteSplitArtistTags" -> { val filePath = call.argument("file_path") ?: "" val artist = call.argument("artist") ?: "" @@ -1481,7 +1396,7 @@ class MainActivity: FlutterFragmentActivity() { val tempPath = copyUriToTemp(uri, ".flac") ?: return@withContext errorJson("Failed to copy SAF file to temp") try { - val raw = Gobackend.rewriteSplitArtistTagsExport(tempPath, artist, albumArtist) + val raw = coreBackend.rewriteSplitArtistTags(tempPath, artist, albumArtist) val obj = JSONObject(raw) if (!obj.optBoolean("success", false)) { return@withContext raw @@ -1501,17 +1416,11 @@ class MainActivity: FlutterFragmentActivity() { } catch (_: Exception) {} } } else { - Gobackend.rewriteSplitArtistTagsExport(filePath, artist, albumArtist) + coreBackend.rewriteSplitArtistTags(filePath, artist, albumArtist) } } result.success(response) } - "cleanupConnections" -> { - withContext(Dispatchers.IO) { - Gobackend.cleanupConnections() - } - result.success(null) - } "readFileMetadata" -> { val filePath = call.argument("file_path") ?: "" val displayName = call.argument("display_name") ?: "" @@ -1521,7 +1430,7 @@ class MainActivity: FlutterFragmentActivity() { readCompleteMetadataFromUri(Uri.parse(filePath), displayName) ?.toString() ?: errorJson("Failed to read SAF metadata") } else { - Gobackend.readFileMetadataWithHint(filePath, displayName) + coreBackend.readFileMetadata(filePath, displayName) } } catch (e: Exception) { errorJson(e.message ?: "Failed to read metadata") @@ -1539,7 +1448,7 @@ class MainActivity: FlutterFragmentActivity() { val tempPath = copyUriToTemp(uri) ?: return@withContext """{"error":"Failed to copy SAF file to temp"}""" try { - val raw = Gobackend.editFileMetadata(tempPath, metadataJson) + val raw = coreBackend.editFileMetadata(tempPath, metadataJson) val obj = JSONObject(raw) val method = obj.optString("method", "") if (method == "ffmpeg") { @@ -1549,7 +1458,7 @@ class MainActivity: FlutterFragmentActivity() { return@withContext obj.toString() // Note: temp file NOT deleted here - Dart will clean up after FFmpeg + writeTempToSaf } - // FLAC: Go wrote directly to temp, copy back now + // FLAC: the backend wrote to temp, copy back now if (!writeUriFromPath(uri, tempPath)) { try { File(tempPath).delete() } catch (_: Exception) {} return@withContext """{"error":"Failed to write metadata back to SAF file"}""" @@ -1561,7 +1470,7 @@ class MainActivity: FlutterFragmentActivity() { throw e } } else { - Gobackend.editFileMetadata(filePath, metadataJson) + coreBackend.editFileMetadata(filePath, metadataJson) } } catch (e: Exception) { android.util.Log.e("SpotiFLAC", "editFileMetadata failed: ${e.message}", e) @@ -1575,7 +1484,7 @@ class MainActivity: FlutterFragmentActivity() { val metadataJson = call.argument("metadata_json") ?: "{}" val response = withContext(Dispatchers.IO) { try { - Gobackend.writeM4AFreeformTags(filePath, metadataJson) + coreBackend.writeM4aFreeformTags(filePath, metadataJson) } catch (e: Exception) { android.util.Log.e("SpotiFLAC", "writeM4AFreeformTags failed: ${e.message}", e) """{"error":${org.json.JSONObject.quote(e.message ?: "unknown")}}""" @@ -1588,7 +1497,7 @@ class MainActivity: FlutterFragmentActivity() { val sourcePath = call.argument("source_path") ?: "" val response = withContext(Dispatchers.IO) { try { - Gobackend.ensureAC4Config(filePath, sourcePath) + coreBackend.ensureAc4Config(filePath, sourcePath) } catch (e: Exception) { android.util.Log.e("SpotiFLAC", "ensureAC4Config failed: ${e.message}", e) """{"error":${org.json.JSONObject.quote(e.message ?: "unknown")}}""" @@ -1602,7 +1511,7 @@ class MainActivity: FlutterFragmentActivity() { val coverPath = call.argument("cover_path") ?: "" val response = withContext(Dispatchers.IO) { try { - Gobackend.writeAC4Metadata(filePath, metadataJson, coverPath) + coreBackend.writeAc4Metadata(filePath, metadataJson, coverPath) } catch (e: Exception) { android.util.Log.e("SpotiFLAC", "writeAC4Metadata failed: ${e.message}", e) """{"error":${org.json.JSONObject.quote(e.message ?: "unknown")}}""" @@ -1652,14 +1561,28 @@ class MainActivity: FlutterFragmentActivity() { ?.coerceAtLeast(0L) ?: 0L val response = withContext(Dispatchers.IO) { + var temporaryCover: File? = null try { - Gobackend.downloadCoverToFileSized( + val destination = if (outputPath.isBlank()) { + coreBackend.createTemporaryMediaFile( + this@MainActivity, + "cover_", + ".jpg", + ).also { temporaryCover = it }.absolutePath + } else { + outputPath + } + coreBackend.downloadCoverToFileSized( coverUrl, - outputPath, + destination, maxDimension ) - """{"success":true}""" + JSONObject() + .put("success", true) + .put("file_path", destination) + .toString() } catch (e: Exception) { + temporaryCover?.delete() """{"success":false,"error":"${e.message?.replace("\"", "'")}"}""" } } @@ -1675,13 +1598,13 @@ class MainActivity: FlutterFragmentActivity() { val tempPath = copyUriToTemp(uri) ?: return@withContext """{"success":false,"error":"Failed to copy SAF file to temp"}""" try { - Gobackend.extractCoverToFile(tempPath, outputPath) + coreBackend.extractCoverToFile(tempPath, outputPath) """{"success":true}""" } finally { try { File(tempPath).delete() } catch (_: Exception) {} } } else { - Gobackend.extractCoverToFile(audioPath, outputPath) + coreBackend.extractCoverToFile(audioPath, outputPath) """{"success":true}""" } } catch (e: Exception) { @@ -1690,91 +1613,6 @@ class MainActivity: FlutterFragmentActivity() { } result.success(response) } - "fetchAndSaveLyrics" -> { - val trackName = call.argument("track_name") ?: "" - val artistName = call.argument("artist_name") ?: "" - val spotifyId = call.argument("spotify_id") ?: "" - val durationMs = call.argument("duration_ms")?.toLong() ?: 0L - val outputPath = call.argument("output_path") ?: "" - val rawAudioFilePath = call.argument("audio_file_path") ?: "" - val response = withContext(Dispatchers.IO) { - var safAudioTemp: String? = null - try { - // Resolve SAF content:// URI to a temp file the Go backend can read - val audioFilePath = if (rawAudioFilePath.startsWith("content://")) { - val uri = Uri.parse(rawAudioFilePath) - val tempPath = copyUriToTemp(uri) - safAudioTemp = tempPath - tempPath ?: "" - } else { - rawAudioFilePath - } - Gobackend.fetchAndSaveLyrics(trackName, artistName, spotifyId, durationMs, outputPath, audioFilePath) - """{"success":true}""" - } catch (e: Exception) { - """{"success":false,"error":"${e.message?.replace("\"", "'")}"}""" - } finally { - if (safAudioTemp != null) { - try { File(safAudioTemp).delete() } catch (_: Exception) {} - } - } - } - result.success(response) - } - "setLyricsProviders" -> { - val providersJson = call.argument("providers_json") ?: "[]" - val response = withContext(Dispatchers.IO) { - try { - Gobackend.setLyricsProvidersJSON(providersJson) - """{"success":true}""" - } catch (e: Exception) { - """{"success":false,"error":"${e.message?.replace("\"", "'")}"}""" - } - } - result.success(response) - } - "getLyricsProviders" -> { - val response = withContext(Dispatchers.IO) { - try { - Gobackend.getLyricsProvidersJSON() - } catch (e: Exception) { - "[]" - } - } - result.success(response) - } - "getAvailableLyricsProviders" -> { - val response = withContext(Dispatchers.IO) { - try { - Gobackend.getAvailableLyricsProvidersJSON() - } catch (e: Exception) { - "[]" - } - } - result.success(response) - } - "setLyricsFetchOptions" -> { - val optionsJson = call.argument("options_json") ?: "{}" - val response = withContext(Dispatchers.IO) { - try { - Gobackend.setLyricsFetchOptionsJSON(optionsJson) - """{"success":true}""" - } catch (e: Exception) { - """{"success":false,"error":"${e.message?.replace("\"", "'")}"}""" - } - } - result.success(response) - } - "getLyricsFetchOptions" -> { - val response = withContext(Dispatchers.IO) { - try { - Gobackend.getLyricsFetchOptionsJSON() - } catch (e: Exception) { - "{}" - } - } - result.success(response) - } "reEnrichFile" -> { val requestJson = call.argument("request_json") ?: "{}" val response = withContext(Dispatchers.IO) { @@ -1788,9 +1626,10 @@ class MainActivity: FlutterFragmentActivity() { val uri = Uri.parse(filePath) val tempPath = copyUriToTemp(uri) ?: return@withContext """{"error":"Failed to copy SAF file to temp"}""" + var retainedForFfmpeg = false try { reqObj.put("file_path", tempPath) - val raw = Gobackend.reEnrichFile(reqObj.toString()) + val raw = coreBackend.reEnrichFile(reqObj.toString()) val obj = JSONObject(raw) if (obj.has("error")) { @@ -1802,11 +1641,12 @@ class MainActivity: FlutterFragmentActivity() { // MP3/Opus: Dart handles FFmpeg on temp file, then writes back obj.put("temp_path", tempPath) obj.put("saf_uri", filePath) + retainedForFfmpeg = true return@withContext obj.toString() - // temp file NOT deleted - Dart cleans up after FFmpeg + writeTempToSaf + // Dart cleans up after FFmpeg + writeTempToSaf. } - // FLAC: Go wrote directly to temp, copy back now + // FLAC: the selected backend wrote to temp; copy back now. if (!writeUriFromPath(uri, tempPath)) { return@withContext """{"error":"Failed to write enriched metadata back to SAF file"}""" } @@ -1814,12 +1654,13 @@ class MainActivity: FlutterFragmentActivity() { writeSafSidecarLrc(uri, obj.optString("lyrics", "")) } raw - } catch (e: Exception) { - try { File(tempPath).delete() } catch (_: Exception) {} - throw e + } finally { + if (!retainedForFfmpeg) { + try { File(tempPath).delete() } catch (_: Exception) {} + } } } else { - Gobackend.reEnrichFile(requestJson) + coreBackend.reEnrichFile(requestJson) } } catch (e: Exception) { """{"error":${org.json.JSONObject.quote(e.message ?: "unknown")}}""" @@ -1928,431 +1769,19 @@ class MainActivity: FlutterFragmentActivity() { } result.success(payload) } - "getTrackCacheSize" -> { - val size = withContext(Dispatchers.IO) { - Gobackend.getTrackCacheSize() - } - result.success(size.toInt()) - } - "clearTrackCache" -> { - withContext(Dispatchers.IO) { - Gobackend.clearTrackIDCache() - } - result.success(null) - } - "getProviderMetadata" -> { - val providerId = call.argument("provider_id") ?: "" - val resourceType = call.argument("resource_type") ?: "" - val resourceId = call.argument("resource_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.getProviderMetadataJSON(providerId, resourceType, resourceId) - } - result.success(response) - } - "searchDeezerByISRC" -> { - val isrc = call.argument("isrc") ?: "" - val itemId = call.argument("item_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.searchDeezerByISRCForItemID(isrc, itemId) - } - result.success(response) - } - "getDeezerExtendedMetadata" -> { - val trackId = call.argument("track_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.getDeezerExtendedMetadata(trackId) - } - result.success(response) - } - "convertSpotifyToDeezer" -> { - val resourceType = call.argument("resource_type") ?: "" - val spotifyId = call.argument("spotify_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.convertSpotifyToDeezer(resourceType, spotifyId) - } - result.success(response) - } - "getSpotifyIDFromDeezerTrack" -> { - val deezerTrackId = call.argument("deezer_track_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.getSpotifyIDFromDeezerTrack(deezerTrackId) - } - result.success(response) - } - "getTidalURLFromDeezerTrack" -> { - val deezerTrackId = call.argument("deezer_track_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.getTidalURLFromDeezerTrack(deezerTrackId) - } - result.success(response) - } - "getLogsSince" -> { - val index = call.argument("index") ?: 0 - val response = withContext(Dispatchers.IO) { - Gobackend.getLogsSince(index.toLong()) - } - result.success(response) - } - "clearLogs" -> { - withContext(Dispatchers.IO) { - Gobackend.clearLogs() - } - result.success(null) - } "releaseMemory" -> { withContext(Dispatchers.IO) { - Gobackend.releaseMemory() + coreBackend.releaseIdleResources() } result.success(null) } "releaseMemoryUnderPressure" -> { withContext(Dispatchers.IO) { - Gobackend.releaseMemoryUnderPressure() + coreBackend.releaseMemoryUnderPressure() } + android.util.Log.d("SpotiFLAC", "Backend memory pressure release completed") result.success(null) } - "getGoRuntimeMetrics" -> { - val response = withContext(Dispatchers.IO) { - Gobackend.getRuntimeMetricsJSON() - } - result.success(response) - } - "setMetadataLanguage" -> { - val tag = call.argument("tag") ?: "" - withContext(Dispatchers.IO) { - Gobackend.setMetadataLanguage(tag) - } - result.success(null) - } - "setLoggingEnabled" -> { - val enabled = call.argument("enabled") ?: false - withContext(Dispatchers.IO) { - Gobackend.setLoggingEnabled(enabled) - } - result.success(null) - } - "initExtensionSystem" -> { - val extensionsDir = call.argument("extensions_dir") ?: "" - val dataDir = call.argument("data_dir") ?: "" - val masterKey = call.argument("master_key") ?: "" - withContext(Dispatchers.IO) { - Gobackend.setExtensionStorageMasterKey(masterKey) - Gobackend.initExtensionSystem(extensionsDir, dataDir) - } - result.success(null) - } - "loadExtensionsFromDir" -> { - val dirPath = call.argument("dir_path") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.loadExtensionsFromDir(dirPath) - } - result.success(response) - } - "loadExtensionFromPath" -> { - val filePath = call.argument("file_path") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.loadExtensionFromPath(filePath) - } - result.success(response) - } - "unloadExtension" -> { - val extensionId = call.argument("extension_id") ?: "" - withContext(Dispatchers.IO) { - Gobackend.unloadExtensionByID(extensionId) - } - result.success(null) - } - "removeExtension" -> { - val extensionId = call.argument("extension_id") ?: "" - withContext(Dispatchers.IO) { - Gobackend.removeExtensionByID(extensionId) - } - result.success(null) - } - "upgradeExtension" -> { - val filePath = call.argument("file_path") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.upgradeExtensionFromPath(filePath) - } - result.success(response) - } - "checkExtensionUpgrade" -> { - val filePath = call.argument("file_path") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.checkExtensionUpgradeFromPath(filePath) - } - result.success(response) - } - "getInstalledExtensions" -> { - val response = withContext(Dispatchers.IO) { - Gobackend.getInstalledExtensions() - } - result.success(response) - } - "setExtensionEnabled" -> { - val extensionId = call.argument("extension_id") ?: "" - val enabled = call.argument("enabled") ?: false - withContext(Dispatchers.IO) { - Gobackend.setExtensionEnabledByID(extensionId, enabled) - } - result.success(null) - } - "setProviderPriority" -> { - val priorityJson = call.argument("priority") ?: "[]" - withContext(Dispatchers.IO) { - Gobackend.setProviderPriorityJSON(priorityJson) - } - result.success(null) - } - "getProviderPriority" -> { - val response = withContext(Dispatchers.IO) { - Gobackend.getProviderPriorityJSON() - } - result.success(response) - } - "setDownloadFallbackExtensionIds" -> { - val extensionIdsJson = call.argument("extension_ids") ?: "" - withContext(Dispatchers.IO) { - Gobackend.setExtensionFallbackProviderIDsJSON(extensionIdsJson) - } - result.success(null) - } - "setMetadataProviderPriority" -> { - val priorityJson = call.argument("priority") ?: "[]" - withContext(Dispatchers.IO) { - Gobackend.setMetadataProviderPriorityJSON(priorityJson) - } - result.success(null) - } - "getMetadataProviderPriority" -> { - val response = withContext(Dispatchers.IO) { - Gobackend.getMetadataProviderPriorityJSON() - } - result.success(response) - } - "getExtensionSettings" -> { - val extensionId = call.argument("extension_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.getExtensionSettingsJSON(extensionId) - } - result.success(response) - } - "checkExtensionHealth" -> { - val extensionId = call.argument("extension_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.checkExtensionHealthJSON(extensionId) - } - result.success(response) - } - "setExtensionSettings" -> { - val extensionId = call.argument("extension_id") ?: "" - val settingsJson = call.argument("settings") ?: "{}" - withContext(Dispatchers.IO) { - Gobackend.setExtensionSettingsJSON(extensionId, settingsJson) - } - result.success(null) - } - "invokeExtensionAction" -> { - val extensionId = call.argument("extension_id") ?: "" - val actionName = call.argument("action") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.invokeExtensionActionJSON(extensionId, actionName) - } - result.success(response) - } - "searchTracksWithMetadataProviders" -> { - val query = call.argument("query") ?: "" - val limit = call.argument("limit") ?: 20 - val includeExtensions = call.argument("include_extensions") ?: true - val response = withContext(Dispatchers.IO) { - Gobackend.searchTracksWithMetadataProvidersJSON(query, limit.toLong(), includeExtensions) - } - result.success(response) - } - "searchTracksWithMetadataProvider" -> { - val extensionId = call.argument("extension_id") ?: "" - val query = call.argument("query") ?: "" - val limit = call.argument("limit") ?: 20 - val response = withContext(Dispatchers.IO) { - Gobackend.searchTracksWithMetadataProviderJSON( - extensionId, - query, - limit.toLong() - ) - } - result.success(response) - } - "findCollectionAcrossExtensions" -> { - val requestJson = call.arguments as? String ?: "{}" - val response: String = withContext(Dispatchers.IO) { - val method = Gobackend::class.java.getMethod( - "findCollectionAcrossExtensionsJSON", - String::class.java - ) - method.invoke(null, requestJson) as? String ?: "[]" - } - result.success(response) - } - "enrichTrackWithExtension" -> { - val extensionId = call.argument("extension_id") ?: "" - val trackJson = call.argument("track") ?: "{}" - val response = withContext(Dispatchers.IO) { - Gobackend.enrichTrackWithExtensionJSON(extensionId, trackJson) - } - result.success(response) - } - "cleanupExtensions" -> { - withContext(Dispatchers.IO) { - Gobackend.cleanupExtensions() - } - result.success(null) - } - "getExtensionPendingAuth" -> { - val extensionId = call.argument("extension_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.getExtensionPendingAuthJSON(extensionId) - } - if (response.isNullOrEmpty()) { - result.success(null) - } else { - result.success(response) - } - } - "setExtensionAuthCode" -> { - val extensionId = call.argument("extension_id") ?: "" - val authCode = call.argument("auth_code") ?: "" - withContext(Dispatchers.IO) { - Gobackend.setExtensionAuthCodeByID(extensionId, authCode) - } - result.success(null) - } - "completeExtensionSessionGrant" -> { - val extensionId = call.argument("extension_id") ?: "" - val grant = call.argument("grant") ?: "" - withContext(Dispatchers.IO) { - Gobackend.setExtensionSessionGrantByID(extensionId, grant) - val json = Gobackend.invokeExtensionActionJSON(extensionId, "completeGrant") - requireSuccessfulExtensionAction(extensionId, "completeGrant", json) - } - result.success(true) - } - "setExtensionTokens" -> { - val extensionId = call.argument("extension_id") ?: "" - val accessToken = call.argument("access_token") ?: "" - val refreshToken = call.argument("refresh_token") ?: "" - val expiresIn = call.argument("expires_in") ?: 0 - withContext(Dispatchers.IO) { - Gobackend.setExtensionTokensByID(extensionId, accessToken, refreshToken, expiresIn.toLong()) - } - result.success(null) - } - "clearExtensionPendingAuth" -> { - val extensionId = call.argument("extension_id") ?: "" - withContext(Dispatchers.IO) { - Gobackend.clearExtensionPendingAuthByID(extensionId) - } - result.success(null) - } - "isExtensionAuthenticated" -> { - val extensionId = call.argument("extension_id") ?: "" - val isAuth = withContext(Dispatchers.IO) { - Gobackend.isExtensionAuthenticatedByID(extensionId) - } - result.success(isAuth) - } - "getAllPendingAuthRequests" -> { - val response = withContext(Dispatchers.IO) { - Gobackend.getAllPendingAuthRequestsJSON() - } - result.success(response) - } - "getPendingFFmpegCommand" -> { - val commandId = call.argument("command_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.getPendingFFmpegCommandJSON(commandId) - } - if (response.isNullOrEmpty()) { - result.success(null) - } else { - result.success(response) - } - } - "setFFmpegCommandResult" -> { - val commandId = call.argument("command_id") ?: "" - val success = call.argument("success") ?: false - val output = call.argument("output") ?: "" - val error = call.argument("error") ?: "" - withContext(Dispatchers.IO) { - Gobackend.setFFmpegCommandResultByID(commandId, success, output, error) - } - result.success(null) - } - "getAllPendingFFmpegCommands" -> { - val response = withContext(Dispatchers.IO) { - Gobackend.getAllPendingFFmpegCommandsJSON() - } - result.success(response) - } - "customSearchWithExtension" -> { - val extensionId = call.argument("extension_id") ?: "" - val query = call.argument("query") ?: "" - val optionsJson = call.argument("options") ?: "" - val requestId = call.argument("request_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.customSearchWithExtensionJSONWithRequestID(extensionId, query, optionsJson, requestId) - } - result.success(response) - } - "cancelExtensionRequest" -> { - val requestId = call.argument("request_id") ?: "" - withContext(Dispatchers.IO) { - Gobackend.cancelExtensionRequestJSON(requestId) - } - result.success(null) - } - "handleURLWithExtension" -> { - val url = call.argument("url") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.handleURLWithExtensionJSON(url) - } - result.success(response) - } - "findURLHandler" -> { - val url = call.argument("url") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.findURLHandlerJSON(url) - } - result.success(response) - } - "getTrackPlatformLinks" -> { - val spotifyId = call.argument("spotify_id") ?: "" - val isrc = call.argument("isrc") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.getTrackPlatformLinksJSON(spotifyId, isrc) - } - result.success(response) - } - "fetchMusicBrainzTags" -> { - val isrc = call.argument("isrc") ?: "" - val albumName = call.argument("album_name") ?: "" - val response = withContext(Dispatchers.IO) { - val genre = try { - Gobackend.fetchMusicBrainzGenreByISRC(isrc) - } catch (_: Exception) { - "" - } - val albumArtist = try { - Gobackend.fetchMusicBrainzAlbumArtistByISRC(isrc, albumName) - } catch (_: Exception) { - "" - } - JSONObject() - .put("genre", genre) - .put("album_artist", albumArtist) - .toString() - } - result.success(response) - } "runPostProcessingV2" -> { val inputJson = call.argument("input") ?: "" val metadataJson = call.argument("metadata") ?: "" @@ -2367,90 +1796,21 @@ class MainActivity: FlutterFragmentActivity() { } if (effectiveUri.isNotBlank()) { - runPostProcessingSafV2(effectiveUri, metadataJson) + runPostProcessingSafV2(effectiveUri, metadataJson, inputObj.optString("item_id", "")) } else { if (pathStr.isNotBlank()) { inputObj.put("name", File(pathStr).name) inputObj.put("is_saf", false) } - Gobackend.runPostProcessingV2JSON(inputObj.toString(), metadataJson) + coreBackend.runPostProcessing(inputObj.toString(), metadataJson) } } result.success(response) } - "initExtensionRepo" -> { - val cacheDir = call.argument("cache_dir") ?: "" - withContext(Dispatchers.IO) { - Gobackend.initExtensionRepoJSON(cacheDir) - } - result.success(null) - } - "setRepoRegistryUrl" -> { - val registryUrl = call.argument("registry_url") ?: "" - withContext(Dispatchers.IO) { - Gobackend.setRepoRegistryURLJSON(registryUrl) - } - result.success(null) - } - "getRepoRegistryUrl" -> { - val response = withContext(Dispatchers.IO) { - Gobackend.getRepoRegistryURLJSON() - } - result.success(response) - } - "clearRepoRegistryUrl" -> { - withContext(Dispatchers.IO) { - Gobackend.clearRepoRegistryURLJSON() - } - result.success(null) - } - "getRepoExtensions" -> { - val forceRefresh = call.argument("force_refresh") ?: false - val response = withContext(Dispatchers.IO) { - Gobackend.getRepoExtensionsJSON(forceRefresh) - } - result.success(response) - } - "searchRepoExtensions" -> { - val query = call.argument("query") ?: "" - val category = call.argument("category") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.searchRepoExtensionsJSON(query, category) - } - result.success(response) - } - "getRepoCategories" -> { - val response = withContext(Dispatchers.IO) { - Gobackend.getRepoCategoriesJSON() - } - result.success(response) - } - "downloadRepoExtension" -> { - val extensionId = call.argument("extension_id") ?: "" - val destDir = call.argument("dest_dir") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.downloadRepoExtensionJSON(extensionId, destDir) - } - result.success(response) - } - "clearRepoCache" -> { - withContext(Dispatchers.IO) { - Gobackend.clearRepoCacheJSON() - } - result.success(null) - } - "getExtensionHomeFeed" -> { - val extensionId = call.argument("extension_id") ?: "" - val requestId = call.argument("request_id") ?: "" - val response = withContext(Dispatchers.IO) { - Gobackend.getExtensionHomeFeedJSONWithRequestID(extensionId, requestId) - } - result.success(response) - } "setLibraryCoverCacheDir" -> { val cacheDir = call.argument("cache_dir") ?: "" withContext(Dispatchers.IO) { - Gobackend.setLibraryCoverCacheDirJSON(cacheDir) + coreBackend.setLibraryCoverCacheDirectory(cacheDir) } result.success(null) } @@ -2458,7 +1818,7 @@ class MainActivity: FlutterFragmentActivity() { val folderPath = call.argument("folder_path") ?: "" val response = withContext(Dispatchers.IO) { safScanActive = false - bridgeJsonResult(Gobackend.scanLibraryFolderJSON(folderPath)) + bridgeJsonResult(coreBackend.scanLibraryFolder(folderPath)) } result.success(response) } @@ -2467,7 +1827,7 @@ class MainActivity: FlutterFragmentActivity() { val outputPath = call.argument("output_path") ?: "" val count = withContext(Dispatchers.IO) { safScanActive = false - Gobackend.scanLibraryFolderToNDJSONFileJSON( + coreBackend.scanLibraryFolderToNdjsonFile( folderPath, outputPath, ) @@ -2485,7 +1845,7 @@ class MainActivity: FlutterFragmentActivity() { val response = withContext(Dispatchers.IO) { safScanActive = false bridgeJsonResult( - Gobackend.scanLibraryFolderIncrementalJSON(folderPath, existingFiles) + coreBackend.scanLibraryFolderIncremental(folderPath, existingFiles) ) } result.success(response) @@ -2496,7 +1856,7 @@ class MainActivity: FlutterFragmentActivity() { val response = withContext(Dispatchers.IO) { safScanActive = false bridgeJsonResult( - Gobackend.scanLibraryFolderIncrementalFromSnapshotJSON( + coreBackend.scanLibraryFolderIncrementalFromSnapshot( folderPath, snapshotPath, ) @@ -2549,7 +1909,7 @@ class MainActivity: FlutterFragmentActivity() { if (safScanActive) { safProgressToJson() } else { - Gobackend.getLibraryScanProgressJSON() + coreBackend.getLibraryScanProgress() } } result.success(parseJsonPayload(response)) @@ -2557,7 +1917,7 @@ class MainActivity: FlutterFragmentActivity() { "cancelLibraryScan" -> { withContext(Dispatchers.IO) { safScanCancel = true - Gobackend.cancelLibraryScanJSON() + coreBackend.cancelLibraryScan() } result.success(null) } @@ -2572,7 +1932,7 @@ class MainActivity: FlutterFragmentActivity() { metadata.put("filePath", filePath) metadata.toString() } else { - Gobackend.readAudioMetadataJSON(filePath) + coreBackend.readAudioMetadata(filePath, "", "") } } catch (e: Exception) { """{"error":${org.json.JSONObject.quote(e.message ?: "unknown")}}""" @@ -2631,7 +1991,7 @@ class MainActivity: FlutterFragmentActivity() { } } - val resultJson = Gobackend.parseCueSheet(tempCuePath, tempDir) + val resultJson = coreBackend.parseCueSheet(tempCuePath, tempDir) if (audioDoc != null) { val resultObj = JSONObject(resultJson) @@ -2646,7 +2006,7 @@ class MainActivity: FlutterFragmentActivity() { try { tempAudioPath?.let { File(it).delete() } } catch (_: Exception) {} } } else { - Gobackend.parseCueSheet(cuePath, audioDir) + coreBackend.parseCueSheet(cuePath, audioDir) } } catch (e: Exception) { """{"error":${org.json.JSONObject.quote(e.message ?: "unknown")}}""" diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafIo.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafIo.kt index faf53954..b73ff607 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafIo.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafIo.kt @@ -22,7 +22,6 @@ import io.flutter.embedding.engine.FlutterShellArgs import io.flutter.plugin.common.EventChannel import io.flutter.plugin.common.MethodChannel import com.ryanheise.audioservice.AudioServicePlugin -import gobackend.Gobackend import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -135,8 +134,8 @@ internal fun MainActivity.copyUriToTemp(uri: Uri, fallbackExt: String? = null): val extFromName = extFromFileName(nameHint) val extFromMime = extFromMimeType(mime) val ext = if (extFromName.isNotBlank()) extFromName else if (extFromMime.isNotBlank()) extFromMime else (fallbackExt ?: "") - val suffix: String? = if (ext.isNotBlank()) ext else null - tempFile = File.createTempFile("saf_", suffix, cacheDir) + val suffix = ext.ifBlank { ".tmp" } + tempFile = coreBackend.createTemporaryMediaFile(this, "saf_", suffix) contentResolver.openInputStream(uri)?.use { input -> FileOutputStream(tempFile).use { output -> @@ -156,7 +155,6 @@ internal fun MainActivity.copyUriToTemp(uri: Uri, fallbackExt: String? = null): ) val result = copyMediaStoreUriToTemp(uri, fallbackExt) if (result != null) { - success = true return result } } @@ -190,8 +188,8 @@ internal fun MainActivity.copyMediaStoreUriToTemp(uri: Uri, fallbackExt: String? var tempFile: File? = null try { val ext = resolveMediaStoreExt(uri, fallbackExt) - val suffix: String? = if (ext.isNotBlank()) ext else null - tempFile = File.createTempFile("ms_", suffix, cacheDir) + val suffix = ext.ifBlank { ".tmp" } + tempFile = coreBackend.createTemporaryMediaFile(this, "ms_", suffix) contentResolver.openInputStream(uri)?.use { input -> FileOutputStream(tempFile).use { output -> @@ -295,7 +293,7 @@ internal fun MainActivity.readAudioMetadataFromUri( uri, displayNameHint, fallbackExt, acceptDirect = { !it.optBoolean("metadataFromFilename", false) }, ) { path, name -> - val obj = JSONObject(Gobackend.readAudioMetadataWithHintAndCoverCacheKeyJSON( + val obj = JSONObject(coreBackend.readAudioMetadata( path, name, coverCacheKey, )) obj.takeUnless { it.has("error") } @@ -305,7 +303,7 @@ internal fun MainActivity.readCompleteMetadataFromUri( uri: Uri, displayNameHint: String? = null, ): JSONObject? = readMetadataFromUri(uri, displayNameHint) { path, name -> - JSONObject(Gobackend.readFileMetadataWithHint(path, name)).takeUnless { it.has("error") } + JSONObject(coreBackend.readFileMetadata(path, name)).takeUnless { it.has("error") } } internal fun MainActivity.writeUriFromPath(uri: Uri, srcPath: String): Boolean { @@ -388,20 +386,14 @@ internal fun MainActivity.writeSafSidecarLrc(audioUri: Uri, lrcContent: String): } } -internal fun MainActivity.runPostProcessingSafV2(fileUriStr: String, metadataJson: String): String { +internal fun MainActivity.runPostProcessingSafV2(fileUriStr: String, metadataJson: String, itemId: String): String { val uri = Uri.parse(fileUriStr) val doc = DocumentFile.fromSingleUri(this, uri) ?: return errorJson("SAF file not found") val tempInput = copyUriToTemp(uri) ?: return errorJson("Failed to copy SAF file to temp") - val tempDir = File(tempInput).parentFile?.absolutePath ?: "" - if (tempDir.isNotBlank()) { - try { - Gobackend.allowDownloadDir(tempDir) - } catch (_: Exception) {} - } - val inputObj = JSONObject() + inputObj.put("item_id", itemId) inputObj.put("path", tempInput) inputObj.put("uri", fileUriStr) inputObj.put("name", doc.name ?: File(tempInput).name) @@ -409,7 +401,7 @@ internal fun MainActivity.runPostProcessingSafV2(fileUriStr: String, metadataJso inputObj.put("size", doc.length()) inputObj.put("is_saf", true) - val response = Gobackend.runPostProcessingV2JSON(inputObj.toString(), metadataJson) + val response = coreBackend.runPostProcessing(inputObj.toString(), metadataJson) val respObj = JSONObject(response) if (!respObj.optBoolean("success", false)) { try { diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt index dd62dd2f..5220afdc 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt @@ -22,7 +22,6 @@ import io.flutter.embedding.engine.FlutterShellArgs import io.flutter.plugin.common.EventChannel import io.flutter.plugin.common.MethodChannel import com.ryanheise.audioservice.AudioServicePlugin -import gobackend.Gobackend import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -81,7 +80,7 @@ internal fun MainActivity.readLibraryScanProgressJsonForStream(): String { return if (safScanActive) { safProgressToJson() } else { - Gobackend.getLibraryScanProgressJSON() + coreBackend.getLibraryScanProgress() } } @@ -533,11 +532,8 @@ internal fun MainActivity.extractCueAudioFileName(cueTempPath: String): String? ".flac", ".wav", ".ape", ".mp3", ".ogg", ".wv", ".m4a", ".mp4", ".aac" ) - // Audio file extensions that the local library scanner accepts. Must stay in - // sync with supportedAudioFormats in go_backend/library_scan.go so that every - // format the Go engine can read (FLAC, M4A/MP4/AAC, MP3, Opus/OGG, APE/WV/MPC, - // WAV, AIFF) is also enumerated here during the SAF folder walk. (.cue is - // handled separately.) + // Keep the SAF folder walk aligned with the backend's supported audio formats. + // CUE files are handled separately. private val libraryScanAudioExtensions = setOf( ".flac", ".m4a", ".mp4", ".aac", ".mp3", ".opus", ".ogg", ".ape", ".wv", ".mpc", ".wav", ".aiff", ".aif" @@ -972,7 +968,7 @@ internal fun MainActivity.scanSafTree( val cueLastModified = cue.lastModified - val cueResultsJson = Gobackend.scanCueSheetForLibraryWithCoverCacheKey( + val cueResultsJson = coreBackend.scanCueForLibrary( tempCuePath, tempDir, cueDoc.uri.toString(), @@ -1404,7 +1400,7 @@ internal fun MainActivity.scanSafTreeIncremental( tempAudioPath = renamedAudio.absolutePath } - val cueResultsJson = Gobackend.scanCueSheetForLibraryWithCoverCacheKey( + val cueResultsJson = coreBackend.scanCueForLibrary( tempCuePath, tempDir, cueDoc.uri.toString(), diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeDownloadFinalizer.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeDownloadFinalizer.kt index 8809e0cb..b70c8ee0 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeDownloadFinalizer.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeDownloadFinalizer.kt @@ -20,7 +20,6 @@ import com.zarz.spotiflac.NativeFinalizationPolicy.displayAudioQuality import com.zarz.spotiflac.NativeFinalizationPolicy.formatIndexTag import com.zarz.spotiflac.NativeFinalizationPolicy.normalizeAudioCodec import com.zarz.spotiflac.NativeFinalizationPolicy.resolvePreferredDecryptionExtension -import gobackend.Gobackend import org.json.JSONObject import java.io.File import java.io.RandomAccessFile @@ -289,6 +288,14 @@ object NativeDownloadFinalizer { promoteStagedSafOutputIfNeeded(context, effectiveInput, state) } outputPublished = true + } else { + // Match the Dart queue: an existing download still runs enabled + // extension hooks. It may be the input left by an interrupted + // finalizer, so file existence does not prove the hook finished. + // outputPublished keeps this pre-existing input out of cleanup. + checkCancelled(shouldCancel) + runPostProcessing(context, effectiveInput, state, shouldCancel) + checkCancelled(shouldCancel) } if (!qualityMetadataRefreshed) { try { @@ -917,14 +924,14 @@ object NativeDownloadFinalizer { private fun writeReplayGainFields(context: Context, path: String, fields: JSONObject) { if (!path.startsWith("content://")) { - writeLocalReplayGainFields(path, fields) + writeLocalReplayGainFields(context, path, fields) return } val tempPath = SafDownloadHandler.copyContentUriToTemp(context, path) ?: throw IllegalStateException("failed to copy SAF file for ReplayGain write") try { - writeLocalReplayGainFields(tempPath, fields) + writeLocalReplayGainFields(context, tempPath, fields) val uri = Uri.parse(path) context.contentResolver.openOutputStream(uri, "wt")?.use { output -> File(tempPath).inputStream().use { input -> input.copyTo(output) } @@ -935,8 +942,9 @@ object NativeDownloadFinalizer { } } - private fun writeLocalReplayGainFields(path: String, fields: JSONObject) { - val result = parseObject(Gobackend.editFileMetadata(path, fields.toString())) + private fun writeLocalReplayGainFields(context: Context, path: String, fields: JSONObject) { + val backend = createCoreBackend(context) + val result = parseObject(backend.editFileMetadata(path, fields.toString())) val method = result.optString("method", "") check( result.optBoolean("success", false) && @@ -944,7 +952,7 @@ object NativeDownloadFinalizer { (method == "native" || method.startsWith("native_")), ) { "ReplayGain native write did not complete: $result" } - val metadata = parseObject(Gobackend.readFileMetadata(path)) + val metadata = parseObject(backend.readFileMetadata(path, "")) check(!metadata.has("error")) { "ReplayGain verification failed: $metadata" } val isOpus = metadata.optString("audio_codec", "") == "opus" for (key in fields.keys()) { @@ -970,7 +978,7 @@ object NativeDownloadFinalizer { val deleteProbePath = probePath != state.filePath try { - val metadata = parseObject(Gobackend.readFileMetadata(probePath)) + val metadata = parseObject(createCoreBackend(context).readFileMetadata(probePath, state.fileName)) if (metadata.has("error")) return if (metadata.has("lyrics") || metadata.has("hasLyrics")) { @@ -1080,18 +1088,15 @@ object NativeDownloadFinalizer { ?: throw IllegalStateException("failed to copy SAF file for post-processing") try { val inputObj = JSONObject() + .put("item_id", input.itemId) .put("path", tempInput) .put("uri", uri) .put("name", state.fileName) .put("mime_type", mimeTypeForExt(state.fileName.substringAfterLast('.', ""))) .put("size", File(tempInput).length()) .put("is_saf", true) - val response = JSONObject( - withFFmpegCommandPump(shouldCancel) { - checkCancelled(shouldCancel) - Gobackend.runPostProcessingV2JSON(inputObj.toString(), metadata.toString()) - } - ) + checkCancelled(shouldCancel) + val response = JSONObject(createCoreBackend(context).runPostProcessing(inputObj.toString(), metadata.toString())) checkCancelled(shouldCancel) if (!response.optBoolean("success", false)) return val newPath = response.optString("new_file_path", "") @@ -1118,19 +1123,25 @@ object NativeDownloadFinalizer { } val inputObj = JSONObject() + .put("item_id", input.itemId) .put("path", state.filePath) .put("name", state.fileName) .put("is_saf", false) - val response = JSONObject( - withFFmpegCommandPump(shouldCancel) { - checkCancelled(shouldCancel) - Gobackend.runPostProcessingV2JSON(inputObj.toString(), metadata.toString()) - } - ) + checkCancelled(shouldCancel) + val response = JSONObject(createCoreBackend(context).runPostProcessing(inputObj.toString(), metadata.toString())) checkCancelled(shouldCancel) if (response.optBoolean("success", false)) { val newPath = response.optString("new_file_path", "") if (newPath.isNotBlank() && newPath != state.filePath) { + if (isDeferredSafPublish(input)) { + val output = File(newPath) + check(output.isFile && output.length() > 0L) { + "post-processing output missing or empty" + } + // This input is an owned staging file; publication later + // removes the replacement, so retire the old stage now. + File(state.filePath).delete() + } state.filePath = newPath state.fileName = File(newPath).name } diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerFFmpeg.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerFFmpeg.kt index e8bd5b55..d686065b 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerFFmpeg.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerFFmpeg.kt @@ -22,7 +22,6 @@ import com.zarz.spotiflac.NativeFinalizationPolicy.isLosslessAudioCodec import com.zarz.spotiflac.NativeFinalizationPolicy.isLossyAudioCodec import com.zarz.spotiflac.NativeFinalizationPolicy.normalizeAudioCodec import com.zarz.spotiflac.NativeFinalizationPolicy.resolvePreferredDecryptionExtension -import gobackend.Gobackend import org.json.JSONObject import java.io.File import java.io.RandomAccessFile @@ -31,7 +30,6 @@ import java.util.Locale import java.util.concurrent.CancellationException import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicBoolean import kotlin.math.pow @@ -99,7 +97,7 @@ internal fun NativeDownloadFinalizer.runFFmpeg(command: String, shouldCancel: () return runFFmpegArguments(FFmpegKitConfig.parseArguments(command), shouldCancel) } -internal fun NativeDownloadFinalizer.runFFmpegArguments(arguments: Array, shouldCancel: () -> Boolean = { false }): Pair { +internal fun NativeDownloadFinalizer.runFFmpegArguments(arguments: Array, shouldCancel: () -> Boolean = { false }, trackFinalizerSession: Boolean = true): Pair { checkCancelled(shouldCancel) installNativeFFmpegCallbackFilter() val latch = CountDownLatch(1) @@ -116,7 +114,7 @@ internal fun NativeDownloadFinalizer.runFFmpegArguments(arguments: Array ) val sessionId = session.sessionId synchronized(activeFFmpegSessionLock) { - activeFFmpegSessionIds.add(sessionId) + if (trackFinalizerSession) activeFFmpegSessionIds.add(sessionId) } nativeFFmpegSessionIds.add(sessionId) FFmpegKitConfig.asyncFFmpegExecute(session) @@ -157,71 +155,6 @@ internal fun NativeDownloadFinalizer.installNativeFFmpegCallbackFilter() { } } -internal fun NativeDownloadFinalizer.withFFmpegCommandPump( - shouldCancel: () -> Boolean = { false }, - block: () -> String, -): String { - val running = AtomicBoolean(true) - val handled = mutableSetOf() - val pump = Thread { - while (running.get()) { - try { - val raw = Gobackend.waitForPendingFFmpegCommandsJSON(1_000L) - val commands = org.json.JSONArray(raw) - for (index in 0 until commands.length()) { - val command = commands.optJSONObject(index) ?: continue - val id = command.optString("command_id", "") - val rawArguments = command.optJSONArray("arguments") - val arguments = if (rawArguments == null) { - emptyArray() - } else { - Array(rawArguments.length()) { argumentIndex -> - rawArguments.optString(argumentIndex, "") - } - } - if (id.isBlank() || arguments.isEmpty() || arguments.any { it.isEmpty() } || handled.contains(id)) { - continue - } - handled.add(id) - // Every claimed command must get a result delivered to - // the Go side, even on failure or cancellation: the - // backend blocks until one arrives and never retries a - // claimed id, so bailing out here would strand the - // gomobile call the main thread is sitting in forever. - val result = try { - if (shouldCancel()) { - Pair(false, "cancelled") - } else { - runFFmpegArguments(arguments, shouldCancel) - } - } catch (e: Exception) { - Pair(false, e.message ?: "FFmpeg execution failed") - } - try { - Gobackend.setFFmpegCommandResultByID( - id, - result.first, - result.second, - if (result.first) "" else result.second, - ) - } catch (e: Exception) { - Log.w(TAG, "Failed to deliver FFmpeg result for $id: ${e.message}") - } - } - } catch (_: Exception) { - } - } - } - pump.isDaemon = true - pump.start() - return try { - block() - } finally { - running.set(false) - pump.interrupt() - } -} - /** * Staged sibling name for conversion outputs: "song.flac" -> "song.partial.flac". * The ".partial" shape is ignored by library scans and duplicate checks, diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerMedia.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerMedia.kt index 7a8710a8..856bebf1 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerMedia.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerMedia.kt @@ -26,7 +26,6 @@ import com.zarz.spotiflac.NativeFinalizationPolicy.removeQualityVariantStagingLa import com.zarz.spotiflac.NativeFinalizationPolicy.resolveQualityVariantFilename import com.zarz.spotiflac.NativeFinalizationPolicy.normalizeAudioCodec import com.zarz.spotiflac.NativeFinalizationPolicy.resolvePreferredDecryptionExtension -import gobackend.Gobackend import org.json.JSONObject import java.io.File import java.io.RandomAccessFile @@ -176,7 +175,7 @@ internal fun NativeDownloadFinalizer.writeExternalLrc(context: Context, input: N if (!input.request.optBoolean("embed_metadata", false) || !input.request.optBoolean("embed_lyrics", false)) return val lyricsMode = input.request.optString("lyrics_mode", "") if (lyricsMode != "external" && lyricsMode != "both") return - val lrc = resolveLyricsLrc(input) + val lrc = resolveLyricsLrc(context, input) if ( !NativeFinalizationPolicy.hasUsableLyricsContent(lrc) || lrc.trim().equals("[instrumental:true]", ignoreCase = true) @@ -217,7 +216,7 @@ internal fun NativeDownloadFinalizer.writeExternalLrc(context: Context, input: N } } -internal fun NativeDownloadFinalizer.resolveLyricsLrc(input: NativeDownloadFinalizer.FinalizeInput): String { +internal fun NativeDownloadFinalizer.resolveLyricsLrc(context: Context, input: NativeDownloadFinalizer.FinalizeInput): String { val existing = input.result.optString("lyrics_lrc", "").trim() if (existing.isNotEmpty()) return existing @@ -227,7 +226,7 @@ internal fun NativeDownloadFinalizer.resolveLyricsLrc(input: NativeDownloadFinal if (trackName.isBlank() || artistName.isBlank()) return "" return try { - val fetched = Gobackend.getLyricsLRC( + val fetched = createCoreBackend(context).getLyricsLrc( spotifyId, trackName, artistName, @@ -303,13 +302,13 @@ internal fun NativeDownloadFinalizer.embedBasicMetadata(context: Context, path: val lyricsMode = input.request.optString("lyrics_mode", "embed") val shouldResolveLyrics = input.request.optBoolean("embed_lyrics", false) && (lyricsMode == "embed" || lyricsMode == "both") - val lyrics = if (shouldResolveLyrics) resolveLyricsLrc(input) else "" + val lyrics = if (shouldResolveLyrics) resolveLyricsLrc(context, input) else "" val shouldEmbedLyrics = shouldResolveLyrics && NativeFinalizationPolicy.hasUsableLyricsContent(lyrics) && !lyrics.trim().equals("[instrumental:true]", ignoreCase = true) - // FLAC, MP3, Opus, and M4A all have native Go tag writers that edit the + // FLAC, MP3, Opus, and M4A all have backend tag writers that edit the // tag block atomically without an ffmpeg remux (which drops foreign - // frames and rewrites the whole container). The Go side answers + // frames and rewrites the whole container). The backend answers // method=ffmpeg when it cannot handle the file natively. if (format == "flac" || format == "mp3" || format == "opus" || format == "m4a") { val nativeCover = downloadCoverForMetadata(context, input) @@ -339,7 +338,7 @@ internal fun NativeDownloadFinalizer.embedBasicMetadata(context: Context, path: fields.put("lyrics", lyrics) fields.put("unsyncedlyrics", lyrics) } - val response = Gobackend.editFileMetadata(path, fields.toString()) + val response = createCoreBackend(context).editFileMetadata(path, fields.toString()) val method = try { JSONObject(response).optString("method", "") } catch (_: Exception) { @@ -509,9 +508,10 @@ internal fun NativeDownloadFinalizer.downloadCoverForMetadata(context: Context, val safeItemId = input.itemId.ifBlank { "item" }.replace(Regex("[^A-Za-z0-9._-]"), "_") val maxDimension = input.request.optLong("cover_max_dimension", 0L).coerceAtLeast(0L) - val output = File.createTempFile("native_cover_${safeItemId}_", ".jpg", context.cacheDir) + val backend = createCoreBackend(context) + val output = backend.createTemporaryMediaFile(context, "native_cover_${safeItemId}_", ".jpg") return try { - Gobackend.downloadCoverToFileSized( + backend.downloadCoverToFileSized( coverUrl, output.absolutePath, maxDimension diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerSafPublish.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerSafPublish.kt index 1d3d88c2..7a5ebc6b 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerSafPublish.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeFinalizerSafPublish.kt @@ -23,7 +23,6 @@ import com.zarz.spotiflac.NativeFinalizationPolicy.isLossyAudioCodec import com.zarz.spotiflac.NativeFinalizationPolicy.normalizeAudioCodec import com.zarz.spotiflac.NativeFinalizationPolicy.removeQualityVariantStagingLabel import com.zarz.spotiflac.NativeFinalizationPolicy.resolvePreferredDecryptionExtension -import gobackend.Gobackend import org.json.JSONObject import java.io.File import java.io.RandomAccessFile diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/SafDownloadHandler.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/SafDownloadHandler.kt index 701db325..d0807800 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/SafDownloadHandler.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/SafDownloadHandler.kt @@ -55,26 +55,26 @@ object SafDownloadHandler { } } - fun handle(context: Context, requestJson: String, downloader: (String) -> String): String { + internal fun handle(context: Context, requestJson: String, backend: CoreBackend): String { val req = JSONObject(requestJson) val storageMode = req.optString("storage_mode", "") val treeUriStr = req.optString("saf_tree_uri", "") if (storageMode != "saf" || treeUriStr.isBlank()) { - return downloader(requestJson) + return backend.downloadByStrategy(requestJson) } val relativeDir = sanitizeRelativeDir(req.optString("saf_relative_dir", "")) val outputExt = normalizeExt(req.optString("saf_output_ext", "")) val fileName = buildSafFileName(req, outputExt) return withSafNameLock(treeUriStr, relativeDir, fileName) { - handleSafLocked(context, req, downloader, treeUriStr, relativeDir, outputExt, fileName) + handleSafLocked(context, req, backend, treeUriStr, relativeDir, outputExt, fileName) } } private fun handleSafLocked( context: Context, req: JSONObject, - downloader: (String) -> String, + backend: CoreBackend, treeUriStr: String, relativeDir: String, outputExt: String, @@ -112,12 +112,12 @@ object SafDownloadHandler { if (deferSafPublish) { existingDir?.let { deleteStaleStagedFiles(it, fileName, outputExt) } val workingExt = outputExt.ifBlank { ".tmp" } - val workingFile = File.createTempFile("native_saf_work_", workingExt, context.cacheDir) + val workingFile = backend.createTemporaryMediaFile(context, "native_saf_work_", workingExt) return try { req.put("output_path", workingFile.absolutePath) req.put("output_ext", outputExt) req.remove("output_fd") - val response = downloader(req.toString()) + val response = backend.downloadByStrategy(req.toString()) val respObj = JSONObject(response) if (respObj.optBoolean("success", false)) { val resolvedFileName = respObj.optString("resolved_file_name", "") @@ -162,16 +162,32 @@ object SafDownloadHandler { var document = createOrReuseDocumentFile(targetDir, stagedMimeType, stagedFileName) ?: return errorJson("Failed to create SAF file") - val pfd = context.contentResolver.openFileDescriptor(document.uri, "rw") - ?: return errorJson("Failed to open SAF file") - + var pfd: android.os.ParcelFileDescriptor? = null var detachedFd: Int? = null + var workingFile: File? = null try { - detachedFd = pfd.detachFd() - req.put("output_path", "") - req.put("output_fd", detachedFd) + if (backend.supportsOutputDescriptors) { + val descriptor = context.contentResolver.openFileDescriptor(document.uri, "rw") + ?: throw IllegalStateException("Failed to open SAF file") + pfd = descriptor + detachedFd = descriptor.detachFd() + req.put("output_path", "") + req.put("output_fd", detachedFd) + } else { + // The OS adapter retains descriptor ownership. A path-based + // backend writes into its granted staging directory, then the + // existing SAF copy/promotion publishes the completed output. + val staged = backend.createTemporaryMediaFile( + context, + "native_saf_work_", + outputExt.ifBlank { ".tmp" }, + ) + workingFile = staged + req.put("output_path", staged.absolutePath) + req.remove("output_fd") + } req.put("output_ext", outputExt) - val response = downloader(req.toString()) + val response = backend.downloadByStrategy(req.toString()) val respObj = JSONObject(response) if (respObj.optBoolean("success", false)) { val resolvedFileName = respObj.optString("resolved_file_name", "").trim() @@ -180,15 +196,16 @@ object SafDownloadHandler { } else { fileName } - val goFilePath = respObj.optString("file_path", "") - if (goFilePath.isNotEmpty() && - !goFilePath.startsWith("content://") && - !goFilePath.startsWith("/proc/self/fd/") - ) { + val backendFilePath = respObj.optString("file_path", "") + val localFilePath = backendFilePath.takeIf { + it.isNotEmpty() && !it.startsWith("content://") && + !it.startsWith("/proc/self/fd/") + } ?: workingFile?.absolutePath + if (localFilePath != null) { try { - val srcFile = File(goFilePath) + val srcFile = File(localFilePath) if (!srcFile.exists() || srcFile.length() <= 0) { - throw IllegalStateException("extension output missing or empty: $goFilePath") + throw IllegalStateException("extension output missing or empty: $localFilePath") } val actualExt = normalizeExt(srcFile.extension) if (actualExt.isNotBlank()) { @@ -259,9 +276,10 @@ object SafDownloadHandler { document.delete() return errorJson("SAF download failed: ${e.message}") } finally { + workingFile?.delete() if (detachedFd == null) { try { - pfd.close() + pfd?.close() } catch (_: Exception) { } } @@ -315,7 +333,7 @@ object SafDownloadHandler { ?.takeIf { it.isNotBlank() } ?.let { ".$it" } ?: ".tmp" - val createdTemp = File.createTempFile("native_saf_", extension, context.cacheDir) + val createdTemp = createCoreBackend(context).createTemporaryMediaFile(context, "native_saf_", extension) temp = createdTemp context.contentResolver.openInputStream(uri)?.use { input -> createdTemp.outputStream().use { output -> diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/SafMetadataReadPolicy.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/SafMetadataReadPolicy.kt index 5c2f3a7d..84ca629e 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/SafMetadataReadPolicy.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/SafMetadataReadPolicy.kt @@ -1,5 +1,7 @@ package com.zarz.spotiflac +import java.io.File + /** Failure belongs to this read, never to a different document or provider. * The direct callback owns and closes its descriptor before fallback starts. */ internal fun readSafMetadataWithFallback( @@ -10,3 +12,17 @@ internal fun readSafMetadataWithFallback( if (direct != null) return direct return try { fallbackRead() } catch (_: Exception) { null } } + +internal fun readLyricsWithSafCopy( + path: String, + copyToTemp: (String) -> File?, + read: (String) -> T, +): T? { + if (!path.startsWith("content://")) return read(path) + val temporary = copyToTemp(path) ?: return null + return try { + read(temporary.absolutePath) + } finally { + temporary.delete() + } +} diff --git a/android/app/src/rust/kotlin/com/zarz/spotiflac/SelectedCoreBackend.kt b/android/app/src/rust/kotlin/com/zarz/spotiflac/SelectedCoreBackend.kt new file mode 100644 index 00000000..b5da804e --- /dev/null +++ b/android/app/src/rust/kotlin/com/zarz/spotiflac/SelectedCoreBackend.kt @@ -0,0 +1,813 @@ +package com.zarz.spotiflac + +import android.content.Context +import android.net.Uri +import com.spotiflac.backend.CancellationDomain +import com.spotiflac.backend.CancellationRegistry +import com.spotiflac.backend.ExtensionManager +import com.spotiflac.backend.ExtensionRepository +import com.spotiflac.backend.LyricsRequest +import com.spotiflac.backend.RequestLease +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.security.MessageDigest +import java.util.UUID +import org.json.JSONArray +import org.json.JSONObject + +internal fun createCoreBackend(context: Context): CoreBackend = RustCoreBackend.initialize(context) + +internal suspend fun MainActivity.dispatchBackendApplication(call: MethodCall, result: MethodChannel.Result): Boolean { + val response = withContext(Dispatchers.IO) { + when (call.method) { + "getLyricsLRC", "getLyricsLRCWithSource" -> { + val arguments = call.arguments as? Map<*, *> ?: emptyMap() + val path = arguments["file_path"] as? String ?: "" + readLyricsWithSafCopy( + path, + copyToTemp = { copyUriToTemp(Uri.parse(it))?.let(::File) }, + read = { localPath -> + coreBackend.invokeApplication(call.method, arguments + ("file_path" to localPath)) + }, + ) ?: if (call.method == "getLyricsLRC") "" else { + """{"lyrics":"","source":"","sync_type":"","instrumental":false}""" + } + } + else -> coreBackend.invokeApplication(call.method, call.arguments) + } + } + result.success(response) + return true +} + +internal object RustCoreBackend : CoreBackend { + override val implementation = "rust" + override val routesApplication = true + private lateinit var root: File + private var manager: ExtensionManager? = null + private var repository: ExtensionRepository? = null + private var requests: CancellationRegistry? = null + private var identity: Triple>? = null + private var loggingEnabled = false + private var allowPrivateNetwork = false + private var allowHttpFallback = false + private var fallbackProviders: List? = null + private var runtimeState: Pair? = null + private val directoryScopes = mutableMapOf() + private var libraryCoverScope: AutoCloseable? = null + + @Synchronized + fun initialize(context: Context): RustCoreBackend { + if (!::root.isInitialized) root = File(context.applicationContext.cacheDir, "rust-core-pilot") + return this + } + + @Synchronized + private fun owner(): ExtensionManager { + return checkNotNull(manager) { "Rust backend is not initialized" } + } + + private fun directoryAliases(value: String, sources: String, data: String): List { + require(File(value).isAbsolute) { "Output directories must be absolute" } + val path = File(value).canonicalPath + require(path != "/" && listOf(sources, data).none { + path == it || path.startsWith("$it/") || it.startsWith("$path/") + }) { "Output directory overlaps extension storage" } + return listOf(path, File(value).absolutePath).distinct() + } + + @Synchronized + override fun openDownloadDirectory(path: String): AutoCloseable { + val current = owner() + val storage = checkNotNull(identity) + val scope = current.environment().use { + it.grantDownloadDirectories(directoryAliases(path, storage.first, storage.second)) + } + return object : AutoCloseable { + private var released = false + + @Synchronized + override fun close() { + if (released) return + released = true + try { scope.release() } finally { scope.close() } + } + } + } + + @Synchronized + private fun setDownloadDirectory(path: String) { + val current = owner() + val storage = checkNotNull(identity) + val files = File(root, "files") + val allowed = listOf(files.canonicalPath, files.absolutePath) + + if (path.isEmpty()) emptyList() else directoryAliases(path, storage.first, storage.second) + current.environment().use { it.setAllowedDownloadDirectories(allowed.distinct()) } + } + + @Synchronized + private fun requestRegistry(): CancellationRegistry = requests ?: CancellationRegistry( + CancellationDomain.EXTENSION_REQUEST, + ).also { requests = it } + + @Synchronized + private fun acquireRequest(id: String): Pair = + owner() to requestRegistry().acquire(id) + + @Synchronized + private fun repositoryOwner(): ExtensionRepository = + checkNotNull(repository) { "Extension repository is not initialized" } + + @Synchronized + private fun initializeRepository(cachePath: String) { + val current = owner() + if (repository != null) return + val cache = File(cachePath) + require(cache.isAbsolute) { "Repository cache directory must be absolute" } + repository = ExtensionRepository(current, cache.canonicalPath) + } + + @Synchronized + private fun initializeOwner(arguments: Map<*, *>) { + val sourcePath = File(arguments["extensions_dir"] as String) + val dataPath = File(arguments["data_dir"] as String) + require(sourcePath.isAbsolute && dataPath.isAbsolute) { "Extension storage directories must be absolute" } + val sources = sourcePath.canonicalPath + val data = dataPath.canonicalPath + val key = arguments["master_key"] as String + val requested = Triple( + sources, + data, + MessageDigest.getInstance("SHA-256").digest(key.toByteArray()).toList(), + ) + val files = File(root, "files") + check(files.mkdirs() || files.isDirectory) + val rawDirectories = arguments["allowed_directories"] + require(rawDirectories == null || rawDirectories is List<*>) { "Invalid output directories" } + val allowedDirectories = listOf(files.canonicalPath, files.absolutePath) + + (rawDirectories as? List<*>).orEmpty().flatMap { value -> + require(value is String) { "Output directories must be strings" } + directoryAliases(value, sources, data) + } + if (manager != null) { + check(identity == requested) { "Rust backend is already initialized with different storage" } + manager!!.environment().use { it.setAllowedDownloadDirectories(allowedDirectories.distinct()) } + return + } + val preparedRuntime = runtimeState + require(preparedRuntime == null || preparedRuntime.first == data) { + "Runtime state directory does not match the initialized owner" + } + val created = ExtensionManager.withLyricsSettings( + sources, + data, + key, + BuildConfig.VERSION_NAME, + 30_000uL, + arguments["lyrics_providers_json"] as? String ?: "[]", + arguments["lyrics_options_json"] as? String ?: "{}", + ) + try { + created.environment().use { environment -> + environment.setAllowedDownloadDirectories(allowedDirectories.distinct()) + environment.setAllowPrivateNetwork(allowPrivateNetwork) + environment.setNetworkCompatibilityOptions(allowHttpFallback, false) + environment.logBuffer().use { it.setEnabled(loggingEnabled) } + preparedRuntime?.let { environment.setRuntimeState(it.second) } + } + created.setFallbackProviders(fallbackProviders) + } catch (error: Exception) { + created.shutdown() + created.close() + throw error + } + identity = requested + manager = created + } + + @Synchronized + private fun shutdownOwner() { + requests?.let { + it.shutdown() + it.close() + } + requests = null + repository?.let { + it.shutdown() + it.close() + } + repository = null + manager?.let { + it.shutdown() + it.close() + } + manager = null + directoryScopes.values.forEach { it.close() } + directoryScopes.clear() + libraryCoverScope?.close() + libraryCoverScope = null + identity = null + runtimeState = null + } + + override fun fileMetadataImplementation(path: String): String = "rust" + + override fun readFileMetadata(path: String, hint: String): String = + com.spotiflac.backend.readFileMetadata(path, hint, null) + + override fun readAudioMetadata(path: String, hint: String, cacheKey: String): String = + owner().readAudioMetadata(File(path).canonicalPath, hint, cacheKey, null) + + private fun withLibraryDirectories(paths: List, block: (ExtensionManager) -> T): T { + val (current, scope) = synchronized(this) { + val current = owner() + val storage = checkNotNull(identity) + val aliases = paths.flatMap { directoryAliases(it, storage.first, storage.second) }.distinct() + current to current.environment().use { it.grantDownloadDirectories(aliases) } + } + return scope.use { + try { block(current) } finally { it.release() } + } + } + + @Synchronized + override fun setLibraryCoverCacheDirectory(path: String) { + val current = owner() + require(path.isEmpty() || File(path).isAbsolute) { "Library cover directory must be absolute" } + val directory = if (path.isEmpty()) null else File(path).canonicalFile + if (directory != null) check(directory.mkdirs() || directory.isDirectory) + val next = directory?.let { openDownloadDirectory(it.path) } + try { current.setLibraryCoverCacheDirectory(directory?.path.orEmpty()) } + catch (error: Exception) { next?.close(); throw error } + libraryCoverScope?.close() + libraryCoverScope = next + } + + override fun scanLibraryFolder(folder: String): String = withLibraryDirectories(listOf(folder)) { + it.scanLibraryFolder(File(folder).canonicalPath, null) + } + + override fun scanLibraryFolderToNdjsonFile(folder: String, output: String): Long = + withLibraryDirectories(listOf(folder)) { current -> + require(File(output).isAbsolute && File(output).extension.equals("ndjson", ignoreCase = true)) { + "Library scan output must be an absolute NDJSON path" + } + // The support directory also contains extension storage. Keep the + // Rust write inside its existing private staging root, then publish + // the completed result through the native app's file access. + val staged = File.createTempFile("library_scan_", ".ndjson", File(root, "files")) + try { + val count = current.scanLibraryFolderToNdjsonFile(File(folder).canonicalPath, staged.canonicalPath, null) + check(staged.renameTo(File(output))) { "Failed to publish library scan output" } + count.toLong() + } finally { staged.delete() } + } + + override fun scanLibraryFolderIncremental(folder: String, existing: String): String = + withLibraryDirectories(listOf(folder)) { + it.scanLibraryFolderIncremental(File(folder).canonicalPath, existing, null) + } + + override fun scanLibraryFolderIncrementalFromSnapshot(folder: String, snapshot: String): String = + withLibraryDirectories(listOf(folder)) { current -> + if (snapshot.isEmpty()) current.scanLibraryFolderIncremental(File(folder).canonicalPath, "{}", null) + else { + val staged = File.createTempFile("library_snapshot_", ".ndjson", File(root, "files")) + try { + File(snapshot).copyTo(staged, overwrite = true) + current.scanLibraryFolderIncrementalFromSnapshot(File(folder).canonicalPath, staged.canonicalPath, null) + } finally { staged.delete() } + } + } + + override fun getLibraryScanProgress(): String = + synchronized(this) { manager }?.getLibraryScanProgress() ?: "{}" + + override fun cancelLibraryScan() { synchronized(this) { manager }?.cancelLibraryScan() } + + override fun parseCueSheet(path: String, audioDirectory: String): String { + val cue = File(path).canonicalFile + val audio = if (audioDirectory.isEmpty()) cue.parentFile!! else File(audioDirectory).canonicalFile + return withLibraryDirectories(listOf(cue.parent, audio.path)) { + it.parseCueFileJson(cue.path, audio.path, null) + } + } + + override fun scanCueForLibrary(path: String, audioDirectory: String, virtualPrefix: String, modTime: Long, cacheKey: String): String { + val cue = File(path).canonicalFile + val audio = if (audioDirectory.isEmpty()) cue.parentFile!! else File(audioDirectory).canonicalFile + return withLibraryDirectories(listOf(cue.parent, audio.path)) { + it.scanCueFileForLibrary(cue.path, audio.path, virtualPrefix, modTime, cacheKey, java.time.Instant.now().toString(), null) + } + } + + override fun editFileMetadata(path: String, metadataJson: String): String = + owner().editFileMetadata(File(path).canonicalPath, metadataJson, null) + + private fun mediaPath(path: String): String = if (path.isEmpty()) "" else File(path).canonicalPath + + override fun reEnrichFile(requestJson: String): String { + val request = JSONObject(requestJson) + val path = request.opt("file_path") as? String + if (!request.optBoolean("preview_only", false) && path?.startsWith("/") == true) { + request.put("file_path", mediaPath(path)) + } + return owner().reenrichFile(request.toString(), null) + } + + override fun rewriteSplitArtistTags(path: String, artist: String, albumArtist: String): String = + owner().rewriteSplitArtistTags(mediaPath(path), artist, albumArtist, null) + + override fun extractCoverToFile(audioPath: String, outputPath: String) { + owner().extractCoverToFile(mediaPath(audioPath), mediaPath(outputPath), null) + } + + override fun writeM4aFreeformTags(path: String, metadataJson: String): String = + owner().writeM4aFreeformTags(mediaPath(path), metadataJson, null) + + override fun ensureAc4Config(path: String, reference: String): String = + owner().ensureAc4Config(mediaPath(path), mediaPath(reference), null) + + override fun writeAc4Metadata(path: String, metadataJson: String, coverPath: String): String = + owner().writeAc4Metadata(mediaPath(path), metadataJson, mediaPath(coverPath), null) + + override fun getLyricsLrc( + spotifyId: String, + trackName: String, + artistName: String, + filePath: String, + durationMs: Long, + ): String = owner().getLyricsLrc( + LyricsRequest(spotifyId, trackName, artistName, filePath, durationMs), + null, + ) + + override fun downloadCoverToFileSized( + url: String, + outputPath: String, + maxDimension: Long, + ) = owner().downloadCoverToFileSized( + url, + File(outputPath).canonicalPath, + maxDimension, + null, + ) + + override fun releaseIdleResources() { + owner().releaseMemory(false) + } + + override fun releaseMemoryUnderPressure() { + owner().releaseMemory(true) + } + + override fun createTemporaryMediaFile( + context: Context, + prefix: String, + suffix: String, + ): File { + owner() + return File.createTempFile(prefix, suffix, File(root, "files")) + } + + override fun openExtensionExecution(): CoreExtensionExecution { + val current = owner() + val commands = current.environment().use { it.ffmpegCommands() } + return object : CoreExtensionExecution { + override fun download(requestJson: String): String = current.downloadByStrategy(requestJson) + override fun postProcess(inputJson: String, metadataJson: String): String = + current.runPostProcessing(inputJson, metadataJson, 120_000uL) + override fun waitPending(timeoutMs: Long): List = + parseCoreFFmpegCommands(commands.waitPending(timeoutMs)) + override fun commandIsActive(commandId: String): Boolean = commands.getCommand(commandId).isNotEmpty() + override fun complete(commandId: String, success: Boolean, output: String, error: String) { + commands.complete(commandId, success, output, error) + } + override fun close() { commands.close() } + } + } + + override fun waitForDownloadProgressDelta(since: Long, timeoutMs: Long): String { + val current = owner() + return current.environment().use { environment -> + environment.downloadState().use { state -> + state.waitProgressDelta(since, timeoutMs) + } + } + } + + override fun openDownloadProgress(): CoreDownloadProgress { + val current = owner() + val subscription = current.environment().use { environment -> + environment.downloadState().use { it.subscribeProgress() } + } + return object : CoreDownloadProgress { + override fun waitDelta(since: Long, timeoutMs: Long): String = subscription.waitDelta(since, timeoutMs) + override fun close() { + subscription.stop() + subscription.close() + } + } + } + + override fun initItemProgress(itemId: String) { + val current = owner() + current.environment().use { environment -> + environment.downloadState().use { state -> + state.initItemProgress(itemId) + } + } + } + + override fun clearItemProgress(itemId: String) { + val current = owner() + current.environment().use { environment -> + environment.downloadState().use { state -> + state.clearItemProgress(itemId) + } + } + } + + override fun cancelDownload(itemId: String) { + val current = owner() + current.environment().use { environment -> + environment.downloadState().use { state -> + state.cancelDownload(itemId) + } + } + } + + override fun resetDownloadCancel(itemId: String) { + val current = owner() + current.environment().use { environment -> + environment.downloadState().use { state -> + state.resetDownloadCancel(itemId) + } + } + } + + override fun completeAuthCallback(state: String, code: String, sessionGrant: Boolean, onResolved: (String) -> Unit) { + val current = owner() + current.environment().use { environment -> + val id = if (sessionGrant) environment.resolveCallbackState(state) else environment.consumeCallbackState(state) + onResolved(id) + if (sessionGrant) { + completeSessionGrant(current, id, code) + } else { + environment.setAuthCode(id, code) + current.invokeAction(id, "completeSpotifyLogin") + } + } + } + + private fun completeSessionGrant(current: ExtensionManager, id: String, grant: String) { + current.environment().use { it.setSessionGrant(id, grant) } + requireSuccessfulExtensionAction(id, "completeGrant", current.invokeAction(id, "completeGrant")) + } + + override fun invokeApplication(method: String, arguments: Any?): Any? { + val args = arguments as? Map<*, *> ?: emptyMap() + fun string(key: String, default: String = "") = args[key] as? String ?: default + fun lyricsRequest(fileKey: String) = LyricsRequest( + string("spotify_id"), + string("track_name"), + string("artist_name"), + string(fileKey), + (args["duration_ms"] as? Number)?.toLong() ?: 0L, + ) + fun ids(raw: String): List { + val values = JSONArray(raw) + return (0 until values.length()).map { values.getString(it) } + } + when (method) { + "cancelExtensionRequest" -> synchronized(this) { + requestRegistry().cancel(string("request_id")) + return null + } + "customSearchWithExtension", "getExtensionHomeFeed" -> { + val (current, lease) = acquireRequest(string("request_id")) + return lease.use { + try { + if (method == "customSearchWithExtension") { + current.customSearchJson(string("extension_id"), string("query"), string("options"), it) + } else { + current.getExtensionHomeFeedJson(string("extension_id"), it) + } + } finally { + it.release() + } + } + } + "prepareRuntimeState" -> synchronized(this) { + val directory = File(string("data_dir")) + require(directory.isAbsolute) { "Extension data directory must be absolute" } + val data = directory.canonicalPath + check(manager == null || identity?.second == data) { + "Runtime state directory does not match the initialized owner" + } + val raw = string("runtime_state") + manager?.environment()?.use { it.setRuntimeState(raw) } + runtimeState = data to raw + return null + } + "initExtensionSystem" -> { + initializeOwner(args) + return null + } + "setDownloadDirectory" -> { + setDownloadDirectory(string("path")) + return null + } + "acquireDownloadDirectory" -> synchronized(this) { + val scope = openDownloadDirectory(string("path")) + val token = UUID.randomUUID().toString() + directoryScopes[token] = scope + return token + } + "releaseDownloadDirectory" -> synchronized(this) { + directoryScopes.remove(string("token"))?.close() + return null + } + "cleanupExtensions" -> { + shutdownOwner() + return null + } + "buildFilename" -> return buildFilename(string("template"), string("metadata", "{}")) + "sanitizeFilename" -> return sanitizeFilename(string("filename")) + "readFileMetadata", "editFileMetadata" -> return try { + if (method == "readFileMetadata") { + readFileMetadata(string("file_path"), string("display_name")) + } else { + editFileMetadata(string("file_path"), string("metadata_json", "{}")) + } + } catch (error: Exception) { + JSONObject().put("error", error.message ?: "File metadata operation failed").toString() + } + "getLogsSince" -> return synchronized(this) { + manager?.environment()?.use { environment -> + environment.logBuffer().use { it.since((args["index"] as? Number)?.toLong() ?: 0L) } + } ?: """{"logs":[],"next_index":0}""" + } + "clearLogs" -> return synchronized(this) { + manager?.environment()?.use { environment -> environment.logBuffer().use { it.clear() } } + null + } + "setLoggingEnabled", "setAllowPrivateNetwork", "setDownloadFallbackExtensionIds", + "setNetworkCompatibilityOptions", "setSongLinkNetworkOptions", + "setLyricsProviders", "setLyricsFetchOptions" -> synchronized(this) { + when (method) { + "setLoggingEnabled" -> { + val enabled = args["enabled"] as? Boolean ?: false + manager?.environment()?.use { environment -> + environment.logBuffer().use { it.setEnabled(enabled) } + } + loggingEnabled = enabled + } + "setAllowPrivateNetwork" -> { + val allowed = args["allowed"] as? Boolean ?: false + manager?.environment()?.use { it.setAllowPrivateNetwork(allowed) } + allowPrivateNetwork = allowed + } + "setNetworkCompatibilityOptions", "setSongLinkNetworkOptions" -> { + val allowed = args["allow_http"] as? Boolean ?: false + val insecureTls = args["insecure_tls"] as? Boolean ?: false + manager?.environment()?.use { it.setNetworkCompatibilityOptions(allowed, insecureTls) } + allowHttpFallback = allowed + } + "setDownloadFallbackExtensionIds" -> { + val raw = string("extension_ids") + val value = if (raw.isBlank() || raw.trim() == "null") null else ids(raw) + manager?.setFallbackProviders(value) + fallbackProviders = value + } + "setLyricsProviders" -> { + val raw = string("providers_json", "[]") + if (manager != null) manager!!.setLyricsProvidersJson(raw) else ids(raw) + return """{"success":true}""" + } + "setLyricsFetchOptions" -> { + val raw = string("options_json", "{}") + if (manager != null) manager!!.setLyricsFetchOptionsJson(raw) else JSONObject(raw) + return """{"success":true}""" + } + } + return null + } + "initExtensionRepo" -> { + initializeRepository(string("cache_dir")) + return null + } + "setRepoRegistryUrl" -> { + repositoryOwner().setRegistryUrl(string("registry_url")) + return null + } + "getRepoRegistryUrl" -> return repositoryOwner().registryUrl() + "clearRepoRegistryUrl" -> { + repositoryOwner().clearRegistryUrl() + return null + } + "getRepoExtensions" -> return repositoryOwner().extensions( + args["force_refresh"] as? Boolean ?: false, + ) + "searchRepoExtensions" -> return repositoryOwner().search( + string("query"), + string("category"), + ) + "getRepoCategories" -> return JSONArray(repositoryOwner().categories()).toString() + "downloadRepoExtension" -> return repositoryOwner().download( + string("extension_id"), + string("dest_dir"), + ) + "clearRepoCache" -> { + repositoryOwner().clearCache() + return null + } + } + val current = owner() + return when (method) { + "loadExtensionsFromDir" -> { + synchronized(this) { + check(File(string("dir_path")).canonicalPath == identity?.first) { "Extension source directory does not match the initialized owner" } + } + current.loadAll() + } + "loadExtensionFromPath" -> current.install(string("file_path")) + "upgradeExtension" -> current.upgrade(string("file_path")) + "checkExtensionUpgrade" -> current.checkUpgrade(string("file_path")) + "getInstalledExtensions" -> current.installed() + "setExtensionEnabled" -> { + current.setEnabled(string("extension_id"), args["enabled"] as? Boolean ?: false) + null + } + "unloadExtension" -> { + current.unload(string("extension_id")) + null + } + "removeExtension" -> { + current.remove(string("extension_id")) + null + } + "getExtensionSettings" -> current.environment().use { it.settings(string("extension_id")) } + "setExtensionSettings" -> { + current.updateSettings(string("extension_id"), string("settings", "{}")) + null + } + "invokeExtensionAction" -> current.invokeAction(string("extension_id"), string("action")) + "checkExtensionHealth" -> current.checkExtensionHealthJson(string("extension_id")) + "setProviderPriority", "setMetadataProviderPriority" -> { + current.setProviderPriority(if (method == "setProviderPriority") "download" else "metadata", ids(string("priority", "[]"))) + null + } + "getProviderPriority", "getMetadataProviderPriority" -> { + JSONObject(current.providerPriorities()).optJSONArray(if (method == "getProviderPriority") "download" else "metadata")?.toString() ?: "[]" + } + "getLyricsProviders" -> current.getLyricsProvidersJson() + "getLyricsFetchOptions" -> current.getLyricsFetchOptionsJson() + "getAvailableLyricsProviders" -> current.getAvailableLyricsProvidersJson() + "searchTracksWithMetadataProviders" -> current.searchMetadataProviders( + string("query"), + (args["limit"] as? Number)?.toLong() ?: 20L, + args["include_extensions"] as? Boolean ?: true, + "", + 30_000uL, + ) + "searchTracksWithMetadataProvider" -> current.searchMetadataProvider( + string("extension_id"), + string("query"), + (args["limit"] as? Number)?.toLong() ?: 20L, + 30_000uL, + ) + "getProviderMetadata" -> current.getProviderMetadataJson( + string("provider_id"), + string("resource_type"), + string("resource_id"), + null, + ) + "searchDeezerByISRC" -> current.searchDeezerByIsrcForItemId( + string("isrc"), + string("item_id"), + null, + ) + "getDeezerExtendedMetadata" -> current.getDeezerExtendedMetadata( + string("track_id"), + null, + ) + "convertSpotifyToDeezer" -> current.convertSpotifyToDeezer( + string("resource_type"), + string("spotify_id"), + null, + ) + "getSpotifyIDFromDeezerTrack" -> current.getSpotifyIdFromDeezerTrack( + string("deezer_track_id"), + null, + ) + "getTidalURLFromDeezerTrack" -> current.getTidalUrlFromDeezerTrack( + string("deezer_track_id"), + null, + ) + "getTrackPlatformLinks" -> current.getTrackPlatformLinksJson( + string("spotify_id"), + string("isrc"), + null, + ) + "fetchMusicBrainzTags" -> { + val genre = try { + current.fetchMusicBrainzGenreByIsrc(string("isrc"), null) + } catch (_: Exception) { + "" + } + val albumArtist = try { + current.fetchMusicBrainzAlbumArtistByIsrc( + string("isrc"), + string("album_name"), + null, + ) + } catch (_: Exception) { + "" + } + JSONObject() + .put("genre", genre) + .put("album_artist", albumArtist) + .toString() + } + "getTrackCacheSize" -> current.getTrackCacheSize().toInt() + "clearTrackCache" -> { + current.clearTrackIdCache() + null + } + "setMetadataLanguage" -> { + current.setMetadataLanguage(string("tag")) + null + } + "findURLHandler" -> current.findUrlHandler(string("url")) ?: "" + "handleURLWithExtension" -> current.handleUrlJson(string("url")) + "enrichTrackWithExtension" -> current.enrichTrackJson(string("extension_id"), string("track", "{}")) + "getExtensionPendingAuth" -> current.getExtensionPendingAuthJson(string("extension_id")).ifEmpty { null } + "setExtensionAuthCode" -> current.environment().use { it.setAuthCode(string("extension_id"), string("auth_code")); null } + "completeExtensionSessionGrant" -> { + completeSessionGrant(current, string("extension_id"), string("grant")) + true + } + "setExtensionTokens" -> current.environment().use { + it.setAuthTokens(string("extension_id"), string("access_token"), string("refresh_token"), (args["expires_in"] as? Int)?.toLong() ?: 0L) + null + } + "clearExtensionPendingAuth" -> current.environment().use { it.clearPendingAuth(string("extension_id")); null } + "isExtensionAuthenticated" -> current.environment().use { it.isAuthenticated(string("extension_id")) } + "getAllPendingAuthRequests" -> current.environment().use { it.allPendingAuth() } + "getAllDownloadProgress" -> current.environment().use { environment -> + environment.downloadState().use { state -> state.allProgress() } + } + "cleanupConnections" -> current.environment().use { it.cleanupConnections(); null } + "getPendingFFmpegCommand", "getAllPendingFFmpegCommands", "setFFmpegCommandResult" -> current.environment().use { environment -> + environment.ffmpegCommands().use { commands -> + when (method) { + "getPendingFFmpegCommand" -> commands.getCommand(string("command_id")) + "getAllPendingFFmpegCommands" -> commands.pending() + else -> { + commands.complete(string("command_id"), args["success"] as? Boolean ?: false, string("output"), string("error")) + null + } + } + } + } + "clearItemProgress", "cancelDownload", "resetDownloadCancel" -> current.environment().use { environment -> + environment.downloadState().use { state -> + when (method) { + "clearItemProgress" -> state.clearItemProgress(string("item_id")) + "cancelDownload" -> state.cancelDownload(string("item_id")) + else -> state.resetDownloadCancel(string("item_id")) + } + null + } + } + "getLyricsLRC" -> current.getLyricsLrc(lyricsRequest("file_path"), null) + "getLyricsLRCWithSource" -> current.getLyricsLrcWithSource(lyricsRequest("file_path"), null) + "embedLyricsToFile" -> current.embedLyricsToFile( + string("file_path"), + string("lyrics"), + null, + ) + "fetchAndSaveLyrics" -> try { + current.fetchAndSaveLyrics(lyricsRequest("audio_file_path"), string("output_path"), null) + """{"success":true}""" + } catch (error: Exception) { + JSONObject().put("success", false).put("error", error.message ?: "Lyrics operation failed").toString() + } + "findCollectionAcrossExtensions" -> current.findCollectionAcrossExtensionsJson( + arguments as? String ?: "{}", + null, + ) + else -> error("Rust application method is not connected yet: $method") + } + } + + override fun buildFilename(template: String, metadataJson: String): String = + com.spotiflac.backend.buildFilename(template, metadataJson) + + override fun sanitizeFilename(filename: String): String = + com.spotiflac.backend.sanitizeFilename(filename) +} diff --git a/android/app/src/test/kotlin/com/zarz/spotiflac/CoreFFmpegExecutionTest.kt b/android/app/src/test/kotlin/com/zarz/spotiflac/CoreFFmpegExecutionTest.kt new file mode 100644 index 00000000..27844669 --- /dev/null +++ b/android/app/src/test/kotlin/com/zarz/spotiflac/CoreFFmpegExecutionTest.kt @@ -0,0 +1,528 @@ +package com.zarz.spotiflac + +import java.io.File +import java.nio.file.Files +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +private const val TIMEOUT_SECONDS = 2L + +private fun createTempDirectory(): File { + val directory = File.createTempFile("core-ffmpeg-command-", "") + check(directory.delete()) { "failed to remove temporary file" } + check(directory.mkdir()) { "failed to create temporary directory" } + return directory +} + +private fun awaitWorkerLatch( + latch: CountDownLatch, + name: String, + failures: AtomicReference, +): Boolean { + val reached = latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS) + if (!reached) { + failures.compareAndSet(null, AssertionError("worker timed out waiting for $name")) + } + return reached +} + +private data class Completion( + val success: Boolean, + val output: String, + val error: String, +) + +private class FakeCoreExtensionExecution : CoreExtensionExecution { + private class PendingBatch(val commands: List) + + private val batches = LinkedBlockingQueue() + private val closedBatch = PendingBatch(emptyList()) + private val active = ConcurrentHashMap.newKeySet() + private val completions = ConcurrentHashMap() + private val completionLatches = ConcurrentHashMap() + private val closed = AtomicBoolean(false) + + val waitEntered = CountDownLatch(1) + val closedHandle = CountDownLatch(1) + val secondClose = CountDownLatch(1) + val closeCalls = AtomicInteger(0) + + fun enqueue(vararg commands: CoreFFmpegCommand) { + commands.forEach { command -> + active.add(command.id) + completionLatches[command.id] = CountDownLatch(1) + } + batches.offer(PendingBatch(commands.toList())) + } + + fun remove(commandId: String) { + active.remove(commandId) + } + + fun isActive(commandId: String): Boolean = active.contains(commandId) + + fun isClosed(): Boolean = closed.get() + + fun awaitCompletion(commandId: String): Boolean = + completionLatches.getValue(commandId).await(TIMEOUT_SECONDS, TimeUnit.SECONDS) + + fun hasCompletion(commandId: String): Boolean = completions.containsKey(commandId) + + fun completion(commandId: String): Completion = completions.getValue(commandId) + + override fun download(requestJson: String): String = error("unused fake download") + + override fun postProcess(inputJson: String, metadataJson: String): String = + error("unused fake post-process") + + override fun waitPending(timeoutMs: Long): List { + waitEntered.countDown() + if (closed.get()) throw IllegalStateException("execution closed") + val batch = batches.poll(timeoutMs, TimeUnit.MILLISECONDS) ?: return emptyList() + if (batch === closedBatch) throw IllegalStateException("execution closed") + return batch.commands + } + + override fun commandIsActive(commandId: String): Boolean = active.contains(commandId) + + override fun complete( + commandId: String, + success: Boolean, + output: String, + error: String, + ) { + completions[commandId] = Completion(success, output, error) + active.remove(commandId) + completionLatches.getValue(commandId).countDown() + } + + override fun close() { + val calls = closeCalls.incrementAndGet() + if (calls >= 2) secondClose.countDown() + if (closed.compareAndSet(false, true)) { + closedHandle.countDown() + batches.offer(closedBatch) + } + } +} + +class CoreFFmpegExecutionTest { + @Test + fun backendCancellationRemainsCancellationForNativeFinalization() { + val execution = FakeCoreExtensionExecution() + val failure = runCatching { + withCoreFFmpegExecution(execution) { throw IllegalStateException("download cancelled") } + }.exceptionOrNull() + assertTrue(failure is java.util.concurrent.CancellationException) + assertTrue(failure?.cause is IllegalStateException) + assertTrue(execution.closedHandle.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + } + + @Test + fun callerFinishingDoesNotInterruptClaimedCommandOrCloseHandle() { + val execution = FakeCoreExtensionExecution() + val commandStarted = CountDownLatch(1) + val releaseCommand = CountDownLatch(1) + val workerFailure = AtomicReference(null) + execution.enqueue(CoreFFmpegCommand("command-b", arrayOf("convert-b"))) + + try { + val result = withCoreFFmpegExecution( + execution, + execute = { arguments, cancelled -> + if (arguments.firstOrNull() != "convert-b") { + workerFailure.compareAndSet( + null, + AssertionError("unexpected command: ${arguments.toList()}"), + ) + false to "unexpected command" + } else if (cancelled()) { + workerFailure.compareAndSet( + null, + AssertionError("claimed command B was cancelled"), + ) + false to "cancelled" + } else { + commandStarted.countDown() + if (!awaitWorkerLatch(releaseCommand, "command B release", workerFailure)) { + false to "worker timed out" + } else { + true to "command B complete" + } + } + }, + ) { + assertTrue(commandStarted.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + "caller A complete" + } + + assertEquals("caller A complete", result) + assertFalse(execution.isClosed()) + assertFalse(execution.hasCompletion("command-b")) + + releaseCommand.countDown() + assertTrue(execution.awaitCompletion("command-b")) + assertTrue(execution.closedHandle.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + assertEquals(true, execution.completion("command-b").success) + assertNull(workerFailure.get()) + } finally { + releaseCommand.countDown() + execution.close() + execution.closedHandle.await(TIMEOUT_SECONDS, TimeUnit.SECONDS) + } + } + + @Test + fun registryRemovalCancelsOnlyCommandAWhileCommandBRemainsActive() { + val execution = FakeCoreExtensionExecution() + val commandAStarted = CountDownLatch(1) + val releaseCommandA = CountDownLatch(1) + val commandBStarted = CountDownLatch(1) + val releaseCommandB = CountDownLatch(1) + val workerFailure = AtomicReference(null) + execution.enqueue( + CoreFFmpegCommand("command-a", arrayOf("convert-a")), + CoreFFmpegCommand("command-b", arrayOf("convert-b")), + ) + + try { + val result = withCoreFFmpegExecution( + execution, + execute = { arguments, cancelled -> + if (arguments.firstOrNull() == "convert-a") { + commandAStarted.countDown() + if (!awaitWorkerLatch(releaseCommandA, "command A release", workerFailure)) { + false to "worker timed out" + } else if (!cancelled()) { + workerFailure.compareAndSet( + null, + AssertionError("command A was not cancelled"), + ) + false to "not cancelled" + } else { + false to "cancelled" + } + } else if (arguments.firstOrNull() != "convert-b") { + workerFailure.compareAndSet( + null, + AssertionError("unexpected command: ${arguments.toList()}"), + ) + false to "unexpected command" + } else if (cancelled()) { + workerFailure.compareAndSet( + null, + AssertionError("command B was cancelled"), + ) + false to "cancelled" + } else { + commandBStarted.countDown() + if (!awaitWorkerLatch(releaseCommandB, "command B release", workerFailure)) { + false to "worker timed out" + } else { + true to "command B complete" + } + } + }, + ) { + assertTrue(commandAStarted.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + execution.remove("command-a") + releaseCommandA.countDown() + assertTrue(commandBStarted.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + "caller A complete" + } + + assertEquals("caller A complete", result) + assertFalse(execution.isClosed()) + assertTrue(execution.isActive("command-b")) + assertTrue(execution.awaitCompletion("command-a")) + assertFalse(execution.completion("command-a").success) + assertEquals("cancelled", execution.completion("command-a").error) + + releaseCommandB.countDown() + assertTrue(execution.awaitCompletion("command-b")) + assertTrue(execution.closedHandle.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + assertTrue(execution.completion("command-b").success) + assertNull(workerFailure.get()) + } finally { + releaseCommandA.countDown() + releaseCommandB.countDown() + execution.close() + execution.closedHandle.await(TIMEOUT_SECONDS, TimeUnit.SECONDS) + } + } + + @Test + fun emptyArgumentsFailAndFollowingCommandStillRuns() { + val execution = FakeCoreExtensionExecution() + val validCommandStarted = CountDownLatch(1) + val releaseValidCommand = CountDownLatch(1) + val workerFailure = AtomicReference(null) + val executed = ConcurrentLinkedQueue() + execution.enqueue( + CoreFFmpegCommand("empty-command", emptyArray()), + CoreFFmpegCommand("valid-command", arrayOf("convert")), + ) + + try { + val result = withCoreFFmpegExecution( + execution, + execute = { arguments, _ -> + executed.add(arguments.firstOrNull() ?: "") + if (arguments.firstOrNull() != "convert") { + workerFailure.compareAndSet( + null, + AssertionError("empty command reached executor"), + ) + false to "unexpected command" + } else { + validCommandStarted.countDown() + if (!awaitWorkerLatch( + releaseValidCommand, + "valid command release", + workerFailure, + ) + ) { + false to "worker timed out" + } else { + true to "valid command complete" + } + } + }, + ) { + assertTrue(validCommandStarted.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + "caller complete" + } + + assertEquals("caller complete", result) + assertEquals(listOf("convert"), executed.toList()) + assertTrue(execution.awaitCompletion("empty-command")) + assertFalse(execution.completion("empty-command").success) + assertEquals("FFmpeg arguments are empty", execution.completion("empty-command").error) + + releaseValidCommand.countDown() + assertTrue(execution.awaitCompletion("valid-command")) + assertTrue(execution.closedHandle.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + assertTrue(execution.completion("valid-command").success) + assertNull(workerFailure.get()) + } finally { + releaseValidCommand.countDown() + execution.close() + execution.closedHandle.await(TIMEOUT_SECONDS, TimeUnit.SECONDS) + } + } + + @Test + fun closedWaitExitsAndPumpReleasesHandle() { + val execution = FakeCoreExtensionExecution() + val workerFailure = AtomicReference(null) + + val result = withCoreFFmpegExecution( + execution, + execute = { _, _ -> + workerFailure.compareAndSet(null, AssertionError("unexpected command")) + false to "unexpected command" + }, + ) { + assertTrue(execution.waitEntered.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + execution.close() + "caller complete" + } + + assertEquals("caller complete", result) + assertTrue(execution.closedHandle.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + assertTrue(execution.secondClose.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) + assertNull(workerFailure.get()) + } + + @Test + fun cancelledExecutionPreservesExistingTargetAndCleansStaging() { + val directory = createTempDirectory() + try { + val target = File(directory, "output.flac") + target.writeText("old output") + val arguments = arrayOf("-i", "input.wav", target.absolutePath) + val command = CoreFFmpegCommand("cancel", arguments, target.absolutePath) + val cancelled = AtomicBoolean(false) + var receivedArguments: Array? = null + var staging: File? = null + + val result = executeCoreFFmpegCommand( + command, + cancelled = { cancelled.get() }, + ) { received, _ -> + receivedArguments = received.copyOf() + staging = File(received.last()).also { it.writeText("partial output") } + cancelled.set(true) + true to "converted" + } + + assertFalse(result.first) + assertEquals(arguments.dropLast(1).toList(), receivedArguments!!.dropLast(1).toList()) + assertEquals("old output", target.readText()) + assertFalse(staging!!.exists()) + } finally { + directory.deleteRecursively() + } + } + + @Test + fun failedExecutionLeavesMissingTargetAndCleansStaging() { + val directory = createTempDirectory() + try { + val target = File(directory, "output.flac") + val arguments = arrayOf("-i", "input.wav", target.absolutePath) + val command = CoreFFmpegCommand("failed", arguments, target.absolutePath) + var staging: File? = null + + val result = executeCoreFFmpegCommand(command, cancelled = { false }) { received, _ -> + staging = File(received.last()).also { it.writeText("partial output") } + false to "conversion failed" + } + + assertFalse(result.first) + assertFalse(target.exists()) + assertFalse(staging!!.exists()) + } finally { + directory.deleteRecursively() + } + } + + @Test + fun successfulExecutionReplacesTargetPreservesInputArgumentsAndCleansStaging() { + val directory = createTempDirectory() + try { + val input = File(directory, "input.wav") + input.writeText("input") + val target = File(directory, "output.flac") + target.writeText("old output") + val arguments = arrayOf("-i", input.absolutePath, "-c:a", "flac", target.absolutePath) + val command = CoreFFmpegCommand("success", arguments, target.absolutePath) + var receivedArguments: Array? = null + var staging: File? = null + + val result = executeCoreFFmpegCommand(command, cancelled = { false }) { received, _ -> + receivedArguments = received.copyOf() + staging = File(received.last()).also { it.writeText("new complete output") } + true to "converted" + } + + assertTrue(result.first) + assertEquals(arguments.dropLast(1).toList(), receivedArguments!!.dropLast(1).toList()) + assertTrue(receivedArguments!!.last() != target.absolutePath) + assertEquals(target.parentFile.canonicalFile, staging!!.parentFile) + assertEquals(target.extension, staging!!.extension) + assertEquals("new complete output", target.readText()) + assertFalse(staging!!.exists()) + } finally { + directory.deleteRecursively() + } + } + + @Test + fun firstExecutionSweepsOnlyOwnOrphanFilesAndPreservesOtherEntries() { + val directory = createTempDirectory() + try { + val prefix = ".spotiflac-ffmpeg-${BuildConfig.APPLICATION_ID}-" + val ownOrphan = File(directory, "$prefix${UUID.randomUUID()}.flac") + .also { it.writeText("orphan") } + val foreignFile = File(directory, ".spotiflac-ffmpeg-foreign-${UUID.randomUUID()}.flac") + .also { it.writeText("foreign") } + val malformedFile = File(directory, "${prefix}not-a-uuid.flac") + .also { it.writeText("malformed") } + val preservedDirectory = File(directory, "$prefix${UUID.randomUUID()}.flac") + check(preservedDirectory.mkdir()) { "failed to create preserved directory" } + val directoryContent = File(preservedDirectory, "content.txt") + .also { it.writeText("directory content") } + val symlinkTarget = File(directory, "symlink-target.txt") + .also { it.writeText("symlink content") } + val symlink = File(directory, "$prefix${UUID.randomUUID()}.flac") + Files.createSymbolicLink(symlink.toPath(), symlinkTarget.toPath()) + + val target = File(directory, "output.flac") + val command = CoreFFmpegCommand("sweep", arrayOf("convert", target.absolutePath), target.absolutePath) + var staged: File? = null + val result = executeCoreFFmpegCommand(command, cancelled = { false }) { arguments, _ -> + assertFalse(ownOrphan.exists()) + assertEquals("foreign", foreignFile.readText()) + assertEquals("malformed", malformedFile.readText()) + assertTrue(preservedDirectory.isDirectory) + assertEquals("directory content", directoryContent.readText()) + assertTrue(Files.isSymbolicLink(symlink.toPath())) + assertEquals("symlink content", symlink.readText()) + + staged = File(arguments.last()).also { it.writeText("published") } + true to "converted" + } + + assertTrue(result.first) + assertEquals("published", target.readText()) + assertFalse(staged!!.exists()) + assertFalse(ownOrphan.exists()) + assertEquals("foreign", foreignFile.readText()) + assertEquals("malformed", malformedFile.readText()) + assertEquals("directory content", directoryContent.readText()) + assertTrue(Files.isSymbolicLink(symlink.toPath())) + assertEquals("symlink content", symlink.readText()) + } finally { + directory.deleteRecursively() + } + } + + @Test + fun overlappingExecutionsPreserveLiveStagingAndPublishBothOutputs() { + val directory = createTempDirectory() + try { + val firstTarget = File(directory, "first.flac") + val secondTarget = File(directory, "second.flac") + val firstCommand = CoreFFmpegCommand( + "first", + arrayOf("convert-first", firstTarget.absolutePath), + firstTarget.absolutePath, + ) + val secondCommand = CoreFFmpegCommand( + "second", + arrayOf("convert-second", secondTarget.absolutePath), + secondTarget.absolutePath, + ) + var firstStage: File? = null + var secondStage: File? = null + + val result = executeCoreFFmpegCommand(firstCommand, cancelled = { false }) { arguments, _ -> + firstStage = File(arguments.last()).also { it.writeText("first staged") } + val nestedResult = executeCoreFFmpegCommand(secondCommand, cancelled = { false }) { nestedArguments, _ -> + secondStage = File(nestedArguments.last()) + assertTrue(firstStage!!.exists()) + assertEquals("first staged", firstStage!!.readText()) + secondStage!!.writeText("second published") + true to "second complete" + } + + assertTrue(nestedResult.first) + assertEquals("second published", secondTarget.readText()) + assertTrue(firstStage!!.exists()) + assertEquals("first staged", firstStage!!.readText()) + true to "first complete" + } + + assertTrue(result.first) + assertEquals("first staged", firstTarget.readText()) + assertEquals("second published", secondTarget.readText()) + assertFalse(firstStage!!.exists()) + assertFalse(secondStage!!.exists()) + } finally { + directory.deleteRecursively() + } + } +} diff --git a/android/app/src/test/kotlin/com/zarz/spotiflac/SafMetadataReadPolicyTest.kt b/android/app/src/test/kotlin/com/zarz/spotiflac/SafMetadataReadPolicyTest.kt index 32f27e22..84e3e760 100644 --- a/android/app/src/test/kotlin/com/zarz/spotiflac/SafMetadataReadPolicyTest.kt +++ b/android/app/src/test/kotlin/com/zarz/spotiflac/SafMetadataReadPolicyTest.kt @@ -1,11 +1,61 @@ package com.zarz.spotiflac import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Assert.assertNull import org.junit.Test +import java.io.File class SafMetadataReadPolicyTest { + @Test fun lyricsReadUsesSafCopyAndDeletesItAfterSuccessOrFailure() { + for (fails in listOf(false, true)) { + val temporary = File.createTempFile("lyrics_", ".flac") + temporary.writeText("embedded lyrics fixture") + try { + val result = runCatching { + readLyricsWithSafCopy( + "content://example.documents/track.flac", + copyToTemp = { uri -> + assertEquals("content://example.documents/track.flac", uri) + temporary + }, + read = { path -> + assertEquals(temporary.absolutePath, path) + assertEquals("flac", File(path).extension) + assertTrue(File(path).exists()) + if (fails) throw IllegalStateException("read failed") + File(path).readText() + }, + ) + } + assertEquals(fails, result.isFailure) + if (!fails) assertEquals("embedded lyrics fixture", result.getOrThrow()) + assertFalse(temporary.exists()) + } finally { + temporary.delete() + } + } + } + + @Test fun revokedSafLyricsReadDoesNotFetchOnline() { + assertNull(readLyricsWithSafCopy( + "content://example.documents/revoked.flac", + copyToTemp = { null }, + read = { error("Unreadable document must not become an online request") }, + )) + } + + @Test fun localAndOnlineLyricsReadsKeepOriginalPath() { + for (path in listOf("/example/track.flac", "")) { + assertEquals(path, readLyricsWithSafCopy( + path, + copyToTemp = { error("Unexpected SAF copy") }, + read = { it }, + )) + } + } + @Test fun completeMetadataReadErrorUsesTemporaryCopy() { val metadata = mapOf("lyrics" to "words", "comment" to "notes", "track_number" to 3) assertEquals(metadata, readSafMetadataWithFallback( diff --git a/docs/EXTENSION_DEVELOPMENT.md b/docs/EXTENSION_DEVELOPMENT.md deleted file mode 100644 index e5eaf32b..00000000 --- a/docs/EXTENSION_DEVELOPMENT.md +++ /dev/null @@ -1,278 +0,0 @@ -# SpotiFLAC Mobile Extension Development - -This guide defines the extension package and manifest contract implemented by -the current SpotiFLAC Mobile codebase. The expanded runtime API reference is -available at . - -## Quick start - -Create a directory with these two root files: - -```text -my-extension/ -├── manifest.json -└── index.js -``` - -Use the current camel-case manifest schema: - -```json -{ - "name": "my-extension", - "displayName": "My Extension", - "version": "1.0.0", - "description": "What this extension provides", - "homepage": "https://github.com/you/my-extension", - "type": ["metadata_provider"], - "minAppVersion": "4.2.3", - "permissions": { - "network": ["api.example.com", "*.example.com"], - "storage": false, - "file": false - }, - "settings": [] -} -``` - -Register the implementation from `index.js`: - -```js -registerExtension({ - searchTracks: async function (query, limit) { - const response = await http.get( - "https://api.example.com/search?q=" + - encodeURIComponent(query) + - "&limit=" + - String(limit) - ); - - return { - tracks: response.data.items.map((item) => ({ - id: String(item.id), - name: item.title, - artist: item.artist, - album_name: item.album, - duration_ms: item.duration_ms, - cover_url: item.cover_url - })), - total: response.data.total - }; - } -}); -``` - -Package the contents—not their parent directory—as ZIP: - -```bash -cd my-extension -zip -r ../my-extension.sflx manifest.json index.js -``` - -An extension package is a plain ZIP archive renamed to `.sflx`. The longer -`.spotiflac-ext` suffix is the legacy alias; both are accepted everywhere -(manual import, repo downloads) and the layout is identical. Use `.sflx` for -new packages. - -`manifest.json` and `index.js` must be unique files at the archive root. -SpotiFLAC Mobile rejects traversal paths, symlinks, duplicate paths, oversized -manifests, and archives whose extracted size exceeds the safety limit. - -## Manifest contract - -The parser lives in -[`go_backend/extension_manifest.go`](../go_backend/extension_manifest.go). -Use these exact field names: - -| Field | Required | Contract | -| --- | --- | --- | -| `name` | yes | Stable lowercase ID matching `^[a-z0-9][a-z0-9._-]{0,127}$` | -| `displayName` | recommended | Human-readable Store and settings label | -| `version` | yes | Numeric dotted version used by upgrade comparison | -| `description` | yes | Human-readable purpose | -| `type` | yes | Array containing `metadata_provider`, `download_provider`, or `lyrics_provider` | -| `permissions` | yes | Capability object described below | -| `homepage`, `icon`, `minAppVersion` | no | `icon` is a path inside the package | -| `settings` | no | Extension settings shown by the app | -| `qualityOptions` | download provider | Download quality IDs passed to `download()` | -| `searchBehavior` | no | Generic search-tab behavior | -| `urlHandler` | no | URL matching declarations | -| `trackMatching` | no | Generic matching strategy | -| `postProcessing` | no | Generic post-processing hooks | -| `serviceHealth` | no | Health checks shown by the app | -| `signedSession` | no | Signed-session bootstrap contract | -| `requiredRuntimeFeatures` | no | Runtime feature requirements | -| `capabilities` | no | Generic extension capability declarations | - -The behavior flags currently supported are `skipMetadataEnrichment`, -`skipLyrics`, `stopProviderFallback`, and `skipBuiltInFallback`. New -extension-specific behavior must be added as a generic manifest capability; -the host must not branch on a particular provider ID. - -Extensions that depend on canonical gateway/provider error ownership should -declare `requiredRuntimeFeatures: ["signedSession@2"]`. Version 2 only mutates -gateway session state for the exact `SESSION_INVALID/bootstrap_session` or -`VERIFY_REQUIRED/verify` contracts, exposes canonical error fields to JS, and -keeps provider-owned failures separate. `403 REQUEST_AUTH_INVALID` preserves -the session and only retries once when a newer session generation is already -available. BYOA -reauthentication remains a separate provider action. An active generation that -receives canonical `428 VERIFY_REQUIRED` is blocked in the shared coordinator, -so later requests join the same verification flow instead of hitting the -gateway repeatedly. - -Extensions that depend on provider retry modes should require -`signedSession@3`. The runtime exposes `retryMode` and only auto-retries -`PROVIDER_UNAVAILABLE` when `retryable: true` is paired with -`retry_mode: "same_operation"`. Every such retry is newly signed with a fresh -timestamp and nonce. `new_ticket`, `poll_existing`, `none`, missing, and unknown -modes are returned to the extension without automatic replay. - -Do not use legacy spellings such as `display_name`, `types`, -`permissions.network.domains`, or an object for `permissions.network`. - -### Download quality across providers - -Quality IDs belong to the provider that declares them. Add an optional `kind` -to each `qualityOptions` entry: `lossless`, `lossy`, or `spatial`. - -```json -{"id": "best", "label": "Best FLAC", "kind": "lossless"} -``` - -On fallback, the host preserves a compatible quality ID or selects the first -option of the same kind. A lossless request cannot select Atmos or a lossy -tier just because it appears first. Spatial and lossy requests can use -lossless when the target has no option of the requested kind. Providers with -no compatible option are skipped. The host infers kinds from legacy IDs and -labels, with `downloadFallbackTier` helping classify `best` and `default`; -explicit kinds avoid ambiguity for custom IDs. Descriptions are not used -because they may describe other fallback formats. - -### Permissions - -```json -{ - "permissions": { - "network": ["api.example.com", "*.cdn.example.com"], - "storage": true, - "file": false, - "allowHttp": false - } -} -``` - -- `network` is an array of allowed host names. HTTPS is required unless - `allowHttp` is explicitly enabled. -- `storage` is required for extension storage and signed-session state. -- `file` is required for file and raw FFmpeg capabilities. - -Request only what the extension needs. The runtime denies undeclared network, -storage, and file access. - -## Downloading files - -Each `download(trackID, quality, outputPath, onProgress, options)` call has a -cumulative 60-second stream-resolution allowance. It covers metadata/ticket -requests, URL resolution, retry waits, and refreshes, including HTTP headers -and the first audio byte. Native audio reads after the first byte and bounded -native FFmpeg conversion pause this clock; transfer stall limits and the -overall download timeout still apply. Progress callbacks do not reset it. -Within one native segmented or chunked download, a successfully received part -establishes the transfer: headers and the first byte of subsequent parts also -pause the clock. Retry attempts and waits still consume the allowance, and a -new native download call must establish its own transfer. - -`options.resolutionTimeoutMs` reports the initial allowance, while -`utils.getResolutionRemainingMs()` reports the remaining allowance. Check for -the function before using it on older hosts. Stop with `error_type: "timeout"` -if a server's required retry delay cannot fit; do not shorten `Retry-After` -to force another request. The host also enforces the deadline for extensions -that do not check it themselves. - -Extensions with `permissions.file: true` can stream a remote file into their -allowed output path: - -```js -const result = file.download(downloadUrl, outputPath, { - headers: { - "User-Agent": "My Extension/1.0" - }, - onProgress: function (written, total) { - log.debug("Downloaded", written, "of", total, "bytes"); - }, - resume: true -}); -``` - -The third argument is optional and supports: - -| Option | Type | Default | Contract | -| --- | --- | --- | --- | -| `headers` | object | `{}` | Additional request headers. Do not set `Range` when using runtime-managed resume. | -| `onProgress` | function | none | Called as `onProgress(writtenBytes, totalBytes)` when the total is known. | -| `trackItemBytes` | boolean | `true` | Publishes byte progress to the host download queue. The legacy alias `track_item_bytes` is also accepted. | -| `resume` | boolean | `false` | Allows up to three mid-body Range resumes for the normal streaming mode. | -| `chunked` | boolean or positive number | `false` | Uses sequential Range requests. `true` selects 1 MiB chunks; a positive number sets the chunk size in bytes. | - -`resume` is deliberately opt-in. It is attempted only when the server returns -a strong `ETag` or `Last-Modified` validator. Resumed responses must return the -expected `206 Content-Range`; if the server returns `200`, the staged file is -truncated and restarted from byte zero. Enable it only when the origin -guarantees that the same URL and validator identify byte-identical content -across retries and network changes. A CDN can otherwise splice bytes from two -different objects into one apparently successful file. - -Use `chunked` for origins that require bounded Range requests, such as some -media CDNs. Chunked mode has its own per-chunk retries and does not use the -`resume` option. In every mode SpotiFLAC Mobile writes to a staged sibling file -and publishes the final path only after the download completes successfully. - -## Store registry integrity - -Repository maintainers should publish a SHA-256 digest for every package: - -```json -{ - "version": 1, - "extensions": [ - { - "id": "my-extension", - "name": "my-extension", - "display_name": "My Extension", - "version": "1.0.0", - "description": "What this extension provides", - "category": "metadata", - "download_url": "https://example.com/my-extension.sflx", - "sha256": "64-lowercase-hex-characters" - } - ] -} -``` - -Generate the digest after building the package: - -```bash -sha256sum my-extension.sflx -``` - -Store downloads with a published digest are written to a temporary file, -hashed, and only moved into place after the digest matches. A mismatch aborts -installation and preserves any previously cached package. Legacy registry -entries without `sha256` remain compatible but cannot provide package -integrity verification. - -A checksum authenticates a package only as strongly as the HTTPS registry that -publishes it. A manually imported `.spotiflac-ext` or `.sflx` package has no -registry trust context, so install manual packages only from a publisher you -trust. - -## Compatibility checklist - -Before publishing: - -1. Validate that `manifest.json` uses the exact current field names. -2. Keep `manifest.json` and `index.js` at the archive root. -3. Declare every network host and runtime permission used. -4. Set `minAppVersion` when relying on a recently added capability. -5. Test install, enable, disable, upgrade, and removal. -6. Publish the package SHA-256 in the repository registry. diff --git a/go_backend/ac4_config.go b/go_backend/ac4_config.go deleted file mode 100644 index 0d044fcf..00000000 --- a/go_backend/ac4_config.go +++ /dev/null @@ -1,408 +0,0 @@ -package gobackend - -import ( - "encoding/binary" - "fmt" - "io" - "os" -) - -// mp4Box is a minimal ISO-BMFF / QuickTime box view over an in-memory buffer. -type mp4Box struct { - offset int64 - size int64 - hdr int64 - typ string -} - -func (b mp4Box) body() int64 { return b.offset + b.hdr } -func (b mp4Box) end() int64 { return b.offset + b.size } - -func readMP4Box(data []byte, pos int64) (mp4Box, bool) { - n := int64(len(data)) - if pos < 0 || pos+8 > n { - return mp4Box{}, false - } - size := int64(binary.BigEndian.Uint32(data[pos : pos+4])) - typ := string(data[pos+4 : pos+8]) - hdr := int64(8) - switch size { - case 1: - if pos+16 > n { - return mp4Box{}, false - } - size = int64(binary.BigEndian.Uint64(data[pos+8 : pos+16])) - hdr = 16 - case 0: - size = n - pos - } - if size < hdr || pos+size > n { - return mp4Box{}, false - } - return mp4Box{offset: pos, size: size, hdr: hdr, typ: typ}, true -} - -func findChildMP4(data []byte, start, end int64, typ string) (mp4Box, bool) { - pos := start - for pos+8 <= end { - b, ok := readMP4Box(data, pos) - if !ok { - return mp4Box{}, false - } - if b.typ == typ { - return b, true - } - pos = b.end() - } - return mp4Box{}, false -} - -func eachChildMP4(data []byte, start, end int64, typ string, fn func(mp4Box) bool) { - pos := start - for pos+8 <= end { - b, ok := readMP4Box(data, pos) - if !ok { - return - } - if b.typ == typ && !fn(b) { - return - } - pos = b.end() - } -} - -// findBoxBySignature scans [start,end) for a box of the given type, matching the -// 4-byte type tag and validating the preceding size field. Used to locate dac4 -// which may be nested inside an encrypted (enca) sample entry. -func findBoxBySignature(data []byte, start, end int64, typ string) (mp4Box, bool) { - if len(typ) != 4 { - return mp4Box{}, false - } - for i := start; i+8 <= end; i++ { - if data[i+4] == typ[0] && data[i+5] == typ[1] && data[i+6] == typ[2] && data[i+7] == typ[3] { - if b, ok := readMP4Box(data, i); ok && b.typ == typ { - return b, true - } - } - } - return mp4Box{}, false -} - -// audioSampleEntryHeaderLen returns the byte length of the fixed audio sample -// entry header (from the box body start) before child boxes begin. ok is false -// for malformed/truncated entries whose declared header is not fully present. -func audioSampleEntryHeaderLen(data []byte, entry mp4Box) (hdrLen int64, ok bool) { - // 6 bytes reserved + 2 bytes data_reference_index, then the audio fields. - base := entry.body() - if base+10 > entry.end() { - return 0, false - } - version := binary.BigEndian.Uint16(data[base+8 : base+10]) - hdrLen = 8 + 20 - switch version { - case 1: - hdrLen += 16 - case 2: - hdrLen += 36 - } - if base+hdrLen > entry.end() { - return 0, false - } - return hdrLen, true -} - -type ac4Location struct { - chain []mp4Box // moov, trak, mdia, minf, stbl, stsd (ancestors to grow) - entry mp4Box // the ac-4 sample entry -} - -func locateAC4Entry(data []byte) (ac4Location, bool) { - moov, ok := findChildMP4(data, 0, int64(len(data)), "moov") - if !ok { - return ac4Location{}, false - } - var found ac4Location - var ok2 bool - eachChildMP4(data, moov.body(), moov.end(), "trak", func(trak mp4Box) bool { - mdia, ok := findChildMP4(data, trak.body(), trak.end(), "mdia") - if !ok { - return true - } - minf, ok := findChildMP4(data, mdia.body(), mdia.end(), "minf") - if !ok { - return true - } - stbl, ok := findChildMP4(data, minf.body(), minf.end(), "stbl") - if !ok { - return true - } - stsd, ok := findChildMP4(data, stbl.body(), stbl.end(), "stsd") - if !ok { - return true - } - entry, ok := findChildMP4(data, stsd.body()+8, stsd.end(), "ac-4") - if !ok { - return true - } - found = ac4Location{chain: []mp4Box{moov, trak, mdia, minf, stbl, stsd}, entry: entry} - ok2 = true - return false - }) - return found, ok2 -} - -func growBoxSize(data []byte, b mp4Box, delta int64) { - if b.hdr == 16 { - binary.BigEndian.PutUint64(data[b.offset+8:b.offset+16], uint64(b.size+delta)) - } else { - binary.BigEndian.PutUint32(data[b.offset:b.offset+4], uint32(b.size+delta)) - } -} - -// shiftChunkOffsets adds delta to every stco/co64 entry that references a file -// offset at or beyond insertPos, keeping sample pointers valid after bytes are -// inserted into moov. -func shiftChunkOffsets(data []byte, moov mp4Box, insertPos, delta int64) { - eachChildMP4(data, moov.body(), moov.end(), "trak", func(trak mp4Box) bool { - mdia, ok := findChildMP4(data, trak.body(), trak.end(), "mdia") - if !ok { - return true - } - minf, ok := findChildMP4(data, mdia.body(), mdia.end(), "minf") - if !ok { - return true - } - stbl, ok := findChildMP4(data, minf.body(), minf.end(), "stbl") - if !ok { - return true - } - if stco, ok := findChildMP4(data, stbl.body(), stbl.end(), "stco"); ok { - base := stco.body() + 4 - if base+4 <= stco.end() { - count := int64(binary.BigEndian.Uint32(data[base : base+4])) - p := base + 4 - for i := int64(0); i < count && p+4 <= stco.end(); i++ { - v := int64(binary.BigEndian.Uint32(data[p : p+4])) - if v >= insertPos { - binary.BigEndian.PutUint32(data[p:p+4], uint32(v+delta)) - } - p += 4 - } - } - } - if co64, ok := findChildMP4(data, stbl.body(), stbl.end(), "co64"); ok { - base := co64.body() + 4 - if base+4 <= co64.end() { - count := int64(binary.BigEndian.Uint32(data[base : base+4])) - p := base + 4 - for i := int64(0); i < count && p+8 <= co64.end(); i++ { - v := int64(binary.BigEndian.Uint64(data[p : p+8])) - if v >= insertPos { - binary.BigEndian.PutUint64(data[p:p+8], uint64(v+delta)) - } - p += 8 - } - } - } - return true - }) -} - -// normalizeQuickTimeBrandsInBuf rewrites a "qt " ftyp brand to isom/mp42 in -// place. Works on any buffer whose top level contains the ftyp box (whole file -// or the ftyp box alone). Returns whether anything changed. -func normalizeQuickTimeBrandsInBuf(data []byte) bool { - ftyp, ok := findChildMP4(data, 0, int64(len(data)), "ftyp") - if !ok { - return false - } - changed := false - if ftyp.body()+4 <= int64(len(data)) && string(data[ftyp.body():ftyp.body()+4]) != "mp42" { - copy(data[ftyp.body():ftyp.body()+4], []byte("mp42")) - changed = true - } - for p := ftyp.body() + 8; p+4 <= ftyp.end(); p += 4 { - if string(data[p:p+4]) == "qt " { - copy(data[p:p+4], []byte("isom")) - changed = true - } - } - return changed -} - -// normalizeQuickTimeAudioEntry rewrites a version-1 QuickTime sound sample -// entry into a plain version-0 AudioSampleEntry, dropping the 16-byte v1 -// extension. base is the buffer's absolute file offset (0 for a whole-file -// buffer, the moov offset for a moov-only buffer). -func normalizeQuickTimeAudioEntry(data []byte, base int64) []byte { - loc, ok := locateAC4Entry(data) - if !ok { - return data - } - entry := loc.entry - verPos := entry.body() + 8 - if verPos+2 > entry.end() { - return data - } - if binary.BigEndian.Uint16(data[verPos:verPos+2]) != 1 { - return data // already v0 (or v2, left untouched) - } - - // The v1 QuickTime sound extension is the 16 bytes following the 20-byte v0 - // audio fields (samplesPerPacket, bytesPerPacket, bytesPerFrame, bytesPerSample). - extStart := entry.body() + 8 + 20 - extEnd := extStart + 16 - if extEnd > entry.end() { - return data - } - delta := int64(-16) - - binary.BigEndian.PutUint16(data[verPos:verPos+2], 0) - shiftChunkOffsets(data, loc.chain[0], base+extStart, delta) - for _, b := range loc.chain { - growBoxSize(data, b, delta) - } - growBoxSize(data, entry, delta) - - out := make([]byte, 0, len(data)-16) - out = append(out, data[:extStart]...) - out = append(out, data[extEnd:]...) - return out -} - -// normalizeQuickTimeAudioToMP4 rewrites a QuickTime-flavored file (FFmpeg mov -// muxer output: ftyp brand "qt " and a version-1 sound sample entry) into a -// standard ISO MP4: an isom/mp42 brand and a plain version-0 AudioSampleEntry. -// Windows Media Foundation (and other strict parsers) reject the QuickTime -// flavor for AC-4 even when dac4 is present. -func normalizeQuickTimeAudioToMP4(data []byte) []byte { - normalizeQuickTimeBrandsInBuf(data) - return normalizeQuickTimeAudioEntry(data, 0) -} - -// EnsureAC4ConfigBox makes a decrypted AC-4 MP4 standards-compliant and -// playable: it normalizes FFmpeg's QuickTime-flavored mov output to an ISO MP4 -// and injects the AC-4 configuration box (dac4) into the ac-4 sample entry. The -// dac4 box is copied verbatim from sourcePath (the original MP4, whose plaintext -// moov still carries it). No-op when the file has no AC-4 track. Only the ftyp -// and moov boxes are held in memory; the audio bulk is streamed. -func EnsureAC4ConfigBox(decryptedPath, sourcePath string) error { - f, err := os.Open(decryptedPath) - if err != nil { - return err - } - info, err := f.Stat() - if err != nil { - f.Close() - return err - } - // A non-MP4 decrypt output (e.g. a raw FLAC stream) is not an AC-4 file; - // bail out before the box parser reports it as a corrupt MP4. A real - // ISO-BMFF box type is four printable ASCII bytes. - var head [8]byte - if _, err := f.ReadAt(head[:], 0); err != nil { - f.Close() - if err == io.EOF || err == io.ErrUnexpectedEOF { - return nil - } - return err - } - for _, c := range head[4:8] { - if c < 0x20 || c > 0x7e { - f.Close() - return nil - } - } - moovBuf, moovOffset, moovFound, err := loadTopLevelMP4Box(f, info.Size(), "moov") - if err != nil || !moovFound { - f.Close() - return err // parse/read failure, or no moov: nothing to do - } - ftypBuf, ftypOffset, ftypFound, err := loadTopLevelMP4Box(f, info.Size(), "ftyp") - f.Close() - if err != nil { - return err - } - moovLen := int64(len(moovBuf)) - - if _, ok := locateAC4Entry(moovBuf); !ok { - return nil // not an AC-4 file; nothing to do - } - - ftypChanged := ftypFound && normalizeQuickTimeBrandsInBuf(ftypBuf) - dst := normalizeQuickTimeAudioEntry(moovBuf, moovOffset) - - loc, ok := locateAC4Entry(dst) - if !ok { - return nil - } - - hdrLen, ok := audioSampleEntryHeaderLen(dst, loc.entry) - if !ok { - return fmt.Errorf("malformed ac-4 sample entry") - } - childStart := loc.entry.body() + hdrLen - if _, has := findChildMP4(dst, childStart, loc.entry.end(), "dac4"); has { - // Already has dac4; still persist any normalization changes. - return writeAC4Sections(decryptedPath, ftypChanged, ftypBuf, ftypOffset, dst, moovOffset, moovLen) - } - - srcF, err := os.Open(sourcePath) - if err != nil { - return err - } - srcInfo, err := srcF.Stat() - if err != nil { - srcF.Close() - return err - } - srcMoovBuf, _, srcFound, err := loadTopLevelMP4Box(srcF, srcInfo.Size(), "moov") - srcF.Close() - if err != nil { - return err - } - if !srcFound { - return fmt.Errorf("source has no moov") - } - srcMoov, ok := readMP4Box(srcMoovBuf, 0) - if !ok { - return fmt.Errorf("source has no moov") - } - dac4Box, ok := findBoxBySignature(srcMoovBuf, srcMoov.body(), srcMoov.end(), "dac4") - if !ok { - return fmt.Errorf("dac4 not found in source") - } - dac4 := append([]byte{}, srcMoovBuf[dac4Box.offset:dac4Box.end()]...) - - insertPos := childStart - delta := int64(len(dac4)) - - shiftChunkOffsets(dst, loc.chain[0], moovOffset+insertPos, delta) - for _, b := range loc.chain { - growBoxSize(dst, b, delta) - } - growBoxSize(dst, loc.entry, delta) - - out := make([]byte, 0, len(dst)+len(dac4)) - out = append(out, dst[:insertPos]...) - out = append(out, dac4...) - out = append(out, dst[insertPos:]...) - - return writeAC4Sections(decryptedPath, ftypChanged, ftypBuf, ftypOffset, out, moovOffset, moovLen) -} - -// writeAC4Sections streams the edited moov (and, when changed, ftyp) back into -// the file. -func writeAC4Sections(path string, ftypChanged bool, ftypBuf []byte, ftypOffset int64, moovBuf []byte, moovOffset, origMoovLen int64) error { - sections := []fileSection{ - {start: moovOffset, end: moovOffset + origMoovLen, data: moovBuf}, - } - if ftypChanged { - sections = append(sections, fileSection{ - start: ftypOffset, - end: ftypOffset + int64(len(ftypBuf)), - data: ftypBuf, - }) - } - return replaceFileSectionsStreaming(path, sections) -} diff --git a/go_backend/ac4_config_test.go b/go_backend/ac4_config_test.go deleted file mode 100644 index 53dacd23..00000000 --- a/go_backend/ac4_config_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/binary" - "os" - "path/filepath" - "testing" -) - -func mp4TestBox(typ string, body []byte) []byte { - out := make([]byte, 8+len(body)) - binary.BigEndian.PutUint32(out[:4], uint32(len(out))) - copy(out[4:8], typ) - copy(out[8:], body) - return out -} - -func mp4TestAC4Tree(entryBody []byte) []byte { - entry := mp4TestBox("ac-4", entryBody) - stsdBody := append([]byte{ - 0, 0, 0, 0, // version/flags - 0, 0, 0, 1, // entry_count - }, entry...) - stsd := mp4TestBox("stsd", stsdBody) - stbl := mp4TestBox("stbl", stsd) - minf := mp4TestBox("minf", stbl) - mdia := mp4TestBox("mdia", minf) - trak := mp4TestBox("trak", mdia) - moov := mp4TestBox("moov", trak) - return moov -} - -func shortAC4SampleEntryBody(version uint16) []byte { - body := make([]byte, 10) - binary.BigEndian.PutUint16(body[8:10], version) - return body -} - -func TestNormalizeQuickTimeAudioToMP4IgnoresTruncatedAC4Entry(t *testing.T) { - input := mp4TestAC4Tree(shortAC4SampleEntryBody(1)) - - defer func() { - if r := recover(); r != nil { - t.Fatalf("normalizeQuickTimeAudioToMP4 panicked: %v", r) - } - }() - - got := normalizeQuickTimeAudioToMP4(append([]byte{}, input...)) - if !bytes.Equal(got, input) { - t.Fatal("truncated QuickTime AC-4 entry should be left unchanged") - } -} - -func TestEnsureAC4ConfigBoxRejectsTruncatedAC4Entry(t *testing.T) { - dir := t.TempDir() - decryptedPath := filepath.Join(dir, "decrypted.mp4") - sourcePath := filepath.Join(dir, "source.mp4") - - if err := os.WriteFile(decryptedPath, mp4TestAC4Tree(shortAC4SampleEntryBody(2)), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(sourcePath, mp4TestBox("moov", mp4TestBox("dac4", []byte{1, 2, 3, 4})), 0o644); err != nil { - t.Fatal(err) - } - - defer func() { - if r := recover(); r != nil { - t.Fatalf("EnsureAC4ConfigBox panicked: %v", r) - } - }() - - if err := EnsureAC4ConfigBox(decryptedPath, sourcePath); err == nil { - t.Fatal("expected malformed AC-4 sample entry error") - } -} diff --git a/go_backend/ac4_metadata.go b/go_backend/ac4_metadata.go deleted file mode 100644 index 7c8093ad..00000000 --- a/go_backend/ac4_metadata.go +++ /dev/null @@ -1,201 +0,0 @@ -package gobackend - -import ( - "encoding/binary" - "encoding/json" - "os" - "strconv" - "strings" -) - -// ac4Metadata mirrors the tag fields the app embeds for other formats. Numeric -// fields are strings because they arrive as a JSON-encoded map of strings. -type ac4Metadata struct { - Title string `json:"title"` - Artist string `json:"artist"` - Album string `json:"album"` - AlbumArtist string `json:"albumArtist"` - Date string `json:"date"` - Genre string `json:"genre"` - Composer string `json:"composer"` - TrackNumber string `json:"trackNumber"` - TotalTracks string `json:"totalTracks"` - DiscNumber string `json:"discNumber"` - TotalDiscs string `json:"totalDiscs"` - ISRC string `json:"isrc"` - Label string `json:"label"` - Copyright string `json:"copyright"` - Lyrics string `json:"lyrics"` -} - -func atoiSafe(s string) int { - n, err := strconv.Atoi(strings.TrimSpace(s)) - if err != nil { - return 0 - } - return n -} - -func itunesTextTag(atomType, value string) []byte { - data := make([]byte, 8+len(value)) - binary.BigEndian.PutUint32(data[0:4], 1) // well-known type 1 = UTF-8 - copy(data[8:], []byte(value)) - return buildM4AAtom(atomType, buildM4AAtom("data", data)) -} - -func itunesNumberPairTag(atomType string, number, total int) []byte { - payload := make([]byte, 8) - binary.BigEndian.PutUint16(payload[2:4], uint16(number)) - binary.BigEndian.PutUint16(payload[4:6], uint16(total)) - data := make([]byte, 8+len(payload)) - binary.BigEndian.PutUint32(data[0:4], 0) // type 0 = implicit/binary - copy(data[8:], payload) - return buildM4AAtom(atomType, buildM4AAtom("data", data)) -} - -func itunesCoverTag(image []byte) []byte { - typeCode := uint32(13) // JPEG - if len(image) >= 8 && - image[0] == 0x89 && image[1] == 0x50 && image[2] == 0x4E && image[3] == 0x47 { - typeCode = 14 // PNG - } - data := make([]byte, 8+len(image)) - binary.BigEndian.PutUint32(data[0:4], typeCode) - copy(data[8:], image) - return buildM4AAtom("covr", buildM4AAtom("data", data)) -} - -func itunesMetadataHandler() []byte { - payload := make([]byte, 0, 25) - payload = append(payload, 0, 0, 0, 0) // version + flags - payload = append(payload, 0, 0, 0, 0) // pre_defined - payload = append(payload, []byte("mdir")...) // handler type - payload = append(payload, []byte("appl")...) // reserved[0] - payload = append(payload, 0, 0, 0, 0, 0, 0, 0, 0) // reserved[1..2] - payload = append(payload, 0) // empty name - return buildM4AAtom("hdlr", payload) -} - -// buildITunesUdta assembles a fresh udta>meta>(hdlr+ilst) box from metadata. -func buildITunesUdta(md ac4Metadata, cover []byte) []byte { - ilst := make([]byte, 0, 256) - add := func(atomType, value string) { - if strings.TrimSpace(value) != "" { - ilst = append(ilst, itunesTextTag(atomType, value)...) - } - } - add("\xa9nam", md.Title) - add("\xa9ART", md.Artist) - add("\xa9alb", md.Album) - add("aART", md.AlbumArtist) - add("\xa9day", md.Date) - add("\xa9gen", md.Genre) - add("\xa9wrt", md.Composer) - if tn := atoiSafe(md.TrackNumber); tn > 0 { - ilst = append(ilst, itunesNumberPairTag("trkn", tn, atoiSafe(md.TotalTracks))...) - } - if dn := atoiSafe(md.DiscNumber); dn > 0 { - ilst = append(ilst, itunesNumberPairTag("disk", dn, atoiSafe(md.TotalDiscs))...) - } - if strings.TrimSpace(md.ISRC) != "" { - ilst = append(ilst, buildM4AFreeformAtom("ISRC", strings.TrimSpace(md.ISRC))...) - } - if strings.TrimSpace(md.Label) != "" { - ilst = append(ilst, buildM4AFreeformAtom("LABEL", strings.TrimSpace(md.Label))...) - } - if strings.TrimSpace(md.Copyright) != "" { - add("cprt", md.Copyright) - } - if strings.TrimSpace(md.Lyrics) != "" { - add("\xa9lyr", md.Lyrics) - } - if len(cover) > 0 { - ilst = append(ilst, itunesCoverTag(cover)...) - } - - ilstBox := buildM4AAtom("ilst", ilst) - metaPayload := append([]byte{0, 0, 0, 0}, itunesMetadataHandler()...) - metaPayload = append(metaPayload, ilstBox...) - meta := buildM4AAtom("meta", metaPayload) - return buildM4AAtom("udta", meta) -} - -// writeMP4iTunesMetadata replaces (or inserts) a udta>meta>ilst metadata box in -// the moov of an MP4 buffer and returns the rewritten bytes. base is the -// buffer's absolute file offset (0 for a whole-file buffer) so stco/co64 -// shifts compare against the absolute positions the entries hold. -func writeMP4iTunesMetadata(data []byte, base int64, md ac4Metadata, cover []byte) []byte { - moov, ok := findChildMP4(data, 0, int64(len(data)), "moov") - if !ok { - return data - } - newUdta := buildITunesUdta(md, cover) - - if udta, ok := findChildMP4(data, moov.body(), moov.end(), "udta"); ok { - delta := int64(len(newUdta)) - udta.size - shiftChunkOffsets(data, moov, base+udta.offset, delta) - growBoxSize(data, moov, delta) - out := make([]byte, 0, len(data)+len(newUdta)) - out = append(out, data[:udta.offset]...) - out = append(out, newUdta...) - out = append(out, data[udta.end():]...) - return out - } - - delta := int64(len(newUdta)) - insertPos := moov.end() - shiftChunkOffsets(data, moov, base+insertPos, delta) - growBoxSize(data, moov, delta) - out := make([]byte, 0, len(data)+len(newUdta)) - out = append(out, data[:insertPos]...) - out = append(out, newUdta...) - out = append(out, data[insertPos:]...) - return out -} - -// WriteAC4MetadataIfApplicable writes iTunes metadata into an AC-4 MP4. Returns -// true when the file was an AC-4 track and metadata was written; false when the -// file is not AC-4 (the caller should fall back to its normal metadata path). -// Only the moov box is held in memory; the audio bulk is streamed. -func WriteAC4MetadataIfApplicable(decryptedPath, metadataJSON, coverPath string) (bool, error) { - f, err := os.Open(decryptedPath) - if err != nil { - return false, err - } - info, err := f.Stat() - if err != nil { - f.Close() - return false, err - } - moovBuf, moovOffset, found, err := loadTopLevelMP4Box(f, info.Size(), "moov") - f.Close() - if err != nil { - return false, err - } - if !found { - return false, nil - } - moovLen := int64(len(moovBuf)) - if _, ok := locateAC4Entry(moovBuf); !ok { - return false, nil - } - - var md ac4Metadata - if strings.TrimSpace(metadataJSON) != "" { - _ = json.Unmarshal([]byte(metadataJSON), &md) - } - var cover []byte - if strings.TrimSpace(coverPath) != "" { - if b, err := os.ReadFile(coverPath); err == nil { - cover = b - } - } - - out := writeMP4iTunesMetadata(moovBuf, moovOffset, md, cover) - if err := replaceFileSectionsStreaming(decryptedPath, []fileSection{ - {start: moovOffset, end: moovOffset + moovLen, data: out}, - }); err != nil { - return false, err - } - return true, nil -} diff --git a/go_backend/ape_tags.go b/go_backend/ape_tags.go deleted file mode 100644 index c96f0b28..00000000 --- a/go_backend/ape_tags.go +++ /dev/null @@ -1,628 +0,0 @@ -package gobackend - -import ( - "encoding/binary" - "fmt" - "io" - "os" - "strings" -) - -// APEv2 tag format constants. -const ( - apeTagPreamble = "APETAGEX" - apeTagHeaderSize = 32 - apeTagVersion2 = 2000 - apeTagFlagHeader = 1 << 29 // bit 29: this is the header, not the footer - apeTagFlagReadOnly = 1 << 0 - // Item flags: bits 1-2 encode content type - // (00: UTF-8 text, 01: binary data, 10: external link) - apeItemFlagBinary = 1 << 1 -) - -// APETagItem represents a single key-value item in an APEv2 tag. -type APETagItem struct { - Key string - Value string - Flags uint32 -} - -// APETag represents a complete APEv2 tag block. -type APETag struct { - Version uint32 - Items []APETagItem - ReadOnly bool -} - -// ReadAPETags reads APEv2 tags from a file. -// APEv2 tags are typically appended at the end of the file. -// The layout is: [audio data] [APEv2 header (optional)] [items...] [APEv2 footer] -// We locate the footer first (last 32 bytes), then read the tag block. -func ReadAPETags(filePath string) (*APETag, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, fmt.Errorf("failed to open file: %w", err) - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return nil, fmt.Errorf("failed to stat file: %w", err) - } - fileSize := fi.Size() - - if fileSize < apeTagHeaderSize { - return nil, fmt.Errorf("file too small for APE tag") - } - - // The footer is the last 32 bytes before any ID3v1 tag (128 bytes). - tag, err := readAPETagAtOffset(f, fileSize, fileSize-apeTagHeaderSize) - if err == nil { - return tag, nil - } - - // Retry: skip ID3v1 tag (128 bytes) if present - if fileSize > apeTagHeaderSize+128 { - tag, err = readAPETagAtOffset(f, fileSize, fileSize-apeTagHeaderSize-128) - if err == nil { - return tag, nil - } - } - - return nil, fmt.Errorf("no APEv2 tag found") -} - -func readAPETagAtOffset(f *os.File, fileSize, footerOffset int64) (*APETag, error) { - if footerOffset < 0 || footerOffset+apeTagHeaderSize > fileSize { - return nil, fmt.Errorf("invalid footer offset") - } - - footer := make([]byte, apeTagHeaderSize) - if _, err := f.ReadAt(footer, footerOffset); err != nil { - return nil, fmt.Errorf("failed to read APE footer: %w", err) - } - - if string(footer[0:8]) != apeTagPreamble { - return nil, fmt.Errorf("APE preamble not found") - } - - version := binary.LittleEndian.Uint32(footer[8:12]) - tagSize := binary.LittleEndian.Uint32(footer[12:16]) // size of items + footer (32 bytes) - itemCount := binary.LittleEndian.Uint32(footer[16:20]) - flags := binary.LittleEndian.Uint32(footer[20:24]) - - if version != apeTagVersion2 && version != 1000 { - return nil, fmt.Errorf("unsupported APE tag version: %d", version) - } - if tagSize < apeTagHeaderSize { - return nil, fmt.Errorf("APE tag size too small: %d", tagSize) - } - if itemCount > 1000 { - return nil, fmt.Errorf("APE tag item count too large: %d", itemCount) - } - - // This should be the footer (bit 29 clear) - isHeader := (flags & apeTagFlagHeader) != 0 - if isHeader { - return nil, fmt.Errorf("expected APE footer but found header") - } - - // tagSize includes items + footer (32 bytes), but NOT the header. - itemsSize := int64(tagSize) - apeTagHeaderSize - if itemsSize < 0 { - return nil, fmt.Errorf("invalid APE tag: items size negative") - } - - itemsOffset := footerOffset - itemsSize - if itemsOffset < 0 { - return nil, fmt.Errorf("APE tag items extend before file start") - } - - itemsData := make([]byte, itemsSize) - if _, err := f.ReadAt(itemsData, itemsOffset); err != nil { - return nil, fmt.Errorf("failed to read APE items: %w", err) - } - - items, err := parseAPEItems(itemsData, int(itemCount)) - if err != nil { - return nil, fmt.Errorf("failed to parse APE items: %w", err) - } - - return &APETag{ - Version: version, - Items: items, - ReadOnly: (flags & apeTagFlagReadOnly) != 0, - }, nil -} - -func parseAPEItems(data []byte, count int) ([]APETagItem, error) { - items := make([]APETagItem, 0, count) - pos := 0 - - for i := 0; i < count && pos < len(data); i++ { - if pos+8 > len(data) { - break - } - - valueSize := int(binary.LittleEndian.Uint32(data[pos : pos+4])) - itemFlags := binary.LittleEndian.Uint32(data[pos+4 : pos+8]) - pos += 8 - - // Key is null-terminated ASCII (2-255 bytes, case-insensitive) - keyEnd := pos - for keyEnd < len(data) && data[keyEnd] != 0 { - keyEnd++ - } - if keyEnd >= len(data) { - break - } - - key := string(data[pos:keyEnd]) - pos = keyEnd + 1 - - if pos+valueSize > len(data) { - break - } - value := string(data[pos : pos+valueSize]) - pos += valueSize - - items = append(items, APETagItem{ - Key: key, - Value: value, - Flags: itemFlags, - }) - } - - return items, nil -} - -// WriteAPETags writes APEv2 tags to the end of a file. -// If the file already has APEv2 tags, they are replaced. -// The tag is written with both header and footer. -func WriteAPETags(filePath string, tag *APETag) error { - existingSize, err := findExistingAPETagSize(filePath) - if err != nil { - return fmt.Errorf("failed to check existing APE tag: %w", err) - } - - tagData, err := marshalAPETag(tag) - if err != nil { - return fmt.Errorf("failed to marshal APE tag: %w", err) - } - - if existingSize > 0 { - fi, err := os.Stat(filePath) - if err != nil { - return fmt.Errorf("failed to stat file: %w", err) - } - newSize := fi.Size() - int64(existingSize) - if err := os.Truncate(filePath, newSize); err != nil { - return fmt.Errorf("failed to truncate existing APE tag: %w", err) - } - } - - f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return fmt.Errorf("failed to open file for writing: %w", err) - } - defer f.Close() - - if _, err := f.Write(tagData); err != nil { - return fmt.Errorf("failed to write APE tag: %w", err) - } - - return nil -} - -// findExistingAPETagSize returns the total size of an existing APE tag -// (header + items + footer) at the end of the file, or 0 if none exists. -func findExistingAPETagSize(filePath string) (int64, error) { - f, err := os.Open(filePath) - if err != nil { - return 0, err - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return 0, err - } - fileSize := fi.Size() - - offsets := []int64{fileSize - apeTagHeaderSize} - if fileSize > apeTagHeaderSize+128 { - offsets = append(offsets, fileSize-apeTagHeaderSize-128) - } - - for _, offset := range offsets { - if offset < 0 { - continue - } - footer := make([]byte, apeTagHeaderSize) - if _, err := f.ReadAt(footer, offset); err != nil { - continue - } - if string(footer[0:8]) != apeTagPreamble { - continue - } - - flags := binary.LittleEndian.Uint32(footer[20:24]) - if (flags & apeTagFlagHeader) != 0 { - continue - } - - tagSize := int64(binary.LittleEndian.Uint32(footer[12:16])) - - hasHeader := (flags & (1 << 31)) != 0 // bit 31 = tag contains header - totalSize := tagSize - if hasHeader { - totalSize += apeTagHeaderSize - } - - // Include any trailing data after the footer (e.g. ID3v1 128-byte tag). - // When truncating, we must remove the APE tag AND everything after it. - trailingBytes := fileSize - (offset + apeTagHeaderSize) - totalSize += trailingBytes - - return totalSize, nil - } - - return 0, nil -} - -// marshalAPETag serializes an APETag into bytes (header + items + footer). -func marshalAPETag(tag *APETag) ([]byte, error) { - if tag == nil || len(tag.Items) == 0 { - return nil, fmt.Errorf("empty APE tag") - } - - var itemsData []byte - for _, item := range tag.Items { - keyBytes := []byte(item.Key) - valueBytes := []byte(item.Value) - - // 4 bytes: value size (LE) - sizeBuf := make([]byte, 4) - binary.LittleEndian.PutUint32(sizeBuf, uint32(len(valueBytes))) - - // 4 bytes: item flags (LE) - flagsBuf := make([]byte, 4) - binary.LittleEndian.PutUint32(flagsBuf, item.Flags) - - itemsData = append(itemsData, sizeBuf...) - itemsData = append(itemsData, flagsBuf...) - itemsData = append(itemsData, keyBytes...) - itemsData = append(itemsData, 0) - itemsData = append(itemsData, valueBytes...) - } - - // tagSize = items data + footer (32 bytes) - tagSize := uint32(len(itemsData) + apeTagHeaderSize) - itemCount := uint32(len(tag.Items)) - - version := uint32(apeTagVersion2) - if tag.Version != 0 { - version = tag.Version - } - - // flags: bit 29 = 1 (is header), bit 31 = 1 (contains header) - headerFlags := uint32(apeTagFlagHeader | (1 << 31)) - header := buildAPEHeaderFooter(version, tagSize, itemCount, headerFlags) - - // flags: bit 29 = 0 (is footer), bit 31 = 1 (contains header) - footerFlags := uint32(1 << 31) - footer := buildAPEHeaderFooter(version, tagSize, itemCount, footerFlags) - - result := make([]byte, 0, len(header)+len(itemsData)+len(footer)) - result = append(result, header...) - result = append(result, itemsData...) - result = append(result, footer...) - - return result, nil -} - -func buildAPEHeaderFooter(version, tagSize, itemCount, flags uint32) []byte { - buf := make([]byte, apeTagHeaderSize) - copy(buf[0:8], apeTagPreamble) - binary.LittleEndian.PutUint32(buf[8:12], version) - binary.LittleEndian.PutUint32(buf[12:16], tagSize) - binary.LittleEndian.PutUint32(buf[16:20], itemCount) - binary.LittleEndian.PutUint32(buf[20:24], flags) - // bytes 24-31 are reserved (zeros) - return buf -} - -// APETagToAudioMetadata converts an APETag to our unified AudioMetadata struct. -func APETagToAudioMetadata(tag *APETag) *AudioMetadata { - if tag == nil { - return nil - } - - metadata := &AudioMetadata{} - for _, item := range tag.Items { - key := strings.ToUpper(strings.TrimSpace(item.Key)) - value := strings.TrimSpace(item.Value) - if value == "" { - continue - } - - switch key { - case "TITLE": - metadata.Title = value - case "ARTIST": - metadata.Artist = value - case "ALBUM": - metadata.Album = value - case "ALBUMARTIST", "ALBUM ARTIST": - metadata.AlbumArtist = value - case "GENRE": - metadata.Genre = value - case "YEAR": - metadata.Year = value - case "DATE": - metadata.Date = value - case "TRACK", "TRACKNUMBER": - metadata.TrackNumber, metadata.TotalTracks = parseIndexPair(value) - case "DISC", "DISCNUMBER": - metadata.DiscNumber, metadata.TotalDiscs = parseIndexPair(value) - case "ISRC": - metadata.ISRC = value - case "LYRICS", "UNSYNCEDLYRICS", "SYNCEDLYRICS": - if metadata.Lyrics == "" { - metadata.Lyrics = value - } - case "LABEL", "PUBLISHER": - metadata.Label = value - case "COPYRIGHT": - metadata.Copyright = value - case "COMPOSER": - metadata.Composer = value - case "COMMENT": - metadata.Comment = value - case "ITUNESADVISORY": - metadata.Explicit = isTruthyTagValue(value) - case "RELEASETYPE": - metadata.AlbumType = value - case "BARCODE", "UPC": - metadata.UPC = value - case "COMPILATION": - if isTruthyTagValue(value) && metadata.AlbumType == "" { - metadata.AlbumType = "compilation" - } - case "REPLAYGAIN_TRACK_GAIN": - metadata.ReplayGainTrackGain = value - case "REPLAYGAIN_TRACK_PEAK": - metadata.ReplayGainTrackPeak = value - case "REPLAYGAIN_ALBUM_GAIN": - metadata.ReplayGainAlbumGain = value - case "REPLAYGAIN_ALBUM_PEAK": - metadata.ReplayGainAlbumPeak = value - } - } - - return metadata -} - -// AudioMetadataToAPEItems converts metadata fields to APE tag items. -func AudioMetadataToAPEItems(metadata *AudioMetadata) []APETagItem { - if metadata == nil { - return nil - } - - var items []APETagItem - addItem := func(key, value string) { - if value != "" { - items = append(items, APETagItem{Key: key, Value: value}) - } - } - - addItem("Title", metadata.Title) - addItem("Artist", metadata.Artist) - addItem("Album", metadata.Album) - addItem("Album Artist", metadata.AlbumArtist) - addItem("Genre", metadata.Genre) - if metadata.Date != "" { - addItem("Year", metadata.Date) - } else if metadata.Year != "" { - addItem("Year", metadata.Year) - } - if metadata.TrackNumber > 0 { - addItem("Track", formatIndexValue(metadata.TrackNumber, metadata.TotalTracks)) - } - if metadata.DiscNumber > 0 { - addItem("Disc", formatIndexValue(metadata.DiscNumber, metadata.TotalDiscs)) - } - addItem("ISRC", metadata.ISRC) - addItem("Lyrics", metadata.Lyrics) - addItem("Label", metadata.Label) - addItem("Copyright", metadata.Copyright) - addItem("Composer", metadata.Composer) - addItem("Comment", metadata.Comment) - if metadata.Explicit { - addItem("ITUNESADVISORY", "1") - } - addItem("RELEASETYPE", metadata.AlbumType) - addItem("BARCODE", metadata.UPC) - if strings.EqualFold(strings.TrimSpace(metadata.AlbumType), "compilation") { - addItem("COMPILATION", "1") - } - addItem("REPLAYGAIN_TRACK_GAIN", metadata.ReplayGainTrackGain) - addItem("REPLAYGAIN_TRACK_PEAK", metadata.ReplayGainTrackPeak) - addItem("REPLAYGAIN_ALBUM_GAIN", metadata.ReplayGainAlbumGain) - addItem("REPLAYGAIN_ALBUM_PEAK", metadata.ReplayGainAlbumPeak) - - return items -} - -// apeKeysFromFields builds a set of upper-case APE tag keys corresponding to -// the metadata fields map sent by the editor. This is used during merge to -// ensure that even empty (cleared) fields override old values. -func apeKeysFromFields(fields map[string]string) map[string]struct{} { - mapping := map[string]string{ - "title": "TITLE", - "artist": "ARTIST", - "album": "ALBUM", - "album_artist": "ALBUM ARTIST", - "date": "DATE", - "genre": "GENRE", - "track_number": "TRACK", - "disc_number": "DISC", - "isrc": "ISRC", - "lyrics": "LYRICS", - "label": "LABEL", - "copyright": "COPYRIGHT", - "composer": "COMPOSER", - "comment": "COMMENT", - "explicit": "ITUNESADVISORY", - "album_type": "RELEASETYPE", - "upc": "BARCODE", - "compilation": "COMPILATION", - "replaygain_track_gain": "REPLAYGAIN_TRACK_GAIN", - "replaygain_track_peak": "REPLAYGAIN_TRACK_PEAK", - "replaygain_album_gain": "REPLAYGAIN_ALBUM_GAIN", - "replaygain_album_peak": "REPLAYGAIN_ALBUM_PEAK", - } - result := make(map[string]struct{}) - for fk, apeKey := range mapping { - if _, present := fields[fk]; present { - result[strings.ToUpper(apeKey)] = struct{}{} - } - } - // Some fields have reader aliases that must also be cleared when the - // canonical key is updated (e.g. DATE writer ↔ DATE/YEAR reader, - // DISC ↔ DISCNUMBER, TRACK ↔ TRACKNUMBER, "ALBUM ARTIST" ↔ ALBUMARTIST, - // LABEL ↔ PUBLISHER, and the supported lyrics aliases). - if _, present := fields["date"]; present { - result["DATE"] = struct{}{} - } - if _, present := fields["disc_number"]; present { - result["DISCNUMBER"] = struct{}{} - } - if _, present := fields["disc_total"]; present { - result["DISCNUMBER"] = struct{}{} - } - if _, present := fields["track_number"]; present { - result["TRACKNUMBER"] = struct{}{} - } - if _, present := fields["track_total"]; present { - result["TRACKNUMBER"] = struct{}{} - } - if _, present := fields["album_artist"]; present { - result["ALBUMARTIST"] = struct{}{} - } - if _, present := fields["label"]; present { - result["PUBLISHER"] = struct{}{} - } - if _, present := fields["lyrics"]; present { - result["UNSYNCEDLYRICS"] = struct{}{} - result["SYNCEDLYRICS"] = struct{}{} - } - return result -} - -// MergeAPEItems overlays newItems on top of existing items. -// For each new item, if a matching key exists (case-insensitive) in existing, -// it is replaced. New keys are appended. Existing items whose keys are NOT -// in newItems are preserved (cover art, ReplayGain, custom tags, etc.). -// -// overrideKeys is an optional set of upper-case keys that should be removed -// from existing even if they do not appear in newItems. This handles field -// deletion: the caller sends an empty value which is not serialized into -// newItems, but the old value must still be dropped. -func MergeAPEItems(existing, newItems []APETagItem, overrideKeys map[string]struct{}) []APETagItem { - combined := make(map[string]struct{}, len(newItems)+len(overrideKeys)) - for k := range overrideKeys { - combined[strings.ToUpper(k)] = struct{}{} - } - for _, item := range newItems { - combined[strings.ToUpper(item.Key)] = struct{}{} - } - - var merged []APETagItem - for _, item := range existing { - if _, overwritten := combined[strings.ToUpper(item.Key)]; !overwritten { - merged = append(merged, item) - } - } - - merged = append(merged, newItems...) - - return merged -} - -// ReadAPETagsFromReader reads APEv2 tags from an io.ReaderAt + size. -// This is useful for reading APE tags from files opened via SAF or other abstractions. -func ReadAPETagsFromReader(r io.ReaderAt, fileSize int64) (*APETag, error) { - if fileSize < apeTagHeaderSize { - return nil, fmt.Errorf("file too small for APE tag") - } - - footer := make([]byte, apeTagHeaderSize) - if _, err := r.ReadAt(footer, fileSize-apeTagHeaderSize); err != nil { - return nil, fmt.Errorf("failed to read APE footer: %w", err) - } - - if string(footer[0:8]) == apeTagPreamble { - tag, err := parseAPETagFromFooter(r, fileSize, fileSize-apeTagHeaderSize, footer) - if err == nil { - return tag, nil - } - } - - // Retry: skip ID3v1 tag (128 bytes) - if fileSize > apeTagHeaderSize+128 { - offset := fileSize - apeTagHeaderSize - 128 - if _, err := r.ReadAt(footer, offset); err == nil { - if string(footer[0:8]) == apeTagPreamble { - tag, err := parseAPETagFromFooter(r, fileSize, offset, footer) - if err == nil { - return tag, nil - } - } - } - } - - return nil, fmt.Errorf("no APEv2 tag found") -} - -func parseAPETagFromFooter(r io.ReaderAt, _, footerOffset int64, footer []byte) (*APETag, error) { - version := binary.LittleEndian.Uint32(footer[8:12]) - tagSize := binary.LittleEndian.Uint32(footer[12:16]) - itemCount := binary.LittleEndian.Uint32(footer[16:20]) - flags := binary.LittleEndian.Uint32(footer[20:24]) - - if version != apeTagVersion2 && version != 1000 { - return nil, fmt.Errorf("unsupported APE tag version: %d", version) - } - if tagSize < apeTagHeaderSize { - return nil, fmt.Errorf("APE tag size too small: %d", tagSize) - } - if itemCount > 1000 { - return nil, fmt.Errorf("APE tag item count too large: %d", itemCount) - } - if (flags & apeTagFlagHeader) != 0 { - return nil, fmt.Errorf("expected footer, found header") - } - - itemsSize := int64(tagSize) - apeTagHeaderSize - itemsOffset := footerOffset - itemsSize - if itemsOffset < 0 { - return nil, fmt.Errorf("APE items extend before file start") - } - - itemsData := make([]byte, itemsSize) - if _, err := r.ReadAt(itemsData, itemsOffset); err != nil { - return nil, fmt.Errorf("failed to read APE items: %w", err) - } - - items, err := parseAPEItems(itemsData, int(itemCount)) - if err != nil { - return nil, fmt.Errorf("failed to parse APE items: %w", err) - } - - return &APETag{ - Version: version, - Items: items, - ReadOnly: (flags & apeTagFlagReadOnly) != 0, - }, nil -} diff --git a/go_backend/ape_tags_supplement_test.go b/go_backend/ape_tags_supplement_test.go deleted file mode 100644 index 74e6bc56..00000000 --- a/go_backend/ape_tags_supplement_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package gobackend - -import ( - "bytes" - "os" - "path/filepath" - "testing" -) - -func TestAPETagReadWriteMergeAndMetadataConversion(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "sample.ape") - if err := os.WriteFile(path, []byte("audio-data"), 0600); err != nil { - t.Fatalf("write sample: %v", err) - } - - metadata := &AudioMetadata{ - Title: "Song", - Artist: "Artist", - Album: "Album", - AlbumArtist: "Album Artist", - Genre: "Pop", - Date: "2026", - TrackNumber: 3, - TotalTracks: 12, - DiscNumber: 1, - TotalDiscs: 2, - ISRC: "USRC17607839", - Lyrics: "lyrics", - Label: "Label", - Copyright: "Copyright", - Composer: "Composer", - Comment: "Comment", - AlbumType: "compilation", - Explicit: true, - UPC: "4006381333931", - ReplayGainTrackGain: "-6.50 dB", - ReplayGainTrackPeak: "0.98", - ReplayGainAlbumGain: "-5.00 dB", - ReplayGainAlbumPeak: "0.99", - } - items := AudioMetadataToAPEItems(metadata) - if len(items) == 0 { - t.Fatal("expected APE items") - } - - tag := &APETag{Items: append(items, APETagItem{Key: "Custom", Value: "Keep"})} - if err := WriteAPETags(path, tag); err != nil { - t.Fatalf("WriteAPETags: %v", err) - } - - readTag, err := ReadAPETags(path) - if err != nil { - t.Fatalf("ReadAPETags: %v", err) - } - if readTag.Version != apeTagVersion2 { - t.Fatalf("version = %d", readTag.Version) - } - readMetadata := APETagToAudioMetadata(readTag) - if readMetadata.Title != "Song" || readMetadata.TrackNumber != 3 || readMetadata.TotalTracks != 12 { - t.Fatalf("metadata = %#v", readMetadata) - } - if !readMetadata.Explicit || readMetadata.AlbumType != "compilation" || readMetadata.UPC != "4006381333931" { - t.Fatalf("release identity = %#v", readMetadata) - } - - readerTag, err := ReadAPETagsFromReader(bytes.NewReader(mustReadFile(t, path)), int64(len(mustReadFile(t, path)))) - if err != nil { - t.Fatalf("ReadAPETagsFromReader: %v", err) - } - if len(readerTag.Items) != len(readTag.Items) { - t.Fatalf("reader items = %d, file items = %d", len(readerTag.Items), len(readTag.Items)) - } - - override := apeKeysFromFields(map[string]string{"title": "", "lyrics": "", "disc_total": ""}) - merged := MergeAPEItems(readTag.Items, []APETagItem{{Key: "Title", Value: "New Song"}}, override) - mergedMeta := APETagToAudioMetadata(&APETag{Items: merged}) - if mergedMeta.Title != "New Song" { - t.Fatalf("merged title = %q", mergedMeta.Title) - } - if mergedMeta.Lyrics != "" { - t.Fatalf("expected lyrics cleared, got %q", mergedMeta.Lyrics) - } - if _, ok := override["SYNCEDLYRICS"]; !ok { - t.Fatal("lyrics edit must also clear the SYNCEDLYRICS alias") - } - if synced := APETagToAudioMetadata(&APETag{Items: []APETagItem{{ - Key: "SYNCEDLYRICS", - Value: "[00:01.00]Synced APE lyrics", - }}}); synced.Lyrics != "[00:01.00]Synced APE lyrics" { - t.Fatalf("APE SYNCEDLYRICS = %q", synced.Lyrics) - } - - if err := WriteAPETags(path, &APETag{Items: []APETagItem{{Key: "Title", Value: "Replacement"}}}); err != nil { - t.Fatalf("replace APE tags: %v", err) - } - replaced, err := ReadAPETags(path) - if err != nil { - t.Fatalf("read replacement: %v", err) - } - if got := APETagToAudioMetadata(replaced).Title; got != "Replacement" { - t.Fatalf("replacement title = %q", got) - } - - if _, err := marshalAPETag(nil); err == nil { - t.Fatal("expected empty tag error") - } - if _, err := ReadAPETags(filepath.Join(dir, "missing.ape")); err == nil { - t.Fatal("expected missing file error") - } - if _, err := ReadAPETagsFromReader(bytes.NewReader([]byte("short")), 5); err == nil { - t.Fatal("expected small reader error") - } -} - -func TestAPETagInvalidFooterBranches(t *testing.T) { - footer := buildAPEHeaderFooter(9999, apeTagHeaderSize, 1, 0) - if _, err := parseAPETagFromFooter(bytes.NewReader(footer), int64(len(footer)), 0, footer); err == nil { - t.Fatal("expected unsupported version") - } - - footer = buildAPEHeaderFooter(apeTagVersion2, apeTagHeaderSize-1, 1, 0) - if _, err := parseAPETagFromFooter(bytes.NewReader(footer), int64(len(footer)), 0, footer); err == nil { - t.Fatal("expected small tag size") - } - - footer = buildAPEHeaderFooter(apeTagVersion2, apeTagHeaderSize, 1001, 0) - if _, err := parseAPETagFromFooter(bytes.NewReader(footer), int64(len(footer)), 0, footer); err == nil { - t.Fatal("expected too many items") - } - - footer = buildAPEHeaderFooter(apeTagVersion2, apeTagHeaderSize, 1, apeTagFlagHeader) - if _, err := parseAPETagFromFooter(bytes.NewReader(footer), int64(len(footer)), 0, footer); err == nil { - t.Fatal("expected header flag error") - } -} diff --git a/go_backend/audio_metadata.go b/go_backend/audio_metadata.go deleted file mode 100644 index 353a537c..00000000 --- a/go_backend/audio_metadata.go +++ /dev/null @@ -1,696 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "os" - "strconv" - "strings" -) - -type AudioMetadata struct { - Title string - Artist string - Album string - AlbumArtist string - Genre string - Year string - Date string - TrackNumber int - TotalTracks int - DiscNumber int - TotalDiscs int - ISRC string - Lyrics string - Label string - Copyright string - Composer string - Comment string - AlbumType string - Explicit bool - UPC string - // ReplayGain fields (text values, e.g. "-6.50 dB", "0.988831") - ReplayGainTrackGain string - ReplayGainTrackPeak string - ReplayGainAlbumGain string - ReplayGainAlbumPeak string -} - -type MP3Quality struct { - SampleRate int - BitDepth int - Duration int - Bitrate int -} - -type OggQuality struct { - SampleRate int - BitDepth int - Duration int - Bitrate int // estimated bitrate in bps -} - -func ReadID3Tags(filePath string) (*AudioMetadata, error) { - metadata, _, _, err := readID3TagsAndCover(filePath, false) - return metadata, err -} - -func readID3TagsAndCover(filePath string, includeCover bool) (*AudioMetadata, []byte, string, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, nil, "", err - } - defer file.Close() - - metadata := &AudioMetadata{} - - id3v2, cover, mime, err := readID3v2WithCover(file, includeCover) - if id3v2 != nil { - metadata = id3v2 - } - - if metadata.Title == "" || metadata.Artist == "" { - id3v1, err := readID3v1(file) - if err == nil && id3v1 != nil { - if metadata.Title == "" { - metadata.Title = id3v1.Title - } - if metadata.Artist == "" { - metadata.Artist = id3v1.Artist - } - if metadata.Album == "" { - metadata.Album = id3v1.Album - } - if metadata.Year == "" { - metadata.Year = id3v1.Year - } - if metadata.Genre == "" { - metadata.Genre = id3v1.Genre - } - } - } - - if metadata.Title == "" && metadata.Artist == "" { - return nil, cover, mime, fmt.Errorf("no ID3 tags found") - } - - return metadata, cover, mime, nil -} - -func parseID3v22Frames(data []byte, metadata *AudioMetadata, tagUnsync bool) { - parseID3Frames(data, metadata, 2, tagUnsync) -} - -func parseID3v23Frames(data []byte, metadata *AudioMetadata, version byte, tagUnsync bool) { - parseID3Frames(data, metadata, version, tagUnsync) -} - -func parseID3Frames(data []byte, metadata *AudioMetadata, version byte, tagUnsync bool) { - _ = walkID3Frames(bytes.NewReader(data), int64(len(data)), version, tagUnsync, nil, - func(id string, payload []byte) { applyID3Frame(metadata, version, id, payload) }) -} - -func applyID3Frame(metadata *AudioMetadata, version byte, frameID string, frameData []byte) { - value := firstTextValue(extractTextFrame(frameData)) - if version == 2 { - switch frameID { - case "TT2": - metadata.Title = value - case "TP1": - metadata.Artist = value - case "TP2": - metadata.AlbumArtist = value - case "TAL": - metadata.Album = value - case "TYE": - metadata.Year = value - case "TCO": - metadata.Genre = cleanGenre(value) - case "TRK": - metadata.TrackNumber, metadata.TotalTracks = parseIndexPair(value) - case "TPA": - metadata.DiscNumber, metadata.TotalDiscs = parseIndexPair(value) - case "TCM": - metadata.Composer = value - case "TPB": - metadata.Label = value - case "TCR": - metadata.Copyright = value - case "ULT": - if v := extractLangTextFrame(frameData); v != "" && metadata.Lyrics == "" { - metadata.Lyrics = v - } - case "TXX": - desc, userValue := extractUserTextFrame(frameData) - if isLyricsDescription(desc) && userValue != "" && metadata.Lyrics == "" { - metadata.Lyrics = userValue - } - switch strings.ToUpper(strings.TrimSpace(desc)) { - case "ITUNESADVISORY": - metadata.Explicit = isTruthyTagValue(userValue) - case "RELEASETYPE": - metadata.AlbumType = userValue - case "BARCODE", "UPC": - metadata.UPC = userValue - } - } - return - } - switch frameID { - case "TIT2": - metadata.Title = value - case "TPE1": - metadata.Artist = value - case "TPE2": - metadata.AlbumArtist = value - case "TALB": - metadata.Album = value - case "TYER", "TDRC": - metadata.Year = value - if len(value) >= 4 { - metadata.Date = value - } - case "TCON": - metadata.Genre = cleanGenre(value) - case "TRCK": - metadata.TrackNumber, metadata.TotalTracks = parseIndexPair(value) - case "TPOS": - metadata.DiscNumber, metadata.TotalDiscs = parseIndexPair(value) - case "TSRC": - metadata.ISRC = value - case "TCOM": - metadata.Composer = value - case "TPUB": - metadata.Label = value - case "TCOP": - metadata.Copyright = value - case "TCMP": - if isTruthyTagValue(value) && metadata.AlbumType == "" { - metadata.AlbumType = "compilation" - } - case "COMM": - if v := extractLangTextFrame(frameData); v != "" { - metadata.Comment = v - } - case "USLT": - if v := extractLangTextFrame(frameData); v != "" && metadata.Lyrics == "" { - metadata.Lyrics = v - } - case "TXXX": - desc, userValue := extractUserTextFrame(frameData) - if isLyricsDescription(desc) && userValue != "" && metadata.Lyrics == "" { - metadata.Lyrics = userValue - } - upperDesc := strings.ToUpper(desc) - switch upperDesc { - case "REPLAYGAIN_TRACK_GAIN": - metadata.ReplayGainTrackGain = userValue - case "REPLAYGAIN_TRACK_PEAK": - metadata.ReplayGainTrackPeak = userValue - case "REPLAYGAIN_ALBUM_GAIN": - metadata.ReplayGainAlbumGain = userValue - case "REPLAYGAIN_ALBUM_PEAK": - metadata.ReplayGainAlbumPeak = userValue - case "ITUNESADVISORY": - metadata.Explicit = isTruthyTagValue(userValue) - case "RELEASETYPE": - metadata.AlbumType = userValue - case "BARCODE", "UPC": - metadata.UPC = userValue - } - } -} - -func readID3v1(file *os.File) (*AudioMetadata, error) { - if _, err := file.Seek(-128, io.SeekEnd); err != nil { - return nil, err - } - - tag := make([]byte, 128) - if _, err := io.ReadFull(file, tag); err != nil { - return nil, err - } - - if string(tag[0:3]) != "TAG" { - return nil, fmt.Errorf("no ID3v1 tag") - } - - metadata := &AudioMetadata{ - Title: strings.TrimRight(string(tag[3:33]), " \x00"), - Artist: strings.TrimRight(string(tag[33:63]), " \x00"), - Album: strings.TrimRight(string(tag[63:93]), " \x00"), - Year: strings.TrimRight(string(tag[93:97]), " \x00"), - } - - if tag[125] == 0 && tag[126] != 0 { - metadata.TrackNumber = int(tag[126]) - } - - genreIndex := int(tag[127]) - if genreIndex < len(id3v1Genres) { - metadata.Genre = id3v1Genres[genreIndex] - } - - return metadata, nil -} - -func extractTextFrame(data []byte) string { - if len(data) == 0 { - return "" - } - - encoding := data[0] - text := data[1:] - - switch encoding { - case 0: // ISO-8859-1 - return strings.TrimRight(string(text), "\x00") - case 1: // UTF-16 with BOM - return decodeUTF16(text) - case 2: // UTF-16BE - return decodeUTF16BE(text) - case 3: // UTF-8 - return strings.TrimRight(string(text), "\x00") - default: - return strings.TrimRight(string(text), "\x00") - } -} - -// extractLangTextFrame decodes ID3 frames with an encoding byte, 3-byte -// language code, and null-terminated descriptor before the text (COMM, USLT). -func extractLangTextFrame(data []byte) string { - if len(data) < 5 { - return "" - } - encoding := data[0] - rest := data[4:] - - var text []byte - switch encoding { - case 1, 2: - for i := 0; i+1 < len(rest); i += 2 { - if rest[i] == 0 && rest[i+1] == 0 { - text = rest[i+2:] - break - } - } - default: - idx := bytes.IndexByte(rest, 0) - if idx >= 0 && idx+1 < len(rest) { - text = rest[idx+1:] - } else { - text = rest - } - } - - if len(text) == 0 { - return "" - } - - framed := make([]byte, 1+len(text)) - framed[0] = encoding - copy(framed[1:], text) - return extractTextFrame(framed) -} - -func extractUserTextFrame(data []byte) (string, string) { - if len(data) < 2 { - return "", "" - } - - encoding := data[0] - payload := data[1:] - - var descRaw, valueRaw []byte - switch encoding { - case 1, 2: - for i := 0; i+1 < len(payload); i += 2 { - if payload[i] == 0 && payload[i+1] == 0 { - descRaw = payload[:i] - valueRaw = payload[i+2:] - break - } - } - default: - idx := bytes.IndexByte(payload, 0) - if idx >= 0 { - descRaw = payload[:idx] - if idx+1 <= len(payload) { - valueRaw = payload[idx+1:] - } - } - } - - if len(valueRaw) == 0 { - return "", "" - } - - descFramed := make([]byte, 1+len(descRaw)) - descFramed[0] = encoding - copy(descFramed[1:], descRaw) - - valueFramed := make([]byte, 1+len(valueRaw)) - valueFramed[0] = encoding - copy(valueFramed[1:], valueRaw) - - return strings.TrimSpace(extractTextFrame(descFramed)), strings.TrimSpace(extractTextFrame(valueFramed)) -} - -func isLyricsDescription(description string) bool { - switch strings.ToLower(strings.TrimSpace(description)) { - case - "lyrics", - "lyric", - "unsyncedlyrics", - "unsynced lyrics", - "syncedlyrics", - "synced lyrics", - "uslt", - "sylt", - "lrc": - return true - default: - return false - } -} - -func decodeUTF16(data []byte) string { - if len(data) < 2 { - return "" - } - - var littleEndian bool - if data[0] == 0xFF && data[1] == 0xFE { - littleEndian = true - data = data[2:] - } else if data[0] == 0xFE && data[1] == 0xFF { - littleEndian = false - data = data[2:] - } - - return decodeUTF16Data(data, littleEndian) -} - -func decodeUTF16BE(data []byte) string { - return decodeUTF16Data(data, false) -} - -func decodeUTF16Data(data []byte, littleEndian bool) string { - if len(data) < 2 { - return "" - } - - var runes []rune - for i := 0; i+1 < len(data); i += 2 { - var r uint16 - if littleEndian { - r = uint16(data[i]) | uint16(data[i+1])<<8 - } else { - r = uint16(data[i])<<8 | uint16(data[i+1]) - } - if r == 0 { - break - } - runes = append(runes, rune(r)) - } - return string(runes) -} - -func cleanGenre(genre string) string { - if len(genre) == 0 { - return "" - } - - if genre[0] == '(' { - end := strings.Index(genre, ")") - if end > 0 { - numStr := genre[1:end] - if num, err := strconv.Atoi(numStr); err == nil && num < len(id3v1Genres) { - if end+1 < len(genre) { - return genre[end+1:] - } - return id3v1Genres[num] - } - } - } - return genre -} - -func parseIndexPair(s string) (int, int) { - s = strings.TrimSpace(s) - if s == "" { - return 0, 0 - } - - first := s - second := "" - if idx := strings.Index(s, "/"); idx > 0 { - first = s[:idx] - second = s[idx+1:] - } - - num, _ := strconv.Atoi(strings.TrimSpace(first)) - total, _ := strconv.Atoi(strings.TrimSpace(second)) - return num, total -} - -func removeUnsync(data []byte) []byte { - if len(data) == 0 { - return data - } - out := make([]byte, 0, len(data)) - for i := 0; i < len(data); i++ { - b := data[i] - out = append(out, b) - if b == 0xFF && i+1 < len(data) && data[i+1] == 0x00 { - i++ - } - } - return out -} - -func extendedHeaderSize(data []byte, version byte) int { - if len(data) < 4 { - return 0 - } - var size int - switch version { - case 3: - size = int(binary.BigEndian.Uint32(data[:4])) - case 4: - size = syncsafeToInt(data[:4]) - default: - return 0 - } - if size <= 0 { - return 0 - } - total := size + 4 - if total <= len(data) { - return total - } - if size <= len(data) { - return size - } - return 0 -} - -func syncsafeToInt(b []byte) int { - if len(b) < 4 { - return 0 - } - return int(b[0])<<21 | int(b[1])<<14 | int(b[2])<<7 | int(b[3]) -} - -func firstTextValue(s string) string { - if idx := strings.IndexByte(s, 0); idx >= 0 { - return s[:idx] - } - return s -} - -func GetMP3Quality(filePath string) (*MP3Quality, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer file.Close() - - quality := &MP3Quality{} - - stat, err := file.Stat() - if err != nil { - return nil, err - } - fileSize := stat.Size() - - header := make([]byte, 10) - if _, err := io.ReadFull(file, header); err != nil { - return nil, err - } - - var audioStart int64 = 0 - if string(header[0:3]) == "ID3" { - tagSize := int64(header[6])<<21 | int64(header[7])<<14 | int64(header[8])<<7 | int64(header[9]) - audioStart = 10 + tagSize - } - - file.Seek(audioStart, io.SeekStart) - - frameHeader := make([]byte, 4) - var frameStart int64 = -1 - for i := 0; i < 10000; i++ { - if _, err := io.ReadFull(file, frameHeader); err != nil { - break - } - - if frameHeader[0] == 0xFF && (frameHeader[1]&0xE0) == 0xE0 { - pos, _ := file.Seek(0, io.SeekCurrent) - frameStart = pos - 4 - break - } - - file.Seek(-3, io.SeekCurrent) - } - - if frameStart < 0 { - return quality, nil - } - - version := (frameHeader[1] >> 3) & 0x03 - layer := (frameHeader[1] >> 1) & 0x03 - bitrateIdx := (frameHeader[2] >> 4) & 0x0F - sampleRateIdx := (frameHeader[2] >> 2) & 0x03 - channelMode := (frameHeader[3] >> 6) & 0x03 - - sampleRates := [][]int{ - {11025, 12000, 8000}, - {0, 0, 0}, - {22050, 24000, 16000}, - {44100, 48000, 32000}, - } - if version < 4 && sampleRateIdx < 3 { - quality.SampleRate = sampleRates[version][sampleRateIdx] - } - - if version == 3 && layer == 1 { - bitrates := []int{0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0} - if bitrateIdx < 16 { - quality.Bitrate = bitrates[bitrateIdx] * 1000 - } - } - if (version == 0 || version == 2) && layer == 1 { - bitrates := []int{0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0} - if bitrateIdx < 16 { - quality.Bitrate = bitrates[bitrateIdx] * 1000 - } - } - - samplesPerFrame := 1152 // MPEG1 Layer III - if version == 0 || version == 2 { - samplesPerFrame = 576 // MPEG2/2.5 Layer III - } - - var xingOffset int - if version == 3 { // MPEG1 - if channelMode == 3 { // Mono - xingOffset = 17 - } else { - xingOffset = 32 - } - } else { // MPEG2/2.5 - if channelMode == 3 { - xingOffset = 9 - } else { - xingOffset = 17 - } - } - - xingBuf := make([]byte, 200) - file.Seek(frameStart+4, io.SeekStart) - n, _ := io.ReadFull(file, xingBuf) - xingBuf = xingBuf[:n] - - vbrFrames := 0 - vbrBytes := int64(0) - isVBR := false - - if xingOffset+8 <= n { - tag := string(xingBuf[xingOffset : xingOffset+4]) - if tag == "Xing" || tag == "Info" { - flags := binary.BigEndian.Uint32(xingBuf[xingOffset+4 : xingOffset+8]) - off := xingOffset + 8 - if flags&0x01 != 0 && off+4 <= n { // Frames flag - vbrFrames = int(binary.BigEndian.Uint32(xingBuf[off : off+4])) - off += 4 - } - if flags&0x02 != 0 && off+4 <= n { // Bytes flag - vbrBytes = int64(binary.BigEndian.Uint32(xingBuf[off : off+4])) - } - if vbrFrames > 0 { - isVBR = true - } - } - } - - if !isVBR && 36+26 <= n { - if string(xingBuf[32:36]) == "VBRI" { - vbrBytes = int64(binary.BigEndian.Uint32(xingBuf[36+6 : 36+10])) - vbrFrames = int(binary.BigEndian.Uint32(xingBuf[36+10 : 36+14])) - if vbrFrames > 0 { - isVBR = true - } - } - } - - if isVBR && vbrFrames > 0 && quality.SampleRate > 0 { - totalSamples := int64(vbrFrames) * int64(samplesPerFrame) - quality.Duration = int(totalSamples / int64(quality.SampleRate)) - - if vbrBytes > 0 && quality.Duration > 0 { - quality.Bitrate = int(vbrBytes * 8 / int64(quality.Duration)) - } else if quality.Duration > 0 { - audioSize := fileSize - audioStart - quality.Bitrate = int(audioSize * 8 / int64(quality.Duration)) - } - } else if quality.Bitrate > 0 { - audioSize := fileSize - audioStart - 128 // subtract possible ID3v1 tag - if audioSize > 0 { - quality.Duration = int(audioSize * 8 / int64(quality.Bitrate)) - } - } - - return quality, nil -} - -var id3v1Genres = []string{ - "Blues", "Classic Rock", "Country", "Dance", "Disco", "Funk", "Grunge", - "Hip-Hop", "Jazz", "Metal", "New Age", "Oldies", "Other", "Pop", "R&B", - "Rap", "Reggae", "Rock", "Techno", "Industrial", "Alternative", "Ska", - "Death Metal", "Pranks", "Soundtrack", "Euro-Techno", "Ambient", - "Trip-Hop", "Vocal", "Jazz+Funk", "Fusion", "Trance", "Classical", - "Instrumental", "Acid", "House", "Game", "Sound Clip", "Gospel", - "Noise", "AlternRock", "Bass", "Soul", "Punk", "Space", "Meditative", - "Instrumental Pop", "Instrumental Rock", "Ethnic", "Gothic", - "Darkwave", "Techno-Industrial", "Electronic", "Pop-Folk", "Eurodance", - "Dream", "Southern Rock", "Comedy", "Cult", "Gangsta", "Top 40", - "Christian Rap", "Pop/Funk", "Jungle", "Native American", "Cabaret", - "New Wave", "Psychedelic", "Rave", "Showtunes", "Trailer", "Lo-Fi", - "Tribal", "Acid Punk", "Acid Jazz", "Polka", "Retro", "Musical", - "Rock & Roll", "Hard Rock", "Folk", "Folk-Rock", "National Folk", - "Swing", "Fast Fusion", "Bebop", "Latin", "Revival", "Celtic", - "Bluegrass", "Avantgarde", "Gothic Rock", "Progressive Rock", - "Psychedelic Rock", "Symphonic Rock", "Slow Rock", "Big Band", - "Chorus", "Easy Listening", "Acoustic", "Humour", "Speech", "Chanson", - "Opera", "Chamber Music", "Sonata", "Symphony", "Booty Bass", "Primus", - "Porn Groove", "Satire", "Slow Jam", "Club", "Tango", "Samba", - "Folklore", "Ballad", "Power Ballad", "Rhythmic Soul", "Freestyle", - "Duet", "Punk Rock", "Drum Solo", "A capella", "Euro-House", - "Dance Hall", "Goa", "Drum & Bass", "Club-House", "Hardcore", - "Terror", "Indie", "BritPop", "Negerpunk", "Polsk Punk", "Beat", - "Christian Gangsta Rap", "Heavy Metal", "Black Metal", "Crossover", - "Contemporary Christian", "Christian Rock", "Merengue", "Salsa", - "Thrash Metal", "Anime", "J-Pop", "Synthpop", -} diff --git a/go_backend/audio_metadata_cache_test.go b/go_backend/audio_metadata_cache_test.go deleted file mode 100644 index 017b31a8..00000000 --- a/go_backend/audio_metadata_cache_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package gobackend - -import ( - "os" - "strings" - "testing" -) - -func TestResolveLibraryCoverCacheKeyUsesExplicitKey(t *testing.T) { - t.Parallel() - - const explicitKey = "content://media/external/audio/media/42|123456" - got := resolveLibraryCoverCacheKey("/tmp/saf_random.flac", explicitKey) - if got != explicitKey { - t.Fatalf("expected explicit cache key %q, got %q", explicitKey, got) - } -} - -func TestResolveLibraryCoverCacheKeyUsesFilePathAndStatWhenNoExplicitKey(t *testing.T) { - t.Parallel() - - tempFile, err := os.CreateTemp("", "cover-cache-*.flac") - if err != nil { - t.Fatalf("CreateTemp failed: %v", err) - } - tempPath := tempFile.Name() - tempFile.Close() - defer os.Remove(tempPath) - - got := resolveLibraryCoverCacheKey(tempPath, "") - if !strings.HasPrefix(got, tempPath+"|") { - t.Fatalf("expected stat-based cache key to start with %q, got %q", tempPath+"|", got) - } -} diff --git a/go_backend/audio_metadata_cover.go b/go_backend/audio_metadata_cover.go deleted file mode 100644 index 3a063241..00000000 --- a/go_backend/audio_metadata_cover.go +++ /dev/null @@ -1,359 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/base64" - "encoding/binary" - "fmt" - "io" - "os" - "path/filepath" - "strings" -) - -func extractMP3CoverArt(filePath string) ([]byte, string, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, "", err - } - defer file.Close() - _, cover, mime, err := readID3v2WithCover(file, true) - if len(cover) > 0 { - return cover, mime, nil - } - if err != nil { - return nil, "", err - } - return nil, "", fmt.Errorf("no cover art found") -} - -func parseAPICFrame(data []byte, version byte) ([]byte, string) { - if len(data) < 4 { - return nil, "" - } - - pos := 0 - encoding := data[pos] - pos++ - - var mimeType string - if version == 2 { - if pos+3 > len(data) { - return nil, "" - } - format := string(data[pos : pos+3]) - pos += 3 - switch format { - case "JPG": - mimeType = "image/jpeg" - case "PNG": - mimeType = "image/png" - default: - mimeType = "image/jpeg" - } - } else { - end := pos - for end < len(data) && data[end] != 0 { - end++ - } - mimeType = string(data[pos:end]) - pos = end + 1 - } - - if pos >= len(data) { - return nil, "" - } - - pos++ - - if encoding == 0 || encoding == 3 { - for pos < len(data) && data[pos] != 0 { - pos++ - } - pos++ - } else { - for pos+1 < len(data) { - if data[pos] == 0 && data[pos+1] == 0 { - pos += 2 - break - } - pos++ - } - } - - if pos >= len(data) { - return nil, "" - } - - return data[pos:], mimeType -} - -func extractOggCoverArt(filePath string) ([]byte, string, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, "", err - } - defer file.Close() - - packets, err := collectOggPackets(file, 30, 80) - if err != nil && len(packets) == 0 { - return nil, "", err - } - - streamType := detectOggStreamType(packets) - for _, pkt := range packets { - var comments []byte - if streamType == oggStreamOpus { - if len(pkt) > 8 && string(pkt[0:8]) == "OpusTags" { - comments = pkt[8:] - } - } else { - if len(pkt) > 7 && pkt[0] == 0x03 && string(pkt[1:7]) == "vorbis" { - comments = pkt[7:] - } - } - if len(comments) == 0 && streamType == oggStreamUnknown { - if len(pkt) > 8 && string(pkt[0:8]) == "OpusTags" { - comments = pkt[8:] - } else if len(pkt) > 7 && pkt[0] == 0x03 && string(pkt[1:7]) == "vorbis" { - comments = pkt[7:] - } - } - - if len(comments) > 0 { - imageData, mimeType := extractPictureFromVorbisComments(comments) - if len(imageData) > 0 { - return imageData, mimeType, nil - } - } - } - - return nil, "", fmt.Errorf("no cover art found") -} - -func extractPictureFromVorbisComments(data []byte) ([]byte, string) { - if len(data) < 8 { - return nil, "" - } - - reader := bytes.NewReader(data) - - var vendorLen uint32 - if err := binary.Read(reader, binary.LittleEndian, &vendorLen); err != nil { - return nil, "" - } - if vendorLen > uint32(len(data)-4) { - return nil, "" - } - reader.Seek(int64(vendorLen), io.SeekCurrent) - - var commentCount uint32 - if err := binary.Read(reader, binary.LittleEndian, &commentCount); err != nil { - return nil, "" - } - - for i := uint32(0); i < commentCount && i < 100; i++ { - var commentLen uint32 - if err := binary.Read(reader, binary.LittleEndian, &commentLen); err != nil { - break - } - if commentLen > 10000000 { - break - } - - comment := make([]byte, commentLen) - if _, err := reader.Read(comment); err != nil { - break - } - - key := "METADATA_BLOCK_PICTURE=" - if len(comment) > len(key) && strings.ToUpper(string(comment[:len(key)])) == key { - cleaned := strings.Map(func(r rune) rune { - switch r { - case '\n', '\r', ' ', '\t': - return -1 - } - return r - }, string(comment[len(key):])) - decoded, err := base64.StdEncoding.DecodeString(cleaned) - if err != nil { - decoded, err = base64.RawStdEncoding.DecodeString(cleaned) - } - if err != nil { - continue - } - - imageData, mimeType := parseFLACPictureBlock(decoded) - if len(imageData) > 0 { - return imageData, mimeType - } - } - } - - return nil, "" -} - -func parseFLACPictureBlock(data []byte) ([]byte, string) { - if len(data) < 32 { - return nil, "" - } - - reader := bytes.NewReader(data) - - var pictureType uint32 - binary.Read(reader, binary.BigEndian, &pictureType) - - var mimeLen uint32 - binary.Read(reader, binary.BigEndian, &mimeLen) - if mimeLen > 256 { - return nil, "" - } - - mimeBytes := make([]byte, mimeLen) - reader.Read(mimeBytes) - mimeType := string(mimeBytes) - - var descLen uint32 - binary.Read(reader, binary.BigEndian, &descLen) - if descLen > 10000 { - return nil, "" - } - - reader.Seek(int64(descLen), io.SeekCurrent) - - reader.Seek(16, io.SeekCurrent) - - var dataLen uint32 - binary.Read(reader, binary.BigEndian, &dataLen) - if dataLen > 10000000 { - return nil, "" - } - - imageData := make([]byte, dataLen) - reader.Read(imageData) - - return imageData, mimeType -} - -func extractAnyCoverArtWithHint(filePath, displayNameHint string) ([]byte, string, error) { - ext := strings.ToLower(filepath.Ext(filePath)) - if ext == "" { - ext = strings.ToLower(filepath.Ext(displayNameHint)) - } - - switch ext { - case ".flac": - data, err := ExtractCoverArt(filePath) - if err != nil { - return nil, "", err - } - mimeType := "image/jpeg" - if len(data) > 8 && string(data[1:4]) == "PNG" { - mimeType = "image/png" - } - return data, mimeType, nil - - case ".mp3": - return extractMP3CoverArt(filePath) - - case ".opus", ".ogg": - return extractOggCoverArt(filePath) - - case ".m4a": - data, err := extractCoverFromM4A(filePath) - if err != nil { - return nil, "", err - } - mimeType := "image/jpeg" - if len(data) >= 8 && - data[0] == 0x89 && - data[1] == 0x50 && - data[2] == 0x4E && - data[3] == 0x47 { - mimeType = "image/png" - } - return data, mimeType, nil - - case ".wav", ".aiff", ".aif", ".aifc": - return extractWAVAIFFCover(filePath) - - default: - return nil, "", fmt.Errorf("unsupported format: %s", ext) - } -} - -func resolveLibraryCoverCacheKey(filePath, explicitKey string) string { - explicitKey = strings.TrimSpace(explicitKey) - if explicitKey != "" { - return explicitKey - } - - cacheKey := filePath - if stat, err := os.Stat(filePath); err == nil { - cacheKey = fmt.Sprintf("%s|%d|%d", filePath, stat.Size(), stat.ModTime().UnixNano()) - } - return cacheKey -} - -func libraryCoverCachePaths(cacheDir, cacheKey string) (string, string) { - hash := hashString(cacheKey) - jpgPath := filepath.Join(cacheDir, fmt.Sprintf("cover_%x.jpg", hash)) - pngPath := filepath.Join(cacheDir, fmt.Sprintf("cover_%x.png", hash)) - return jpgPath, pngPath -} - -func existingLibraryCoverCachePath(cacheDir, cacheKey string) string { - jpgPath, pngPath := libraryCoverCachePaths(cacheDir, cacheKey) - - if _, err := os.Stat(jpgPath); err == nil { - return jpgPath - } - if _, err := os.Stat(pngPath); err == nil { - return pngPath - } - return "" -} - -func saveLibraryCoverDataToCache(cacheDir, cacheKey string, imageData []byte, mimeType string) (string, error) { - if existing := existingLibraryCoverCachePath(cacheDir, cacheKey); existing != "" { - return existing, nil - } - if len(imageData) == 0 { - return "", fmt.Errorf("cover data is empty") - } - // Cap cached Library artwork while preserving unsupported formats. - if resized, changed, err := resizeCoverForEmbedding(imageData, libraryCoverMaxDimension); err == nil { - if changed { - imageData = resized - mimeType = embeddedCoverMIME(imageData) - } - } else { - GoLog("[LibraryScan] Cover resize skipped for %s: %v\n", cacheKey, err) - } - if err := os.MkdirAll(cacheDir, 0755); err != nil { - return "", fmt.Errorf("failed to create cache dir: %w", err) - } - - jpgPath, pngPath := libraryCoverCachePaths(cacheDir, cacheKey) - cachePath := jpgPath - if strings.Contains(mimeType, "png") { - cachePath = pngPath - } - if err := os.WriteFile(cachePath, imageData, 0644); err != nil { - return "", fmt.Errorf("failed to write cover: %w", err) - } - return cachePath, nil -} - -func SaveCoverToCacheWithHintAndKey(filePath, displayNameHint, cacheDir, coverCacheKey string) (string, error) { - cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) - if existing := existingLibraryCoverCachePath(cacheDir, cacheKey); existing != "" { - return existing, nil - } - - imageData, mimeType, err := extractAnyCoverArtWithHint(filePath, displayNameHint) - if err != nil { - return "", err - } - return saveLibraryCoverDataToCache(cacheDir, cacheKey, imageData, mimeType) -} diff --git a/go_backend/audio_metadata_mp3_test.go b/go_backend/audio_metadata_mp3_test.go deleted file mode 100644 index b63f7b28..00000000 --- a/go_backend/audio_metadata_mp3_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package gobackend - -import ( - "os" - "os/exec" - "path/filepath" - "strings" - "testing" -) - -func ffmpegCommand(args ...string) *exec.Cmd { - if ffmpegPath, err := exec.LookPath("ffmpeg"); err == nil { - return exec.Command(ffmpegPath, args...) - } - return exec.Command("ffmpeg", args...) -} - -func runFFmpegTestCommand(t *testing.T, args ...string) { - t.Helper() - cmd := ffmpegCommand(args...) - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("ffmpeg failed: %v\n%s", err, string(output)) - } -} - -func TestExtractLyricsReadsMp3AfterCoverEmbed(t *testing.T) { - if _, err := exec.LookPath("ffmpeg"); err != nil { - t.Skip("ffmpeg not available") - } - - tempDir := t.TempDir() - sourceFlac := filepath.Join(tempDir, "source.flac") - baseMp3 := filepath.Join(tempDir, "base.mp3") - finalMp3 := filepath.Join(tempDir, "final.mp3") - coverPath := filepath.Join(tempDir, "cover.jpg") - lyrics := "[ti:Test Song]\n[ar:Test Artist]\n[00:00.00]Hello from embedded lyrics" - - runFFmpegTestCommand( - t, - "-y", - "-f", - "lavfi", - "-i", - "sine=frequency=440:duration=1", - "-c:a", - "flac", - sourceFlac, - ) - - runFFmpegTestCommand( - t, - "-y", - "-f", - "lavfi", - "-i", - "color=c=red:s=32x32:d=1", - "-frames:v", - "1", - coverPath, - ) - - runFFmpegTestCommand( - t, - "-y", - "-i", - sourceFlac, - "-b:a", - "320k", - "-metadata", - "title=Test Song", - "-metadata", - "artist=Test Artist", - "-metadata", - "lyrics="+lyrics, - baseMp3, - ) - - runFFmpegTestCommand( - t, - "-y", - "-i", - baseMp3, - "-i", - coverPath, - "-map", - "0:a", - "-map_metadata", - "-1", - "-map", - "1:0", - "-c:v:0", - "copy", - "-id3v2_version", - "3", - "-metadata", - "title=Test Song", - "-metadata", - "artist=Test Artist", - "-metadata", - "lyrics="+lyrics, - "-metadata:s:v", - "title=Album cover", - "-metadata:s:v", - "comment=Cover (front)", - "-c:a", - "copy", - finalMp3, - ) - - meta, err := ReadID3Tags(finalMp3) - if err != nil { - t.Fatalf("ReadID3Tags failed: %v", err) - } - if meta == nil { - t.Fatalf("ReadID3Tags returned nil metadata") - } - - embeddedLyrics, err := ExtractLyrics(finalMp3) - if err != nil { - t.Fatalf("ExtractLyrics failed: %v (metadata=%+v)", err, meta) - } - if !strings.Contains(embeddedLyrics, "Hello from embedded lyrics") { - t.Fatalf("embedded lyrics missing, got %q (metadata=%+v)", embeddedLyrics, meta) - } - if !strings.Contains(meta.Lyrics, "Hello from embedded lyrics") { - t.Fatalf("ReadID3Tags lyrics missing, got %+v", meta) - } - - if _, err := os.Stat(finalMp3); err != nil { - t.Fatalf("expected final mp3 to exist: %v", err) - } -} diff --git a/go_backend/audio_metadata_ogg.go b/go_backend/audio_metadata_ogg.go deleted file mode 100644 index 2c8f701c..00000000 --- a/go_backend/audio_metadata_ogg.go +++ /dev/null @@ -1,457 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "math" - "os" - "strconv" - "strings" -) - -func ReadOggVorbisComments(filePath string) (*AudioMetadata, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer file.Close() - - metadata := &AudioMetadata{} - - packets, err := collectOggPackets(file, 30, 80) - if err != nil && len(packets) == 0 { - return nil, err - } - - streamType := detectOggStreamType(packets) - for _, pkt := range packets { - if streamType == oggStreamOpus { - if len(pkt) > 8 && string(pkt[0:8]) == "OpusTags" { - parseVorbisComments(pkt[8:], metadata) - break - } - continue - } - if streamType == oggStreamVorbis || streamType == oggStreamUnknown { - if len(pkt) > 7 && pkt[0] == 0x03 && string(pkt[1:7]) == "vorbis" { - parseVorbisComments(pkt[7:], metadata) - break - } - } - if streamType == oggStreamUnknown { - if len(pkt) > 8 && string(pkt[0:8]) == "OpusTags" { - parseVorbisComments(pkt[8:], metadata) - break - } - } - } - - if metadata.Title == "" && metadata.Artist == "" && - metadata.ReplayGainTrackGain == "" && metadata.ReplayGainAlbumGain == "" { - return nil, fmt.Errorf("no Vorbis comments found") - } - - return metadata, nil -} - -type oggPage struct { - headerType byte - segmentTable []byte - data []byte -} - -func readOggPageWithHeader(file *os.File) (*oggPage, error) { - header := make([]byte, 27) - if _, err := io.ReadFull(file, header); err != nil { - return nil, err - } - - if string(header[0:4]) != "OggS" { - return nil, fmt.Errorf("not an Ogg page") - } - - headerType := header[5] - numSegments := int(header[26]) - - segmentTable := make([]byte, numSegments) - if _, err := io.ReadFull(file, segmentTable); err != nil { - return nil, err - } - - var pageSize int - for _, seg := range segmentTable { - pageSize += int(seg) - } - - pageData := make([]byte, pageSize) - if _, err := io.ReadFull(file, pageData); err != nil { - return nil, err - } - - return &oggPage{ - headerType: headerType, - segmentTable: segmentTable, - data: pageData, - }, nil -} - -func collectOggPackets(file *os.File, maxPackets, maxPages int) ([][]byte, error) { - const maxPacketSize = 10 * 1024 * 1024 - var packets [][]byte - var cur []byte - skipPacket := false - - for pageNum := 0; pageNum < maxPages && len(packets) < maxPackets; pageNum++ { - page, err := readOggPageWithHeader(file) - if err != nil { - if len(packets) > 0 { - return packets, nil - } - return nil, err - } - - if page.headerType&0x01 == 0 && len(cur) > 0 { - cur = nil - skipPacket = false - } - - offset := 0 - for _, seg := range page.segmentTable { - segLen := int(seg) - if offset+segLen > len(page.data) { - return packets, fmt.Errorf("invalid ogg segment size") - } - - if skipPacket { - offset += segLen - if segLen < 255 { - skipPacket = false - } - continue - } - - if len(cur)+segLen > maxPacketSize { - cur = nil - skipPacket = true - offset += segLen - if segLen < 255 { - skipPacket = false - } - continue - } - - cur = append(cur, page.data[offset:offset+segLen]...) - offset += segLen - - if segLen < 255 { - if len(cur) > 0 { - packets = append(packets, cur) - } - cur = nil - if len(packets) >= maxPackets { - return packets, nil - } - } - } - } - - return packets, nil -} - -type oggStreamType int - -const ( - oggStreamUnknown oggStreamType = iota - oggStreamOpus - oggStreamVorbis -) - -func detectOggStreamType(packets [][]byte) oggStreamType { - for _, p := range packets { - if len(p) >= 8 && string(p[0:8]) == "OpusHead" { - return oggStreamOpus - } - if len(p) > 7 && p[0] == 0x01 && string(p[1:7]) == "vorbis" { - return oggStreamVorbis - } - } - return oggStreamUnknown -} - -func parseVorbisComments(data []byte, metadata *AudioMetadata) { - if len(data) < 4 { - return - } - - reader := bytes.NewReader(data) - artistValues := make([]string, 0, 1) - albumArtistValues := make([]string, 0, 1) - - var vendorLen uint32 - if err := binary.Read(reader, binary.LittleEndian, &vendorLen); err != nil { - return - } - - if vendorLen > uint32(len(data)-4) { - return - } - vendor := make([]byte, vendorLen) - if _, err := reader.Read(vendor); err != nil { - return - } - - var commentCount uint32 - if err := binary.Read(reader, binary.LittleEndian, &commentCount); err != nil { - return - } - - for i := uint32(0); i < commentCount && i < 100; i++ { - var commentLen uint32 - if err := binary.Read(reader, binary.LittleEndian, &commentLen); err != nil { - break - } - - remaining := uint32(reader.Len()) - if commentLen > remaining { - break - } - if commentLen > 512*1024 { - reader.Seek(int64(commentLen), io.SeekCurrent) - continue - } - - comment := make([]byte, commentLen) - if _, err := reader.Read(comment); err != nil { - break - } - - parts := strings.SplitN(string(comment), "=", 2) - if len(parts) != 2 { - continue - } - - key := strings.ToUpper(parts[0]) - value := parts[1] - - switch key { - case "TITLE": - metadata.Title = value - case "ARTIST": - artistValues = append(artistValues, value) - case "ALBUMARTIST", "ALBUM_ARTIST", "ALBUM ARTIST": - albumArtistValues = append(albumArtistValues, value) - case "ALBUM": - metadata.Album = value - case "DATE", "YEAR": - metadata.Date = value - if len(value) >= 4 { - metadata.Year = value[:4] - } - case "GENRE": - metadata.Genre = value - case "TRACKNUMBER", "TRACK": - metadata.TrackNumber, metadata.TotalTracks = parseIndexPair(value) - case "DISCNUMBER", "DISC": - metadata.DiscNumber, metadata.TotalDiscs = parseIndexPair(value) - case "ISRC": - metadata.ISRC = value - case "COMPOSER": - metadata.Composer = value - case "COMMENT", "DESCRIPTION": - metadata.Comment = value - case "LYRICS", "UNSYNCEDLYRICS", "SYNCEDLYRICS": - if metadata.Lyrics == "" { - metadata.Lyrics = value - } - case "ORGANIZATION", "LABEL", "PUBLISHER": - metadata.Label = value - case "COPYRIGHT": - metadata.Copyright = value - case "ITUNESADVISORY": - metadata.Explicit = isTruthyTagValue(value) - case "RELEASETYPE": - metadata.AlbumType = value - case "BARCODE", "UPC": - metadata.UPC = value - case "COMPILATION": - if isTruthyTagValue(value) && metadata.AlbumType == "" { - metadata.AlbumType = "compilation" - } - case "REPLAYGAIN_TRACK_GAIN": - metadata.ReplayGainTrackGain = value - case "REPLAYGAIN_TRACK_PEAK": - metadata.ReplayGainTrackPeak = value - case "REPLAYGAIN_ALBUM_GAIN": - metadata.ReplayGainAlbumGain = value - case "REPLAYGAIN_ALBUM_PEAK": - metadata.ReplayGainAlbumPeak = value - // Opus gain tags (RFC 7845): Q7.8 fixed point on the R128 -23 LUFS - // reference. Exposed as ReplayGain 2 dB (-18 LUFS reference) so - // consumers see one representation; explicit REPLAYGAIN_* wins. - case "R128_TRACK_GAIN": - if metadata.ReplayGainTrackGain == "" { - if db, ok := r128ToReplayGainDb(value); ok { - metadata.ReplayGainTrackGain = db - } - } - case "R128_ALBUM_GAIN": - if metadata.ReplayGainAlbumGain == "" { - if db, ok := r128ToReplayGainDb(value); ok { - metadata.ReplayGainAlbumGain = db - } - } - } - } - - if len(artistValues) > 0 { - metadata.Artist = joinVorbisCommentValues(artistValues) - } - if len(albumArtistValues) > 0 { - metadata.AlbumArtist = joinVorbisCommentValues(albumArtistValues) - } -} - -// r128ToReplayGainDb converts an R128_*_GAIN value (integer, 1/256 dB steps, -// -23 LUFS reference) to a ReplayGain 2 dB string (-18 LUFS reference): -// rg = q/256 + 5. Inverse of applyOpusReplayGainEdits. -func r128ToReplayGainDb(raw string) (string, bool) { - q, err := strconv.ParseInt(strings.TrimSpace(raw), 10, 16) - if err != nil { - return "", false - } - return fmt.Sprintf("%.2f dB", float64(q)/256.0+5.0), true -} - -func GetOggQuality(filePath string) (*OggQuality, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer file.Close() - - quality := &OggQuality{} - - packets, err := collectOggPackets(file, 5, 10) - if err != nil && len(packets) == 0 { - return nil, err - } - - streamType := detectOggStreamType(packets) - if streamType == oggStreamUnknown { - if strings.HasSuffix(strings.ToLower(filePath), ".opus") { - streamType = oggStreamOpus - } else { - streamType = oggStreamVorbis - } - } - - isOpus := streamType == oggStreamOpus - var preSkip int - - if isOpus { - for _, pkt := range packets { - if len(pkt) >= 19 && string(pkt[0:8]) == "OpusHead" { - quality.SampleRate = int(binary.LittleEndian.Uint32(pkt[12:16])) - if quality.SampleRate == 0 { - quality.SampleRate = 48000 - } - preSkip = int(binary.LittleEndian.Uint16(pkt[10:12])) - break - } - } - } else { - for _, pkt := range packets { - if len(pkt) > 29 && pkt[0] == 0x01 && string(pkt[1:7]) == "vorbis" { - quality.SampleRate = int(binary.LittleEndian.Uint32(pkt[12:16])) - break - } - } - } - - stat, err := file.Stat() - if err != nil { - return quality, nil - } - fileSize := stat.Size() - - granule := readLastOggGranulePosition(file, fileSize) - if granule > 0 { - if isOpus { - totalSamples := granule - int64(preSkip) - if totalSamples > 0 { - durationSec := float64(totalSamples) / 48000.0 - if durationSec > 0 { - quality.Duration = int(math.Round(durationSec)) - quality.Bitrate = int(float64(fileSize*8) / durationSec) - } - } - } else if quality.SampleRate > 0 { - durationSec := float64(granule) / float64(quality.SampleRate) - if durationSec > 0 { - quality.Duration = int(math.Round(durationSec)) - quality.Bitrate = int(float64(fileSize*8) / durationSec) - } - } - } - - if quality.Bitrate <= 0 && quality.Duration > 0 { - quality.Bitrate = int(fileSize * 8 / int64(quality.Duration)) - } - if quality.Duration > 24*60*60 { - quality.Duration = 0 - quality.Bitrate = 0 - } - if quality.Bitrate > 0 && quality.Bitrate < 8000 { - quality.Bitrate = 0 - } - - return quality, nil -} - -func readLastOggGranulePosition(file *os.File, fileSize int64) int64 { - searchSize := int64(65536) - if searchSize > fileSize { - searchSize = fileSize - } - - buf := make([]byte, searchSize) - offset := fileSize - searchSize - if offset < 0 { - offset = 0 - } - n, err := file.ReadAt(buf, offset) - if err != nil && n == 0 { - return 0 - } - buf = buf[:n] - - for i := n - 4; i >= 0; i-- { - if buf[i] != 'O' || buf[i+1] != 'g' || buf[i+2] != 'g' || buf[i+3] != 'S' { - continue - } - if i+27 > n { - continue - } - version := buf[i+4] - headerType := buf[i+5] - if version != 0 || headerType > 0x07 { - continue - } - segmentCount := int(buf[i+26]) - headerLen := 27 + segmentCount - if i+headerLen > n { - continue - } - payloadLen := 0 - for s := 0; s < segmentCount; s++ { - payloadLen += int(buf[i+27+s]) - } - if i+headerLen+payloadLen > n { - continue - } - return int64(binary.LittleEndian.Uint64(buf[i+6 : i+14])) - } - return 0 -} diff --git a/go_backend/audio_metadata_supplement_test.go b/go_backend/audio_metadata_supplement_test.go deleted file mode 100644 index 27723644..00000000 --- a/go_backend/audio_metadata_supplement_test.go +++ /dev/null @@ -1,548 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/base64" - "encoding/binary" - "os" - "path/filepath" - "strings" - "testing" -) - -func TestAudioMetadataID3ParsingBranches(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "tagged.mp3") - tag := buildID3v23Tag( - id3TextFrame("TIT2", "Title"), - id3TextFrame("TPE1", "Artist"), - id3TextFrame("TPE2", "Album Artist"), - id3TextFrame("TALB", "Album"), - id3TextFrame("TDRC", "2026-05-04"), - id3TextFrame("TCON", "(13)Pop"), - id3TextFrame("TRCK", "4/12"), - id3TextFrame("TPOS", "1/2"), - id3TextFrame("TSRC", "USRC17607839"), - id3TextFrame("TCOM", "Composer"), - id3TextFrame("TPUB", "Label"), - id3TextFrame("TCOP", "Copyright"), - id3CommentFrame("COMM", "Comment"), - id3CommentFrame("USLT", "Lyrics"), - id3UserTextFrame("TXXX", "REPLAYGAIN_TRACK_GAIN", "-6.50 dB"), - id3UserTextFrame("TXXX", "REPLAYGAIN_TRACK_PEAK", "0.98"), - id3UserTextFrame("TXXX", "ITUNESADVISORY", "1"), - id3UserTextFrame("TXXX", "RELEASETYPE", "album"), - id3UserTextFrame("TXXX", "BARCODE", "4006381333931"), - ) - if err := os.WriteFile(path, append(tag, []byte("audio")...), 0600); err != nil { - t.Fatalf("write ID3v2: %v", err) - } - - meta, err := ReadID3Tags(path) - if err != nil { - t.Fatalf("ReadID3Tags: %v", err) - } - if meta.Title != "Title" || meta.TrackNumber != 4 || meta.TotalTracks != 12 || meta.Genre != "Pop" { - t.Fatalf("metadata = %#v", meta) - } - if meta.Comment != "Comment" || meta.Lyrics != "Lyrics" || meta.ReplayGainTrackGain == "" { - t.Fatalf("metadata comments/lyrics/replaygain = %#v", meta) - } - if !meta.Explicit || meta.AlbumType != "album" || meta.UPC != "4006381333931" { - t.Fatalf("metadata release identity = %#v", meta) - } - - id3v1Path := filepath.Join(dir, "id3v1.mp3") - if err := os.WriteFile(id3v1Path, append([]byte("audio"), buildID3v1Tag("V1 Title", "V1 Artist", "V1 Album", "1999", 7, 13)...), 0600); err != nil { - t.Fatalf("write ID3v1: %v", err) - } - v1, err := ReadID3Tags(id3v1Path) - if err != nil { - t.Fatalf("ReadID3Tags v1: %v", err) - } - if v1.Title != "V1 Title" || v1.Artist != "V1 Artist" || v1.Genre == "" { - t.Fatalf("v1 = %#v", v1) - } - - v22Path := filepath.Join(dir, "id3v22.mp3") - v22 := buildID3v22Tag( - id3v22TextFrame("TT2", "V22 Title"), - id3v22TextFrame("TP1", "V22 Artist"), - id3v22TextFrame("TRK", "2/5"), - id3v22CommentFrame("ULT", "V22 Lyrics"), - ) - if err := os.WriteFile(v22Path, append(v22, []byte("audio")...), 0600); err != nil { - t.Fatalf("write ID3v2.2: %v", err) - } - v22Meta, err := ReadID3Tags(v22Path) - if err != nil { - t.Fatalf("ReadID3Tags v2.2: %v", err) - } - if v22Meta.Title != "V22 Title" || v22Meta.Artist != "V22 Artist" || v22Meta.Lyrics != "V22 Lyrics" { - t.Fatalf("v22 = %#v", v22Meta) - } - - if got := decodeUTF16([]byte{0xff, 0xfe, 'H', 0, 'i', 0}); got != "Hi" { - t.Fatalf("decodeUTF16 = %q", got) - } - if got := decodeUTF16BE([]byte{0, 'O', 0, 'K'}); got != "OK" { - t.Fatalf("decodeUTF16BE = %q", got) - } - if n, total := parseIndexPair(" 8 / 10 "); n != 8 || total != 10 { - t.Fatalf("parseIndexPair = %d/%d", n, total) - } - if got := removeUnsync([]byte{0xff, 0x00, 0xe0}); !bytes.Equal(got, []byte{0xff, 0xe0}) { - t.Fatalf("removeUnsync = %#v", got) - } - if got := extendedHeaderSize([]byte{0, 0, 0, 6, 0, 0, 0, 0, 0, 0}, 3); got != 10 { - t.Fatalf("extendedHeaderSize = %d", got) - } - if got := syncsafeToInt([]byte{0, 0, 2, 0}); got != 256 { - t.Fatalf("syncsafe = %d", got) - } - if !isLyricsDescription("SYNCEDLYRICS") || !isLyricsDescription("SYLT") { - t.Fatal("synced lyrics descriptions must be recognized") - } -} - -func TestAudioMetadataCoverAndQualityHelpers(t *testing.T) { - png := []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0, 0, 0, 0} - if detectCoverMIME("cover.jpg", png) != "image/png" || detectCoverMIME("cover.webp", []byte("RIFFxxxxWEBPdata")) != "image/webp" { - t.Fatal("cover MIME detection mismatch") - } - if _, err := buildPictureBlock("", nil); err == nil { - t.Fatal("expected empty picture block error") - } - - apic := append([]byte{3}, []byte("image/png\x00")...) - apic = append(apic, 3, 0) - apic = append(apic, png...) - image, mime := parseAPICFrame(apic, 3) - if mime != "image/png" || !bytes.Equal(image, png) { - t.Fatalf("APIC = %s/%v", mime, image) - } - pic := append([]byte{0}, []byte("PNG")...) - pic = append(pic, 3, 0) - pic = append(pic, png...) - image, mime = parseAPICFrame(pic, 2) - if mime != "image/png" || !bytes.Equal(image, png) { - t.Fatalf("PIC = %s/%v", mime, image) - } - - frame := make([]byte, 10) - copy(frame[:4], "APIC") - binary.BigEndian.PutUint32(frame[4:8], uint32(len(apic))) - tag := append(frame, apic...) - header := []byte{'I', 'D', '3', 3, 0, 0, byte(len(tag) >> 21), byte(len(tag) >> 14), byte(len(tag) >> 7), byte(len(tag))} - mp3CoverPath := filepath.Join(t.TempDir(), "cover.mp3") - if err := os.WriteFile(mp3CoverPath, append(append(header, tag...), []byte("audio")...), 0600); err != nil { - t.Fatal(err) - } - extracted, extractedMIME, err := extractMP3CoverArt(mp3CoverPath) - if err != nil || extractedMIME != "image/png" || !bytes.Equal(extracted, png) { - t.Fatalf("extractMP3CoverArt = %s/%v/%v", extractedMIME, extracted, err) - } - - var picture bytes.Buffer - binary.Write(&picture, binary.BigEndian, uint32(3)) - binary.Write(&picture, binary.BigEndian, uint32(len("image/png"))) - picture.WriteString("image/png") - binary.Write(&picture, binary.BigEndian, uint32(0)) - binary.Write(&picture, binary.BigEndian, uint32(1)) - binary.Write(&picture, binary.BigEndian, uint32(1)) - binary.Write(&picture, binary.BigEndian, uint32(32)) - binary.Write(&picture, binary.BigEndian, uint32(0)) - binary.Write(&picture, binary.BigEndian, uint32(len(png))) - picture.Write(png) - flacImage, flacMIME := parseFLACPictureBlock(picture.Bytes()) - if flacMIME != "image/png" || !bytes.Equal(flacImage, png) { - t.Fatalf("FLAC picture = %s/%v", flacMIME, flacImage) - } - - comment := "METADATA_BLOCK_PICTURE=" + base64.StdEncoding.EncodeToString(picture.Bytes()) - var vorbis bytes.Buffer - binary.Write(&vorbis, binary.LittleEndian, uint32(6)) - vorbis.WriteString("vendor") - binary.Write(&vorbis, binary.LittleEndian, uint32(1)) - binary.Write(&vorbis, binary.LittleEndian, uint32(len(comment))) - vorbis.WriteString(comment) - commentImage, commentMIME := extractPictureFromVorbisComments(vorbis.Bytes()) - if commentMIME != "image/png" || !bytes.Equal(commentImage, png) { - t.Fatalf("vorbis picture = %s/%v", commentMIME, commentImage) - } - if detectOggStreamType([][]byte{[]byte("OpusHeadxxxx")}) != oggStreamOpus { - t.Fatal("expected opus stream") - } - if detectOggStreamType([][]byte{append([]byte{1}, []byte("vorbisxxxx")...)}) != oggStreamVorbis { - t.Fatal("expected vorbis stream") - } - - mp3Path := filepath.Join(t.TempDir(), "quality.mp3") - audio := append([]byte{0xFF, 0xFB, 0x90, 0x64}, bytes.Repeat([]byte{0}, 2000)...) - if err := os.WriteFile(mp3Path, audio, 0600); err != nil { - t.Fatal(err) - } - quality, err := GetMP3Quality(mp3Path) - if err != nil || quality.SampleRate != 44100 || quality.Bitrate != 128000 { - t.Fatalf("MP3 quality = %#v/%v", quality, err) - } - if _, _, err := extractMP3CoverArt(filepath.Join(t.TempDir(), "missing.mp3")); err == nil { - t.Fatal("expected missing MP3 cover error") - } -} - -func TestM4AMetadataAtomHelpers(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "tagged.m4a") - cover := []byte{0xFF, 0xD8, 0xFF, 0x00} - ilstPayload := []byte{} - ilstPayload = append(ilstPayload, buildM4ATextTag("\xa9nam", "M4A Title")...) - ilstPayload = append(ilstPayload, buildM4ATextTag("\xa9ART", "M4A Artist")...) - ilstPayload = append(ilstPayload, buildM4ATextTag("\xa9alb", "M4A Album")...) - ilstPayload = append(ilstPayload, buildM4ATextTag("aART", "Album Artist")...) - ilstPayload = append(ilstPayload, buildM4ATextTag("\xa9day", "2026")...) - ilstPayload = append(ilstPayload, buildM4ATextTag("\xa9gen", "Pop")...) - ilstPayload = append(ilstPayload, buildM4ATextTag("\xa9wrt", "Composer")...) - ilstPayload = append(ilstPayload, buildM4ATextTag("\xa9cmt", "[ti:Comment Lyrics]")...) - ilstPayload = append(ilstPayload, buildM4ATextTag("cprt", "Copyright")...) - ilstPayload = append(ilstPayload, buildM4ATextTag("\xa9lyr", "[00:00.00]M4A Lyrics")...) - ilstPayload = append(ilstPayload, buildM4AIndexTag("trkn", 3, 12)...) - ilstPayload = append(ilstPayload, buildM4AIndexTag("disk", 1, 2)...) - ilstPayload = append(ilstPayload, buildM4AFreeformAtom("ISRC", "USRC17607839")...) - ilstPayload = append(ilstPayload, buildM4AFreeformAtom("LABEL", "Label")...) - ilstPayload = append(ilstPayload, buildM4AFreeformAtom("RELEASETYPE", "single")...) - ilstPayload = append(ilstPayload, buildM4AFreeformAtom("BARCODE", "4006381333931")...) - ilstPayload = append(ilstPayload, buildM4AInt8Atom("rtng", 1)...) - ilstPayload = append(ilstPayload, buildM4AFreeformAtom("REPLAYGAIN_TRACK_GAIN", "-6.50 dB")...) - ilstPayload = append(ilstPayload, buildM4AAtom("covr", buildM4AAtom("data", append([]byte{0, 0, 0, 13, 0, 0, 0, 0}, cover...)))...) - fileData := buildM4AFileWithIlst(ilstPayload, true) - if err := os.WriteFile(path, fileData, 0600); err != nil { - t.Fatal(err) - } - - meta, err := ReadM4ATags(path) - if err != nil { - t.Fatalf("ReadM4ATags: %v", err) - } - if meta.Title != "M4A Title" || meta.Artist != "M4A Artist" || meta.TrackNumber != 3 || meta.TotalTracks != 12 || meta.ISRC != "USRC17607839" { - t.Fatalf("M4A metadata = %#v", meta) - } - if !meta.Explicit || meta.AlbumType != "single" || meta.UPC != "4006381333931" { - t.Fatalf("M4A release identity = %#v", meta) - } - if lyrics, err := extractLyricsFromM4A(path); err != nil || !strings.Contains(lyrics, "M4A Lyrics") { - t.Fatalf("extractLyricsFromM4A = %q/%v", lyrics, err) - } - if image, err := extractCoverFromM4A(path); err != nil || !bytes.Equal(image, cover) { - t.Fatalf("extractCoverFromM4A = %#v/%v", image, err) - } - if pathInfo, err := func() (m4aMetadataPath, error) { - f, err := os.Open(path) - if err != nil { - return m4aMetadataPath{}, err - } - defer f.Close() - info, _ := f.Stat() - return findM4AMetadataPath(f, info.Size()) - }(); err != nil || pathInfo.udta == nil { - t.Fatalf("findM4AMetadataPath = %#v/%v", pathInfo, err) - } - if err := EditM4AReplayGain(path, map[string]string{"replaygain_track_gain": "-5.00 dB", "replaygain_track_peak": "0.98"}); err != nil { - t.Fatalf("EditM4AReplayGain: %v", err) - } - edited, err := ReadM4ATags(path) - if err != nil || edited.ReplayGainTrackGain != "-5.00 dB" || edited.ReplayGainTrackPeak != "0.98" { - t.Fatalf("edited M4A = %#v/%v", edited, err) - } - - noUdtaPath := filepath.Join(dir, "noudta.m4a") - if err := os.WriteFile(noUdtaPath, buildM4AFileWithIlst(buildM4ATextTag("\xa9nam", "No Udta"), false), 0600); err != nil { - t.Fatal(err) - } - if meta, err := ReadM4ATags(noUdtaPath); err != nil || meta.Title != "No Udta" { - t.Fatalf("ReadM4ATags no udta = %#v/%v", meta, err) - } - if _, err := ReadM4ATags(filepath.Join(dir, "missing.m4a")); err == nil { - t.Fatal("expected missing M4A error") - } - emptyM4A := filepath.Join(dir, "empty.m4a") - if err := os.WriteFile(emptyM4A, buildM4AFileWithIlst(nil, true), 0600); err != nil { - t.Fatal(err) - } - if _, err := ReadM4ATags(emptyM4A); err == nil { - t.Fatal("expected empty M4A tags error") - } - if _, err := extractCoverFromM4A(emptyM4A); err == nil { - t.Fatal("expected missing M4A cover error") - } - if _, err := extractLyricsFromM4A(emptyM4A); err == nil { - t.Fatal("expected missing M4A lyrics error") - } - - syncedM4A := filepath.Join(dir, "synced.m4a") - syncedIlst := buildM4AFreeformAtom( - "SYNCEDLYRICS", - "[00:01.00]M4A synced lyrics", - ) - if err := os.WriteFile( - syncedM4A, - buildM4AFileWithIlst(syncedIlst, true), - 0600, - ); err != nil { - t.Fatal(err) - } - if synced, err := ReadM4ATags(syncedM4A); err != nil || - synced.Lyrics != "[00:01.00]M4A synced lyrics" { - t.Fatalf("M4A SYNCEDLYRICS = %#v/%v", synced, err) - } - - sidecarAudio := filepath.Join(dir, "sidecar.mp3") - if err := os.WriteFile(sidecarAudio, []byte("audio"), 0600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "sidecar.lrc"), []byte(" [00:00.00]Sidecar "), 0600); err != nil { - t.Fatal(err) - } - if lyrics, err := extractLyricsFromSidecarLRC(sidecarAudio); err != nil || !strings.Contains(lyrics, "Sidecar") { - t.Fatalf("sidecar lyrics = %q/%v", lyrics, err) - } - if !looksLikeEmbeddedLyrics("[ti:Song]") || !looksLikeEmbeddedLyrics("[00:00.00]Line\n[00:01.00]Next") || looksLikeEmbeddedLyrics("plain") { - t.Fatal("embedded lyric heuristic mismatch") - } - if formatIndexValue(3, 12) != "3/12" || formatIndexValue(3, 0) != "3" || formatIndexValue(0, 12) != "" { - t.Fatal("formatIndexValue mismatch") - } - if parsePositiveInt(" 42 ") != 42 || parsePositiveInt("bad") != 0 { - t.Fatal("parsePositiveInt mismatch") - } - if !hasMapKey(map[string]string{"x": "y"}, "x") { - t.Fatal("expected map key") - } - if _, ok := parseReplayGainDb("-6.50 dB"); !ok { - t.Fatal("expected ReplayGain dB parse") - } - if _, ok := parseReplayGainPeak("0.98"); !ok { - t.Fatal("expected ReplayGain peak parse") - } - if norm := buildITunNORMTag("-6.50 dB", "0.98"); norm == "" { - t.Fatal("expected iTunNORM") - } - if fields := collectM4AReplayGainFields(map[string]string{"replaygain_track_gain": "-6 dB", "replaygain_track_peak": "0.9"}); fields["iTunNORM"] == "" { - t.Fatalf("ReplayGain fields = %#v", fields) - } - - qualityPath := filepath.Join(dir, "quality-alac.m4a") - mvhd := make([]byte, 20) - binary.BigEndian.PutUint32(mvhd[12:16], 1000) - binary.BigEndian.PutUint32(mvhd[16:20], 180000) - sampleEntry := make([]byte, 32) - copy(sampleEntry[0:4], "alac") - binary.BigEndian.PutUint16(sampleEntry[22:24], 24) - sampleEntry[28] = 0xAC - sampleEntry[29] = 0x44 - alacConfig := make([]byte, 24) - alacConfig[5] = 24 - binary.BigEndian.PutUint32(alacConfig[20:24], 44100) - alacEntryPayload := append(append([]byte{}, sampleEntry[4:]...), buildM4AAtom("alac", alacConfig)...) - qualityFile := append(buildM4AAtom("ftyp", []byte("M4A \x00\x00\x00\x00")), buildM4AAtom("moov", append(buildM4AAtom("mvhd", mvhd), buildM4AAtom("alac", alacEntryPayload)...))...) - if err := os.WriteFile(qualityPath, qualityFile, 0600); err != nil { - t.Fatal(err) - } - if quality, err := GetM4AQuality(qualityPath); err != nil || quality.BitDepth != 24 || quality.SampleRate != 44100 || quality.Duration != 180 { - t.Fatalf("GetM4AQuality = %#v/%v", quality, err) - } - if quality, err := GetAudioQuality(qualityPath); err != nil || quality.SampleRate != 44100 { - t.Fatalf("GetAudioQuality M4A = %#v/%v", quality, err) - } - aacQualityPath := filepath.Join(dir, "quality-aac.m4a") - copy(sampleEntry[0:4], "mp4a") - aacQualityFile := append(buildM4AAtom("ftyp", []byte("M4A \x00\x00\x00\x00")), buildM4AAtom("moov", append(buildM4AAtom("mvhd", mvhd), sampleEntry...))...) - if err := os.WriteFile(aacQualityPath, aacQualityFile, 0600); err != nil { - t.Fatal(err) - } - if quality, err := GetM4AQuality(aacQualityPath); err != nil || quality.BitDepth != 0 || quality.SampleRate != 44100 || quality.Duration != 180 { - t.Fatalf("GetM4AQuality AAC = %#v/%v", quality, err) - } - eac3QualityPath := filepath.Join(dir, "quality-eac3.m4a") - zeroMvhd := make([]byte, 20) - eac3SampleEntry := make([]byte, 32) - copy(eac3SampleEntry[0:4], "ec-3") - eac3SampleEntry[28] = 0xBB - eac3SampleEntry[29] = 0x80 - mdhd := make([]byte, 20) - binary.BigEndian.PutUint32(mdhd[12:16], 48000) - binary.BigEndian.PutUint32(mdhd[16:20], 48000*123) - eac3QualityFile := append( - buildM4AAtom("ftyp", []byte("M4A \x00\x00\x00\x00")), - buildM4AAtom("moov", append( - append(buildM4AAtom("mvhd", zeroMvhd), buildM4AAtom("trak", buildM4AAtom("mdia", buildM4AAtom("mdhd", mdhd)))...), - eac3SampleEntry..., - ))..., - ) - if err := os.WriteFile(eac3QualityPath, eac3QualityFile, 0600); err != nil { - t.Fatal(err) - } - if quality, err := GetM4AQuality(eac3QualityPath); err != nil || quality.Codec != "eac3" || quality.Duration != 123 { - t.Fatalf("GetM4AQuality EAC3 mdhd fallback = %#v/%v", quality, err) - } - if _, _, ok := parseALACSpecificConfig(make([]byte, 4)); ok { - t.Fatal("short ALAC config should not parse") - } - alac := make([]byte, 24) - alac[5] = 16 - binary.BigEndian.PutUint32(alac[20:24], 48000) - if depth, rate, ok := parseALACSpecificConfig(alac); !ok || depth != 16 || rate != 48000 { - t.Fatalf("ALAC config = %d/%d/%v", depth, rate, ok) - } -} - -func TestOggMetadataQualityAndCoverHelpers(t *testing.T) { - dir := t.TempDir() - opusHead := make([]byte, 19) - copy(opusHead[0:8], "OpusHead") - binary.LittleEndian.PutUint16(opusHead[10:12], 312) - binary.LittleEndian.PutUint32(opusHead[12:16], 48000) - - var comments bytes.Buffer - binary.Write(&comments, binary.LittleEndian, uint32(6)) - comments.WriteString("vendor") - entries := []string{ - "TITLE=Ogg Title", - "ARTIST=Artist", - "ALBUMARTIST=Album Artist", - "TRACKNUMBER=2/9", - "DISCNUMBER=1/2", - "SYNCEDLYRICS=[00:00.00]Ogg Lyrics", - "ITUNESADVISORY=1", - "RELEASETYPE=ep", - "BARCODE=4006381333931", - } - binary.Write(&comments, binary.LittleEndian, uint32(len(entries))) - for _, entry := range entries { - binary.Write(&comments, binary.LittleEndian, uint32(len(entry))) - comments.WriteString(entry) - } - opusTags := append([]byte("OpusTags"), comments.Bytes()...) - oggPath := filepath.Join(dir, "tagged.opus") - oggData := append(buildOggPage(0x02, 0, opusHead), buildOggPage(0x00, 48000+312, opusTags)...) - if err := os.WriteFile(oggPath, oggData, 0600); err != nil { - t.Fatal(err) - } - quality, err := GetOggQuality(oggPath) - if err != nil || quality.SampleRate != 48000 || quality.Duration != 1 { - t.Fatalf("GetOggQuality = %#v/%v", quality, err) - } - meta, err := ReadOggVorbisComments(oggPath) - if err != nil || meta.Title != "Ogg Title" || meta.TrackNumber != 2 || meta.TotalTracks != 9 { - t.Fatalf("ReadOggVorbisComments = %#v/%v", meta, err) - } - if meta.Lyrics != "[00:00.00]Ogg Lyrics" { - t.Fatalf("Ogg SYNCEDLYRICS = %q", meta.Lyrics) - } - if !meta.Explicit || meta.AlbumType != "ep" || meta.UPC != "4006381333931" { - t.Fatalf("Ogg release identity = %#v", meta) - } - - picture := buildTestFLACPictureBlock([]byte{0x89, 0x50, 0x4E, 0x47}, "image/png") - pictureComment := "METADATA_BLOCK_PICTURE=" + base64.StdEncoding.EncodeToString(picture) - var coverComments bytes.Buffer - binary.Write(&coverComments, binary.LittleEndian, uint32(6)) - coverComments.WriteString("vendor") - binary.Write(&coverComments, binary.LittleEndian, uint32(1)) - binary.Write(&coverComments, binary.LittleEndian, uint32(len(pictureComment))) - coverComments.WriteString(pictureComment) - coverPath := filepath.Join(dir, "cover.opus") - coverData := append(buildOggPage(0x02, 0, opusHead), buildOggPage(0x00, 48000+312, append([]byte("OpusTags"), coverComments.Bytes()...))...) - if err := os.WriteFile(coverPath, coverData, 0600); err != nil { - t.Fatal(err) - } - if image, mime, err := extractOggCoverArt(coverPath); err != nil || mime != "image/png" || len(image) == 0 { - t.Fatalf("extractOggCoverArt = %s/%#v/%v", mime, image, err) - } - if image, mime, err := extractAnyCoverArtWithHint(coverPath, "cover.opus"); err != nil || mime != "image/png" || len(image) == 0 { - t.Fatalf("extractAnyCoverArtWithHint = %s/%#v/%v", mime, image, err) - } - extractedCoverPath := filepath.Join(dir, "extracted.png") - if err := ExtractCoverToFile(coverPath, extractedCoverPath); err != nil { - t.Fatalf("ExtractCoverToFile = %v", err) - } - if data := mustReadFile(t, extractedCoverPath); len(data) == 0 { - t.Fatal("expected extracted cover data") - } - cachePath, err := SaveCoverToCacheWithHintAndKey(coverPath, "cover.opus", dir, "key") - if err != nil || cachePath == "" { - t.Fatalf("SaveCoverToCacheWithHintAndKey = %q/%v", cachePath, err) - } - if _, err := SaveCoverToCacheWithHintAndKey(filepath.Join(dir, "missing.opus"), "missing.opus", dir, "missing"); err == nil { - t.Fatal("expected missing cover cache error") - } - - badPath := filepath.Join(dir, "bad.ogg") - if err := os.WriteFile(badPath, []byte("bad"), 0600); err != nil { - t.Fatal(err) - } - if _, err := GetOggQuality(badPath); err == nil { - t.Fatal("expected invalid Ogg quality error") - } -} - -func buildM4ADataPayload(payload []byte) []byte { - return append([]byte{0, 0, 0, 1, 0, 0, 0, 0}, payload...) -} - -func buildM4ATextTag(atomType, value string) []byte { - return buildM4AAtom(atomType, buildM4AAtom("data", buildM4ADataPayload([]byte(value)))) -} - -func buildM4AIndexTag(atomType string, number, total int) []byte { - payload := []byte{0, 0, 0, byte(number), 0, byte(total), 0, 0} - return buildM4AAtom(atomType, buildM4AAtom("data", buildM4ADataPayload(payload))) -} - -func buildM4AFileWithIlst(ilstPayload []byte, withUdta bool) []byte { - ilst := buildM4AAtom("ilst", ilstPayload) - meta := buildM4AAtom("meta", append([]byte{0, 0, 0, 0}, ilst...)) - moovPayload := meta - if withUdta { - moovPayload = buildM4AAtom("udta", meta) - } - return append(buildM4AAtom("ftyp", []byte("M4A \x00\x00\x00\x00")), buildM4AAtom("moov", moovPayload)...) -} - -func buildOggPage(headerType byte, granule uint64, packet []byte) []byte { - header := make([]byte, 27) - copy(header[0:4], "OggS") - header[4] = 0 - header[5] = headerType - binary.LittleEndian.PutUint64(header[6:14], granule) - header[26] = 1 - return append(append(header, byte(len(packet))), packet...) -} - -func buildTestFLACPictureBlock(image []byte, mime string) []byte { - var picture bytes.Buffer - binary.Write(&picture, binary.BigEndian, uint32(3)) - binary.Write(&picture, binary.BigEndian, uint32(len(mime))) - picture.WriteString(mime) - binary.Write(&picture, binary.BigEndian, uint32(0)) - binary.Write(&picture, binary.BigEndian, uint32(1)) - binary.Write(&picture, binary.BigEndian, uint32(1)) - binary.Write(&picture, binary.BigEndian, uint32(32)) - binary.Write(&picture, binary.BigEndian, uint32(0)) - binary.Write(&picture, binary.BigEndian, uint32(len(image))) - picture.Write(image) - return picture.Bytes() -} - -func TestR128ToReplayGainDb(t *testing.T) { - // -1280/256 = -5 dB on the R128 (-23 LUFS) scale -> 0 dB ReplayGain (-18). - if db, ok := r128ToReplayGainDb("-1280"); !ok || db != "0.00 dB" { - t.Fatalf("got %q ok=%v", db, ok) - } - if db, ok := r128ToReplayGainDb(" -2944 "); !ok || db != "-6.50 dB" { - t.Fatalf("got %q ok=%v", db, ok) - } - if _, ok := r128ToReplayGainDb("abc"); ok { - t.Fatal("expected failure for non-numeric input") - } -} diff --git a/go_backend/cancel.go b/go_backend/cancel.go deleted file mode 100644 index 1d4ad52b..00000000 --- a/go_backend/cancel.go +++ /dev/null @@ -1,210 +0,0 @@ -package gobackend - -import ( - "context" - "errors" - "sync" -) - -// ErrDownloadCancelled is returned when a download is cancelled by the user. -var ErrDownloadCancelled = errors.New("download cancelled") - -// ErrExtensionRequestCancelled is returned when a UI-driven extension request -// is superseded by a newer home/search request. -var ErrExtensionRequestCancelled = errors.New("extension request cancelled") - -type cancelEntry struct { - ctx context.Context - cancel context.CancelFunc - canceled bool - refs int -} - -type cancelRegistry struct { - mu sync.Mutex - entries map[string]*cancelEntry -} - -var ( - downloadCancels = &cancelRegistry{entries: make(map[string]*cancelEntry)} - extensionRequestCancels = &cancelRegistry{entries: make(map[string]*cancelEntry)} -) - -func (r *cancelRegistry) init(id string) context.Context { - if id == "" { - return context.Background() - } - - r.mu.Lock() - defer r.mu.Unlock() - - if entry, ok := r.entries[id]; ok { - if entry.ctx == nil { - ctx, cancel := context.WithCancel(context.Background()) - entry.ctx = ctx - entry.cancel = cancel - if entry.canceled && entry.cancel != nil { - entry.cancel() - } - } - entry.refs++ - return entry.ctx - } - - ctx, cancel := context.WithCancel(context.Background()) - r.entries[id] = &cancelEntry{ - ctx: ctx, - cancel: cancel, - canceled: false, - refs: 1, - } - return ctx -} - -func (r *cancelRegistry) context(id string) context.Context { - if id == "" { - return context.Background() - } - r.mu.Lock() - defer r.mu.Unlock() - if entry, ok := r.entries[id]; ok && entry.ctx != nil { - return entry.ctx - } - return context.Background() -} - -func (r *cancelRegistry) requestCancel(id string) { - if id == "" { - return - } - - r.mu.Lock() - if entry, ok := r.entries[id]; ok { - entry.canceled = true - if entry.cancel != nil { - entry.cancel() - } - } else { - r.entries[id] = &cancelEntry{canceled: true} - } - r.mu.Unlock() -} - -// requestCancelActive marks every entry with live work as cancelled and -// returns its ID. Pending cancellation sentinels (refs <= 0) are deliberately -// ignored so a platform lifecycle callback cannot poison a future retry. -func (r *cancelRegistry) requestCancelActive() []string { - r.mu.Lock() - ids := make([]string, 0, len(r.entries)) - for id, entry := range r.entries { - if entry == nil || entry.refs <= 0 { - continue - } - entry.canceled = true - if entry.cancel != nil { - entry.cancel() - } - ids = append(ids, id) - } - r.mu.Unlock() - return ids -} - -func (r *cancelRegistry) isCancelled(id string) bool { - if id == "" { - return false - } - - r.mu.Lock() - entry, ok := r.entries[id] - canceled := ok && entry.canceled - r.mu.Unlock() - return canceled -} - -// resetIfIdle removes a cancellation entry that has no active work attached -// (refs <= 0). Such entries exist to catch an item that is just about to -// start, but if the item never starts the flag lingers and the next explicit -// retry would consume it and abort immediately. -func (r *cancelRegistry) resetIfIdle(id string) { - if id == "" { - return - } - - r.mu.Lock() - if entry, ok := r.entries[id]; ok && entry.refs <= 0 { - delete(r.entries, id) - } - r.mu.Unlock() -} - -func (r *cancelRegistry) release(id string) { - if id == "" { - return - } - - r.mu.Lock() - if entry, ok := r.entries[id]; ok { - entry.refs-- - if entry.refs <= 0 { - delete(r.entries, id) - } - } - r.mu.Unlock() -} - -func initDownloadCancel(itemID string) context.Context { - return downloadCancels.init(itemID) -} - -func downloadCancelContext(itemID string) context.Context { - return downloadCancels.context(itemID) -} - -func cancelDownload(itemID string) { - if itemID == "" { - return - } - downloadCancels.requestCancel(itemID) - RemoveItemProgress(itemID) -} - -func cancelAllActiveDownloads() []string { - itemIDs := downloadCancels.requestCancelActive() - for _, itemID := range itemIDs { - RemoveItemProgress(itemID) - } - return itemIDs -} - -func isDownloadCancelled(itemID string) bool { - return downloadCancels.isCancelled(itemID) -} - -func resetDownloadCancel(itemID string) { - downloadCancels.resetIfIdle(itemID) -} - -func clearDownloadCancel(itemID string) { - downloadCancels.release(itemID) -} - -func initExtensionRequestCancel(requestID string) context.Context { - return extensionRequestCancels.init(requestID) -} - -func extensionRequestCancelContext(requestID string) context.Context { - return extensionRequestCancels.context(requestID) -} - -func cancelExtensionRequest(requestID string) { - extensionRequestCancels.requestCancel(requestID) -} - -func isExtensionRequestCancelled(requestID string) bool { - return extensionRequestCancels.isCancelled(requestID) -} - -func clearExtensionRequestCancel(requestID string) { - extensionRequestCancels.release(requestID) -} diff --git a/go_backend/cover.go b/go_backend/cover.go deleted file mode 100644 index d6573365..00000000 --- a/go_backend/cover.go +++ /dev/null @@ -1,319 +0,0 @@ -package gobackend - -import ( - "bytes" - "fmt" - "image" - _ "image/gif" - "image/jpeg" - "image/png" - "io" - "net/http" - "sync" - "time" - - xdraw "golang.org/x/image/draw" - _ "golang.org/x/image/webp" -) - -// downloadCoverToMemory downloads exactly the URL supplied by the metadata -// provider. Cover-resolution selection belongs to the provider extension; -// the app must not infer a provider from its CDN URL or rewrite that URL. -func downloadCoverToMemory(coverURL string) ([]byte, error) { - if coverURL == "" { - return nil, fmt.Errorf("no cover URL provided") - } - - data, err := fetchCoverCached(coverURL) - if err != nil { - return nil, err - } - // Cached bytes are shared across goroutines and must never be mutated; - // hand callers their own copy. - return append([]byte(nil), data...), nil -} - -const ( - embeddedCoverJPEGQuality = 88 - // Bound Library cover cache dimensions. - libraryCoverMaxDimension = 800 - maxCoverDownloadBytes = 24 * 1024 * 1024 - // Decoding arbitrary provider artwork allocates roughly four bytes per - // pixel. Refuse pathological images before Decode so a malicious extension - // cannot force an unbounded mobile allocation. Normal artwork through - // 4000x4000 is still accepted and downscaled. - maxCoverDecodePixels int64 = 16_000_000 -) - -// downloadCoverToMemorySized returns provider artwork with its aspect ratio -// preserved and its longest side capped at maxDimension. A non-positive limit -// keeps the original bytes. Images already within the limit are also returned -// byte-for-byte so this option never introduces needless generation loss. -func downloadCoverToMemorySized(coverURL string, maxDimension int) ([]byte, error) { - if maxDimension <= 0 { - return downloadCoverToMemory(coverURL) - } - - variantKey := fmt.Sprintf("%s\x00max-dimension=%d", coverURL, maxDimension) - data, err := fetchCoverCachedWithKey(variantKey, func() ([]byte, error) { - original, fetchErr := fetchCoverCached(coverURL) - if fetchErr != nil { - return nil, fetchErr - } - resized, changed, resizeErr := resizeCoverForEmbedding( - original, - maxDimension, - ) - if resizeErr != nil { - // A requested limit is a hard ceiling. Omitting an unsupported cover is - // preferable to silently embedding the oversized original. The default - // (maxDimension == 0) never enters this path and remains compatible. - return nil, fmt.Errorf("resize artwork: %w", resizeErr) - } - if changed { - width, height := coverDimensions(resized) - GoLog( - "[Cover] Downscaled artwork to %dx%d (%d KB -> %d KB)", - width, - height, - len(original)/1024, - len(resized)/1024, - ) - } - return resized, nil - }) - if err != nil { - return nil, err - } - return append([]byte(nil), data...), nil -} - -func resizeCoverForEmbedding(data []byte, maxDimension int) ([]byte, bool, error) { - if len(data) == 0 || maxDimension <= 0 { - return data, false, nil - } - - config, format, err := image.DecodeConfig(bytes.NewReader(data)) - if err != nil { - return nil, false, fmt.Errorf("decode artwork dimensions: %w", err) - } - if config.Width <= 0 || config.Height <= 0 { - return nil, false, fmt.Errorf("invalid artwork dimensions %dx%d", config.Width, config.Height) - } - if config.Width <= maxDimension && config.Height <= maxDimension { - return data, false, nil - } - if int64(config.Width)*int64(config.Height) > maxCoverDecodePixels { - return nil, false, fmt.Errorf( - "artwork dimensions %dx%d exceed safe decode limit", - config.Width, - config.Height, - ) - } - - source, decodedFormat, err := image.Decode(bytes.NewReader(data)) - if err != nil { - return nil, false, fmt.Errorf("decode artwork: %w", err) - } - if decodedFormat != "" { - format = decodedFormat - } - - destinationWidth, destinationHeight := scaledCoverDimensions( - config.Width, - config.Height, - maxDimension, - ) - destination := image.NewRGBA( - image.Rect(0, 0, destinationWidth, destinationHeight), - ) - xdraw.ApproxBiLinear.Scale( - destination, - destination.Bounds(), - source, - source.Bounds(), - xdraw.Over, - nil, - ) - - var encoded bytes.Buffer - if format == "png" { - if err := png.Encode(&encoded, destination); err != nil { - return nil, false, fmt.Errorf("encode resized PNG artwork: %w", err) - } - } else if err := jpeg.Encode( - &encoded, - destination, - &jpeg.Options{Quality: embeddedCoverJPEGQuality}, - ); err != nil { - return nil, false, fmt.Errorf("encode resized JPEG artwork: %w", err) - } - - return encoded.Bytes(), true, nil -} - -func scaledCoverDimensions(width, height, maxDimension int) (int, int) { - if width >= height { - scaledHeight := max(1, (height*maxDimension+width/2)/width) - return maxDimension, scaledHeight - } - scaledWidth := max(1, (width*maxDimension+height/2)/height) - return scaledWidth, maxDimension -} - -func coverDimensions(data []byte) (int, int) { - if len(data) == 0 { - return 0, 0 - } - config, _, err := image.DecodeConfig(bytes.NewReader(data)) - if err != nil || config.Width <= 0 || config.Height <= 0 { - return 0, 0 - } - return config.Width, config.Height -} - -const ( - coverCacheMaxBytes = 24 * 1024 * 1024 - coverCacheTTL = 15 * time.Minute -) - -type coverCacheEntry struct { - data []byte - expiresAt time.Time -} - -type coverInflightCall struct { - wg sync.WaitGroup - data []byte - err error -} - -var ( - coverMu sync.Mutex - coverCache = map[string]*coverCacheEntry{} - coverCacheBytes int - coverInflight = map[string]*coverInflightCall{} - coverFetch = fetchCoverBytes -) - -func clearCoverMemoryCache() { - coverMu.Lock() - coverCache = map[string]*coverCacheEntry{} - coverCacheBytes = 0 - coverMu.Unlock() -} - -// fetchCoverCached returns cover bytes for a final URL, collapsing concurrent -// requests for the same URL into a single fetch (singleflight) and caching -// results in memory for the duration of an album batch. The returned slice is -// shared; callers must copy before mutating. -func fetchCoverCached(downloadURL string) ([]byte, error) { - return fetchCoverCachedWithKey(downloadURL, func() ([]byte, error) { - return coverFetch(downloadURL) - }) -} - -// fetchCoverCachedWithKey collapses both original cover downloads and derived -// size variants. This keeps native-worker album batches from decoding and -// resizing the same artwork once per track. -func fetchCoverCachedWithKey( - cacheKey string, - fetch func() ([]byte, error), -) ([]byte, error) { - coverMu.Lock() - if e, ok := coverCache[cacheKey]; ok { - if time.Now().Before(e.expiresAt) { - data := e.data - coverMu.Unlock() - return data, nil - } - delete(coverCache, cacheKey) - coverCacheBytes -= len(e.data) - } - if call, ok := coverInflight[cacheKey]; ok { - coverMu.Unlock() - call.wg.Wait() - return call.data, call.err - } - call := &coverInflightCall{} - // Default error so a panicking fetch never strands waiters with a - // (nil, nil) "success"; overwritten on normal completion. - call.err = fmt.Errorf("cover fetch aborted") - call.wg.Add(1) - coverInflight[cacheKey] = call - coverMu.Unlock() - - defer func() { - call.wg.Done() - coverMu.Lock() - delete(coverInflight, cacheKey) - coverMu.Unlock() - }() - - data, err := fetch() - call.data, call.err = data, err - if err == nil { - coverCachePut(cacheKey, data) - } - return data, err -} - -func coverCachePut(downloadURL string, data []byte) { - if len(data) == 0 || len(data) > coverCacheMaxBytes { - return - } - coverMu.Lock() - defer coverMu.Unlock() - if e, ok := coverCache[downloadURL]; ok { - coverCacheBytes -= len(e.data) - } - coverCache[downloadURL] = &coverCacheEntry{data: data, expiresAt: time.Now().Add(coverCacheTTL)} - coverCacheBytes += len(data) - for coverCacheBytes > coverCacheMaxBytes && len(coverCache) > 1 { - var oldestKey string - var oldest time.Time - first := true - for k, e := range coverCache { - if first || e.expiresAt.Before(oldest) { - oldest, oldestKey, first = e.expiresAt, k, false - } - } - coverCacheBytes -= len(coverCache[oldestKey].data) - delete(coverCache, oldestKey) - } -} - -func fetchCoverBytes(downloadURL string) ([]byte, error) { - client := NewHTTPClientWithTimeout(DefaultTimeout) - - req, err := http.NewRequest("GET", downloadURL, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - resp, err := DoRequestWithUserAgent(client, req) - if err != nil { - return nil, fmt.Errorf("failed to download cover: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("cover download failed: HTTP %d", resp.StatusCode) - } - if resp.ContentLength > maxCoverDownloadBytes { - return nil, fmt.Errorf("cover download exceeds %d MiB limit", maxCoverDownloadBytes/(1024*1024)) - } - - data, err := io.ReadAll(io.LimitReader(resp.Body, maxCoverDownloadBytes+1)) - if err != nil { - return nil, fmt.Errorf("failed to read cover data: %w", err) - } - if len(data) > maxCoverDownloadBytes { - return nil, fmt.Errorf("cover download exceeds %d MiB limit", maxCoverDownloadBytes/(1024*1024)) - } - - width, height := coverDimensions(data) - GoLog("[Cover] Downloaded %d KB (%dx%d)", len(data)/1024, width, height) - - return data, nil -} diff --git a/go_backend/cover_test.go b/go_backend/cover_test.go deleted file mode 100644 index f9d60866..00000000 --- a/go_backend/cover_test.go +++ /dev/null @@ -1,257 +0,0 @@ -package gobackend - -import ( - "bytes" - "fmt" - "image" - "image/color" - "image/jpeg" - "image/png" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" -) - -func encodedTestCover(t *testing.T, width, height int, format string) []byte { - t.Helper() - img := image.NewRGBA(image.Rect(0, 0, width, height)) - for y := 0; y < height; y++ { - for x := 0; x < width; x++ { - img.SetRGBA(x, y, color.RGBA{ - R: uint8(x % 256), - G: uint8(y % 256), - B: uint8((x + y) % 256), - A: uint8(128 + (x+y)%128), - }) - } - } - - var encoded bytes.Buffer - var err error - if format == "png" { - err = png.Encode(&encoded, img) - } else { - err = jpeg.Encode(&encoded, img, &jpeg.Options{Quality: 95}) - } - if err != nil { - t.Fatalf("encode test cover: %v", err) - } - return encoded.Bytes() -} - -func resetCoverCache() { - coverMu.Lock() - coverCache = map[string]*coverCacheEntry{} - coverInflight = map[string]*coverInflightCall{} - coverCacheBytes = 0 - coverMu.Unlock() -} - -func TestFetchCoverCachedSingleflight(t *testing.T) { - orig := coverFetch - defer func() { coverFetch = orig }() - resetCoverCache() - - var calls int32 - entered := make(chan struct{}) - release := make(chan struct{}) - coverFetch = func(string) ([]byte, error) { - if atomic.AddInt32(&calls, 1) == 1 { - close(entered) - } - <-release - return []byte("coverbytes"), nil - } - - const url = "https://cdn.example/cover_max.jpg" - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - if _, err := fetchCoverCached(url); err != nil { - t.Errorf("leader fetch error: %v", err) - } - }() - - <-entered // leader has registered inflight and is blocked in coverFetch - - wg.Add(1) - go func() { - defer wg.Done() - if _, err := fetchCoverCached(url); err != nil { - t.Errorf("follower fetch error: %v", err) - } - }() - - close(release) - wg.Wait() - - if got := atomic.LoadInt32(&calls); got != 1 { - t.Fatalf("expected 1 fetch for concurrent requests, got %d", got) - } -} - -func TestFetchCoverCachedTTLExpiry(t *testing.T) { - orig := coverFetch - defer func() { coverFetch = orig }() - resetCoverCache() - - var calls int32 - coverFetch = func(string) ([]byte, error) { - atomic.AddInt32(&calls, 1) - return []byte("data"), nil - } - - const url = "https://cdn.example/ttl.jpg" - if _, err := fetchCoverCached(url); err != nil { - t.Fatalf("first fetch error: %v", err) - } - if _, err := fetchCoverCached(url); err != nil { - t.Fatalf("second fetch error: %v", err) - } - if got := atomic.LoadInt32(&calls); got != 1 { - t.Fatalf("expected cache hit, got %d fetches", got) - } - - coverMu.Lock() - coverCache[url].expiresAt = time.Now().Add(-time.Minute) - coverMu.Unlock() - - if _, err := fetchCoverCached(url); err != nil { - t.Fatalf("third fetch error: %v", err) - } - if got := atomic.LoadInt32(&calls); got != 2 { - t.Fatalf("expected refetch after TTL expiry, got %d fetches", got) - } -} - -func TestDownloadCoverUsesProviderURLUnchanged(t *testing.T) { - orig := coverFetch - defer func() { coverFetch = orig }() - resetCoverCache() - - const providerURL = "https://i.scdn.co/image/ab67616d00001e02example" - var requestedURL string - coverFetch = func(url string) ([]byte, error) { - requestedURL = url - return []byte("provider-cover"), nil - } - - got, err := downloadCoverToMemory(providerURL) - if err != nil { - t.Fatalf("download provider cover: %v", err) - } - if requestedURL != providerURL { - t.Fatalf("requested URL = %q, want provider URL %q", requestedURL, providerURL) - } - if string(got) != "provider-cover" { - t.Fatalf("downloaded cover = %q", got) - } -} - -func TestFetchCoverBytesRejectsOversizedResponse(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { - writer.Header().Set("Content-Length", fmt.Sprintf("%d", maxCoverDownloadBytes+1)) - writer.WriteHeader(http.StatusOK) - })) - defer server.Close() - SetAllowPrivateNetwork(true) - defer SetAllowPrivateNetwork(false) - - if _, err := fetchCoverBytes(server.URL); err == nil || !strings.Contains(err.Error(), "exceeds") { - t.Fatalf("expected oversized cover rejection, got %v", err) - } -} - -func TestResizeCoverForEmbeddingPreservesAspectRatio(t *testing.T) { - original := encodedTestCover(t, 1200, 600, "jpeg") - - resized, changed, err := resizeCoverForEmbedding(original, 500) - if err != nil { - t.Fatalf("resize cover: %v", err) - } - if !changed { - t.Fatal("expected oversized artwork to be resized") - } - config, format, err := image.DecodeConfig(bytes.NewReader(resized)) - if err != nil { - t.Fatalf("decode resized cover config: %v", err) - } - if config.Width != 500 || config.Height != 250 { - t.Fatalf("resized dimensions = %dx%d, want 500x250", config.Width, config.Height) - } - if format != "jpeg" { - t.Fatalf("resized format = %q, want jpeg", format) - } -} - -func TestResizeCoverForEmbeddingKeepsSmallArtworkByteForByte(t *testing.T) { - original := encodedTestCover(t, 320, 320, "jpeg") - - resized, changed, err := resizeCoverForEmbedding(original, 500) - if err != nil { - t.Fatalf("resize cover: %v", err) - } - if changed { - t.Fatal("artwork within the limit should not be re-encoded") - } - if !bytes.Equal(resized, original) { - t.Fatal("artwork within the limit was modified") - } -} - -func TestResizeCoverForEmbeddingPreservesPNG(t *testing.T) { - original := encodedTestCover(t, 600, 1200, "png") - - resized, changed, err := resizeCoverForEmbedding(original, 500) - if err != nil { - t.Fatalf("resize PNG cover: %v", err) - } - if !changed { - t.Fatal("expected oversized PNG artwork to be resized") - } - config, format, err := image.DecodeConfig(bytes.NewReader(resized)) - if err != nil { - t.Fatalf("decode resized PNG config: %v", err) - } - if config.Width != 250 || config.Height != 500 { - t.Fatalf("resized dimensions = %dx%d, want 250x500", config.Width, config.Height) - } - if format != "png" { - t.Fatalf("resized format = %q, want png", format) - } -} - -func TestDownloadCoverToMemorySizedCachesDerivedVariant(t *testing.T) { - originalFetch := coverFetch - defer func() { coverFetch = originalFetch }() - resetCoverCache() - - original := encodedTestCover(t, 1200, 600, "jpeg") - var calls int32 - coverFetch = func(string) ([]byte, error) { - atomic.AddInt32(&calls, 1) - return original, nil - } - - for range 2 { - resized, err := downloadCoverToMemorySized( - "https://cdn.example/album.jpg", - 500, - ) - if err != nil { - t.Fatalf("download sized cover: %v", err) - } - width, height := coverDimensions(resized) - if width != 500 || height != 250 { - t.Fatalf("sized cover = %dx%d, want 500x250", width, height) - } - } - if got := atomic.LoadInt32(&calls); got != 1 { - t.Fatalf("provider cover fetched %d times, want once", got) - } -} diff --git a/go_backend/coverage_test_helpers_test.go b/go_backend/coverage_test_helpers_test.go deleted file mode 100644 index c0160a87..00000000 --- a/go_backend/coverage_test_helpers_test.go +++ /dev/null @@ -1,416 +0,0 @@ -package gobackend - -import ( - "archive/zip" - "bytes" - "encoding/binary" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - "testing" -) - -func newTestLoadedExtension(t *testing.T, types ...ExtensionType) *loadedExtension { - t.Helper() - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "index.js"), []byte(testExtensionJS), 0600); err != nil { - t.Fatalf("write index.js: %v", err) - } - return &loadedExtension{ - ID: "coverage-ext", - Manifest: &ExtensionManifest{ - Name: "coverage-ext", - Description: "Coverage extension", - Version: "1.0.0", - Types: types, - Permissions: ExtensionPermissions{File: true, Network: []string{"example.test"}}, - SearchBehavior: &SearchBehaviorConfig{ - Enabled: true, - Placeholder: "Search coverage", - Primary: true, - Icon: "search", - }, - URLHandler: &URLHandlerConfig{Enabled: true, Patterns: []string{"https://example.test/"}}, - TrackMatching: &TrackMatchingConfig{CustomMatching: true}, - PostProcessing: &PostProcessingConfig{ - Enabled: true, - Hooks: []PostProcessingHook{{ID: "hook", Name: "Hook", DefaultEnabled: true, SupportedFormats: []string{"flac"}}}, - }, - }, - Enabled: true, - SourceDir: dir, - DataDir: t.TempDir(), - } -} - -const testExtensionJS = ` -function track(id) { - return { - id: id, - name: "Track " + id, - artists: "Artist", - albumName: "Album", - albumArtist: "Album Artist", - durationMs: 180000, - coverUrl: "https://example.test/cover.jpg", - releaseDate: "2026-05-04", - trackNumber: 1, - totalTracks: 10, - discNumber: 1, - totalDiscs: 1, - isrc: "USRC17607839", - itemType: "track", - albumType: "album", - tidalId: "alternate-id-a", - qobuzId: "alternate-id-b", - deezerId: "alternate-id-c", - spotifyId: "source-track-1", - externalLinks: { provider: "https://provider.example/1" }, - label: "Label", - copyright: "Copyright", - genre: "Pop", - composer: "Composer", - comment: "https://example.test/album/1", - audioQuality: "FLAC 24-bit", - audioModes: "DOLBY_ATMOS", - explicit: true, - upc: "0012345678901" - }; -} - -registerExtension({ - searchTracks: function(query, limit) { - return { tracks: [track("search-1")], total: 1 }; - }, - customSearch: function(query, options) { - var t = track("custom-1"); - t.name = "Custom " + query; - return [t]; - }, - getHomeFeed: function() { - return [{ id: "home-1", title: "Home", tracks: [track("home-track")] }]; - }, - getBrowseCategories: function() { - return [{ id: "cat-1", title: "Category" }]; - }, - getTrack: function(id) { - return track(id); - }, - getAlbum: function(id) { - return { - id: id, - name: "Album " + id, - artists: "Artist", - artistId: "artist-1", - coverUrl: "https://example.test/album.jpg", - releaseDate: "2026-05-04", - totalTracks: 1, - albumType: "album", - tracks: [track("album-track")] - }; - }, - getPlaylist: function(id) { - return { - id: id, - name: "Playlist " + id, - artists: "Owner", - coverUrl: "https://example.test/playlist.jpg", - totalTracks: 1, - tracks: [track("playlist-track")] - }; - }, - getArtist: function(id) { - return { - id: id, - name: "Artist", - imageUrl: "https://example.test/artist.jpg", - headerImage: "https://example.test/header.jpg", - listeners: 123, - albums: [{ id: "album-1", name: "Album", artists: "Artist", totalTracks: 1 }], - releases: [{ id: "release-1", name: "Release", artists: "Artist", totalTracks: 1, tracks: [track("release-track")] }], - topTracks: [track("top-track")] - }; - }, - enrichTrack: function(input) { - var t = track(input.id || "enriched"); - t.name = "Enriched"; - return t; - }, - checkAvailability: function(isrc, name, artist, ids) { - return { - available: true, - reason: "ok", - trackId: "download-track", - skipFallback: true, - prepared_context: { token: "prepared", resolvedTrackId: "download-track" } - }; - }, - getDownloadUrl: function(id, quality) { - return { url: "https://example.test/audio.flac", format: "flac", bitDepth: 24, sampleRate: 96000 }; - }, - download: function(id, quality, outputPath, onProgress, options) { - if (onProgress) onProgress(100); - return { - success: true, - filePath: "EXISTS:" + outputPath, - alreadyExists: false, - bitDepth: 24, - sampleRate: 96000, - title: options && options.preparedContext - ? options.preparedContext.token - : "Downloaded", - artist: "Artist", - album: "Album", - albumArtist: "Album Artist", - trackNumber: 1, - totalTracks: 10, - discNumber: 1, - totalDiscs: 1, - releaseDate: "2026-05-04", - coverUrl: "https://example.test/cover.jpg", - isrc: "USRC17607839", - genre: "Pop", - label: "Label", - copyright: "Copyright", - composer: "Composer", - comment: "https://example.test/album/1", - explicit: true, - albumType: "compilation", - upc: "0012345678901", - lyricsLrc: "[00:00.00]Hello", - decryptionKey: "001122", - decryption: { strategy: "mp4_decryption_key", options: { kid: "1" } } - }; - }, - fetchLyrics: function(name, artist, album, duration) { - return { syncType: "LINE_SYNCED", provider: "coverage-ext", lines: [{ startTimeMs: 0, endTimeMs: 1000, words: "Hello" }] }; - }, - handleUrl: function(url) { - return { type: "track", name: "Handled", coverUrl: "https://example.test/cover.jpg", track: track("url-track"), tracks: [track("url-track")], album: this.getAlbum("url-album"), artist: this.getArtist("url-artist") }; - }, - matchTrack: function(req) { - return { matched: true, trackId: "download-track", confidence: 0.95, reason: "exact" }; - }, - postProcess: function(path, req) { - return { success: true, newFilePath: path, bitDepth: 24, sampleRate: 96000 }; - }, - postProcessV2: function(input, metadata, hookId) { - return { success: true, newFilePath: input.path || input.uri, newFileUri: input.uri || "", bitDepth: 24, sampleRate: 96000 }; - } -}); -` - -func mustReadFile(t *testing.T, path string) []byte { - t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read file: %v", err) - } - return data -} - -func buildID3v23Tag(frames ...[]byte) []byte { - body := bytes.Join(frames, nil) - header := []byte{'I', 'D', '3', 3, 0, 0, 0, 0, 0, 0} - copy(header[6:10], syncsafeBytes(len(body))) - return append(header, body...) -} - -func id3TextFrame(id, value string) []byte { - return id3v23Frame(id, append([]byte{3}, []byte(value)...)) -} - -func id3CommentFrame(id, value string) []byte { - payload := append([]byte{3, 'e', 'n', 'g', 0}, []byte(value)...) - return id3v23Frame(id, payload) -} - -func id3UserTextFrame(id, desc, value string) []byte { - payload := append([]byte{3}, []byte(desc)...) - payload = append(payload, 0) - payload = append(payload, []byte(value)...) - return id3v23Frame(id, payload) -} - -func id3v23Frame(id string, payload []byte) []byte { - frame := make([]byte, 10+len(payload)) - copy(frame[0:4], id) - binary.BigEndian.PutUint32(frame[4:8], uint32(len(payload))) - copy(frame[10:], payload) - return frame -} - -func buildID3v22Tag(frames ...[]byte) []byte { - body := bytes.Join(frames, nil) - header := []byte{'I', 'D', '3', 2, 0, 0, 0, 0, 0, 0} - copy(header[6:10], syncsafeBytes(len(body))) - return append(header, body...) -} - -func id3v22TextFrame(id, value string) []byte { - return id3v22Frame(id, append([]byte{3}, []byte(value)...)) -} - -func id3v22CommentFrame(id, value string) []byte { - payload := append([]byte{3, 'e', 'n', 'g', 0}, []byte(value)...) - return id3v22Frame(id, payload) -} - -func id3v22Frame(id string, payload []byte) []byte { - frame := make([]byte, 6+len(payload)) - copy(frame[0:3], id) - size := len(payload) - frame[3] = byte(size >> 16) - frame[4] = byte(size >> 8) - frame[5] = byte(size) - copy(frame[6:], payload) - return frame -} - -func syncsafeBytes(size int) []byte { - return []byte{ - byte((size >> 21) & 0x7f), - byte((size >> 14) & 0x7f), - byte((size >> 7) & 0x7f), - byte(size & 0x7f), - } -} - -func buildID3v1Tag(title, artist, album, year string, track, genre byte) []byte { - tag := make([]byte, 128) - copy(tag[0:3], "TAG") - copyPadded(tag[3:33], title) - copyPadded(tag[33:63], artist) - copyPadded(tag[63:93], album) - copyPadded(tag[93:97], year) - tag[125] = 0 - tag[126] = track - tag[127] = genre - return tag -} - -func copyPadded(dst []byte, value string) { - for i := range dst { - dst[i] = ' ' - } - copy(dst, value) -} - -func writeExportCueFixture(t *testing.T, dir string) (string, string) { - t.Helper() - audioPath := filepath.Join(dir, "exports.wav") - if err := os.WriteFile(audioPath, []byte("audio"), 0600); err != nil { - t.Fatalf("write export audio: %v", err) - } - cuePath := filepath.Join(dir, "exports.cue") - cue := "PERFORMER \"Artist\"\nTITLE \"Album\"\nFILE \"exports.wav\" WAVE\n TRACK 01 AUDIO\n TITLE \"Song\"\n INDEX 01 00:00:00\n" - if err := os.WriteFile(cuePath, []byte(cue), 0600); err != nil { - t.Fatalf("write export cue: %v", err) - } - return cuePath, audioPath -} - -func escapeJSONPath(path string) string { - data, _ := json.Marshal(path) - return strings.Trim(string(data), `"`) -} - -func fakeDeezerResponse(path, rawQuery string) string { - switch { - case path == "/2.0/search/track": - if strings.Contains(rawQuery, "MISSING") { - return `{"data":[]}` - } - return `{"data":[` + fakeDeezerTrackJSON(101, true) + `]}` - case path == "/2.0/search/artist": - return `{"data":[{"id":301,"name":"Artist","picture_xl":"artist-xl","nb_fan":123}]}` - case path == "/2.0/search/album": - return `{"data":[{"id":201,"title":"Album","cover_xl":"album-xl","nb_tracks":2,"release_date":"2026-05-04","record_type":"compile","artist":{"id":301,"name":"Artist"}}]}` - case path == "/2.0/search/playlist": - return `{"data":[{"id":401,"title":"Playlist","picture_xl":"playlist-xl","nb_tracks":2,"user":{"name":"Owner"}}]}` - case path == "/2.0/track/101", path == "/2.0/track/isrc:USRC17607839": - return fakeDeezerTrackJSON(101, true) - case path == "/2.0/track/102": - return fakeDeezerTrackJSON(102, true) - case path == "/2.0/track/isrc:MISSING": - return `{"id":0}` - case path == "/2.0/album/201": - return `{"id":201,"title":"Album","cover_xl":"album-xl","release_date":"2026-05-04","nb_tracks":2,"record_type":"compile","label":"Label","copyright":"Copyright","genres":{"data":[{"name":"Pop"},{"name":"Dance"}]},"artist":{"id":301,"name":"Album Artist"},"contributors":[{"name":"Contributor A"},{"name":"Contributor B"}],"tracks":{"data":[` + fakeDeezerTrackJSON(101, true) + `,` + fakeDeezerTrackJSON(102, false) + `]}}` - case path == "/2.0/artist/301": - return `{"id":301,"name":"Artist","picture_xl":"artist-xl","nb_fan":123,"nb_album":1}` - case path == "/2.0/artist/301/albums": - return `{"data":[{"id":201,"title":"Album","release_date":"2026-05-04","nb_tracks":0,"cover_xl":"album-xl","record_type":"compile"}]}` - case path == "/2.0/artist/301/related": - return `{"data":[{"id":302,"name":"Related","picture_xl":"related-xl","nb_fan":10}]}` - case path == "/2.0/playlist/401": - return `{"id":401,"title":"Playlist","picture_xl":"playlist-xl","nb_tracks":2,"creator":{"name":"Owner"},"tracks":{"data":[` + fakeDeezerTrackJSON(101, true) + `,` + fakeDeezerTrackJSON(102, false) + `]}}` - default: - return "" - } -} - -func fakeDeezerTrackJSON(id int, withISRC bool) string { - isrc := "" - if withISRC { - isrc = `,"isrc":"USRC17607839"` - if id == 102 { - isrc = `,"isrc":"USRC17607840"` - } - } - return fmt.Sprintf(`{"id":%d,"title":"Track %d","duration":180,"track_position":%d,"disk_number":1%s,"link":"https://deezer.test/track/%d","release_date":"2026-05-04","artist":{"id":301,"name":"Artist"},"contributors":[{"name":"Contributor A"},{"name":"Contributor B"}],"album":{"id":201,"title":"Album","cover_xl":"album-xl","release_date":"2026-05-04","record_type":"album"}}`, id, id, id-100, isrc, id) -} - -func createTestExtensionPackage(t *testing.T, path, name, version, js string, extraFiles map[string]string) { - t.Helper() - out, err := os.Create(path) - if err != nil { - t.Fatalf("create extension package: %v", err) - } - defer out.Close() - - zw := zip.NewWriter(out) - defer zw.Close() - - manifest := fmt.Sprintf(`{ - "name": %q, - "displayName": %q, - "version": %q, - "description": "Packaged test extension", - "type": ["metadata_provider", "download_provider", "lyrics_provider"], - "permissions": {"network": ["example.test"], "storage": true, "file": true}, - "icon": "icon.png", - "settings": [{"key":"quality","type":"string","label":"Quality"}], - "qualityOptions": [{"id":"lossless","label":"Lossless","description":"Lossless"}], - "searchBehavior": {"enabled": true, "placeholder": "Search", "primary": true}, - "urlHandler": {"enabled": true, "patterns": ["https://example.test/"]}, - "trackMatching": {"customMatching": true}, - "postProcessing": {"enabled": true, "hooks": [{"id":"hook","name":"Hook"}]}, - "serviceHealth": [{"id":"main","url":"https://example.test/health"}], - "capabilities": {"homeFeed": true} - }`, name, name, version) - - for fileName, content := range map[string]string{ - "manifest.json": manifest, - "index.js": js, - "icon.png": "png", - } { - writer, err := zw.Create(fileName) - if err != nil { - t.Fatalf("zip create %s: %v", fileName, err) - } - if _, err := writer.Write([]byte(content)); err != nil { - t.Fatalf("zip write %s: %v", fileName, err) - } - } - for fileName, content := range extraFiles { - writer, err := zw.Create(fileName) - if err != nil { - t.Fatalf("zip create extra %s: %v", fileName, err) - } - if _, err := writer.Write([]byte(content)); err != nil { - t.Fatalf("zip write extra %s: %v", fileName, err) - } - } -} diff --git a/go_backend/cross_extension_share.go b/go_backend/cross_extension_share.go deleted file mode 100644 index 5f132b62..00000000 --- a/go_backend/cross_extension_share.go +++ /dev/null @@ -1,432 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "sort" - "strings" - "sync" -) - -type CrossExtensionShareResult struct { - ExtensionID string `json:"extension_id"` - DisplayName string `json:"display_name"` - Found bool `json:"found"` - URL string `json:"url,omitempty"` - ItemName string `json:"item_name,omitempty"` - ItemArtists string `json:"item_artists,omitempty"` - Error string `json:"error,omitempty"` -} - -var crossExtensionShareResultCache = struct { - sync.RWMutex - entries map[string]string - order []string -}{ - entries: make(map[string]string), -} - -const crossExtensionShareResultCacheLimit = 128 - -func FindCollectionAcrossExtensionsJSON(requestJSON string) (string, error) { - var req struct { - Name string `json:"name"` - Artists string `json:"artists"` - Type string `json:"type"` - SourceExtensionID string `json:"source_extension_id"` - } - if err := json.Unmarshal([]byte(requestJSON), &req); err != nil { - return "", err - } - - req.Name = strings.TrimSpace(req.Name) - req.Artists = strings.TrimSpace(req.Artists) - req.Type = strings.ToLower(strings.TrimSpace(req.Type)) - req.SourceExtensionID = strings.TrimSpace(req.SourceExtensionID) - if req.Name == "" { - return "[]", nil - } - if req.Type == "" { - req.Type = "album" - } - - providers := getExtensionManager().GetMetadataProviders() - work := make([]*extensionProviderWrapper, 0, len(providers)) - for _, provider := range providers { - if provider == nil || provider.extension == nil { - continue - } - if provider.extension.ID == req.SourceExtensionID { - continue - } - work = append(work, provider) - } - cacheKey := crossExtensionShareCacheKey(req.Name, req.Artists, req.Type, req.SourceExtensionID, work) - if cached := getCrossExtensionShareCache(cacheKey); cached != "" { - return cached, nil - } - - query := req.Name - if req.Artists != "" { - query += " " + req.Artists - } - - results := make([]CrossExtensionShareResult, len(work)) - var wg sync.WaitGroup - for i, provider := range work { - wg.Add(1) - go func(index int, p *extensionProviderWrapper) { - defer wg.Done() - results[index] = findCollectionForExtension( - p, - req.Type, - req.Name, - req.Artists, - query, - ) - }(i, provider) - } - wg.Wait() - - data, err := json.Marshal(results) - if err != nil { - return "[]", err - } - response := string(data) - if crossExtensionShareResultsCacheable(results) { - setCrossExtensionShareCache(cacheKey, response) - } - return response, nil -} - -func crossExtensionShareCacheKey(name string, artists string, itemType string, sourceExtensionID string, providers []*extensionProviderWrapper) string { - providerKeys := make([]string, 0, len(providers)) - for _, provider := range providers { - if provider == nil || provider.extension == nil { - continue - } - ext := provider.extension - displayName := "" - if ext.Manifest != nil { - displayName = ext.Manifest.DisplayName - } - providerKeys = append(providerKeys, strings.Join([]string{ - strings.TrimSpace(ext.ID), - strings.TrimSpace(displayName), - strings.TrimSpace(ext.SourceDir), - }, "\x1f")) - } - sort.Strings(providerKeys) - - return strings.Join([]string{ - normalizeLooseTitle(itemType), - normalizeLooseTitle(name), - normalizeLooseArtistName(artists), - strings.TrimSpace(sourceExtensionID), - strings.Join(providerKeys, "\x1e"), - }, "\x1d") -} - -func getCrossExtensionShareCache(key string) string { - if key == "" { - return "" - } - crossExtensionShareResultCache.RLock() - defer crossExtensionShareResultCache.RUnlock() - return crossExtensionShareResultCache.entries[key] -} - -func setCrossExtensionShareCache(key string, value string) { - if key == "" || value == "" { - return - } - crossExtensionShareResultCache.Lock() - defer crossExtensionShareResultCache.Unlock() - - if _, exists := crossExtensionShareResultCache.entries[key]; !exists { - crossExtensionShareResultCache.order = append(crossExtensionShareResultCache.order, key) - } - crossExtensionShareResultCache.entries[key] = value - - for len(crossExtensionShareResultCache.order) > crossExtensionShareResultCacheLimit { - oldest := crossExtensionShareResultCache.order[0] - crossExtensionShareResultCache.order = crossExtensionShareResultCache.order[1:] - delete(crossExtensionShareResultCache.entries, oldest) - } -} - -func crossExtensionShareResultsCacheable(results []CrossExtensionShareResult) bool { - for _, result := range results { - if result.Found { - continue - } - errText := strings.ToLower(strings.TrimSpace(result.Error)) - if errText == "" || - errText == "no results" || - errText == "unsupported collection type" || - strings.HasSuffix(errText, " not found") || - strings.Contains(errText, "found without shareable link") { - continue - } - return false - } - return true -} - -func findCollectionForExtension( - provider *extensionProviderWrapper, - itemType string, - name string, - artists string, - query string, -) CrossExtensionShareResult { - result := CrossExtensionShareResult{ - ExtensionID: provider.extension.ID, - } - if provider.extension.Manifest != nil { - result.DisplayName = provider.extension.Manifest.DisplayName - } - if result.DisplayName == "" { - result.DisplayName = provider.extension.ID - } - - searchResult, err := searchCollectionCandidates(provider, itemType, query) - if err != nil { - result.Error = err.Error() - return result - } - if searchResult == nil || len(searchResult.Tracks) == 0 { - result.Error = "no results" - return result - } - - var best *ExtTrackMetadata - switch itemType { - case "artist": - best = bestArtistTrack(searchResult.Tracks, name) - case "album": - best = bestAlbumTrack(searchResult.Tracks, name, artists) - default: - result.Error = "unsupported collection type" - return result - } - if best == nil { - result.Error = itemType + " not found" - return result - } - - url := resolveCollectionShareURL(provider.extension, itemType, best) - if url == "" { - result.Error = itemType + " found without shareable link" - return result - } - - result.Found = true - result.URL = url - if itemType == "artist" { - result.ItemName = collectionArtistName(*best) - } else { - result.ItemName = collectionAlbumName(*best) - result.ItemArtists = best.Artists - } - return result -} - -func searchCollectionCandidates(provider *extensionProviderWrapper, itemType string, query string) (*ExtSearchResult, error) { - filter := "" - switch itemType { - case "album": - filter = "albums" - case "artist": - filter = "artists" - } - - if filter != "" { - tracks, err := provider.CustomSearch(query, map[string]any{ - "filter": filter, - "limit": 10, - }) - if err == nil && len(tracks) > 0 { - return &ExtSearchResult{Tracks: tracks, Total: len(tracks)}, nil - } - } - - return provider.SearchTracks(query, 10) -} - -func bestAlbumTrack(tracks []ExtTrackMetadata, albumName string, artists string) *ExtTrackMetadata { - targetAlbum := normalizeLooseTitle(albumName) - targetArtists := normalizeLooseArtistName(artists) - bestScore := 0 - bestIndex := -1 - - for i := range tracks { - track := tracks[i] - album := normalizeLooseTitle(collectionAlbumName(track)) - trackArtists := normalizeLooseArtistName(track.Artists + " " + track.AlbumArtist) - - score := 0 - if isCollectionItemType(track, "album") { - score += 25 - } - if album == targetAlbum { - score += 100 - } else if album != "" && targetAlbum != "" && (strings.Contains(album, targetAlbum) || strings.Contains(targetAlbum, album)) { - score += 50 - } - if targetArtists != "" && (strings.Contains(trackArtists, targetArtists) || strings.Contains(targetArtists, trackArtists)) { - score += 30 - } - if score > bestScore { - bestScore = score - bestIndex = i - } - } - - if bestIndex < 0 || bestScore < 50 { - return nil - } - return &tracks[bestIndex] -} - -func bestArtistTrack(tracks []ExtTrackMetadata, artistName string) *ExtTrackMetadata { - targetArtist := normalizeLooseArtistName(artistName) - bestScore := 0 - bestIndex := -1 - - for i := range tracks { - artist := normalizeLooseArtistName(collectionArtistName(tracks[i])) - score := 0 - if isCollectionItemType(tracks[i], "artist") { - score += 25 - } - if artist == targetArtist { - score += 100 - } else if artist != "" && targetArtist != "" && (strings.Contains(artist, targetArtist) || strings.Contains(targetArtist, artist)) { - score += 60 - } - if score > bestScore { - bestScore = score - bestIndex = i - } - } - - if bestIndex < 0 || bestScore < 60 { - return nil - } - return &tracks[bestIndex] -} - -func resolveCollectionShareURL(ext *loadedExtension, itemType string, track *ExtTrackMetadata) string { - if track == nil { - return "" - } - - if itemType == "album" { - if isCollectionItemType(*track, "album") { - if url := normalizeShareURL(track.ExternalURL); url != "" { - return url - } - } - if url := normalizeShareURL(track.AlbumURL); url != "" { - return url - } - if url := urlFromExternalLinks(track.ExternalLinks, "album"); url != "" { - return url - } - if url := templateShareURL(ext, "album", firstNonEmptyTrimmed(track.AlbumID, collectionID(*track, "album"), track.AlbumURL)); url != "" { - return url - } - return "" - } - - if isCollectionItemType(*track, "artist") { - if url := normalizeShareURL(track.ExternalURL); url != "" { - return url - } - } - if url := normalizeShareURL(track.ArtistURL); url != "" { - return url - } - if url := urlFromExternalLinks(track.ExternalLinks, "artist"); url != "" { - return url - } - if url := templateShareURL(ext, "artist", firstNonEmptyTrimmed(track.ArtistID, collectionID(*track, "artist"))); url != "" { - return url - } - return "" -} - -func collectionAlbumName(track ExtTrackMetadata) string { - if isCollectionItemType(track, "album") { - return track.Name - } - return track.AlbumName -} - -func collectionArtistName(track ExtTrackMetadata) string { - if isCollectionItemType(track, "artist") { - return track.Name - } - return track.Artists -} - -func collectionID(track ExtTrackMetadata, itemType string) string { - if isCollectionItemType(track, itemType) { - return track.ID - } - return "" -} - -func isCollectionItemType(track ExtTrackMetadata, itemType string) bool { - return strings.EqualFold(strings.TrimSpace(track.ItemType), itemType) -} - -func normalizeShareURL(value string) string { - trimmed := strings.TrimSpace(value) - if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") { - return trimmed - } - return "" -} - -func urlFromExternalLinks(links map[string]string, preferredKey string) string { - for key, value := range links { - if strings.Contains(strings.ToLower(key), preferredKey) { - if url := normalizeShareURL(value); url != "" { - return url - } - } - } - return "" -} - -func templateShareURL(ext *loadedExtension, itemType string, id string) string { - if ext == nil || ext.Manifest == nil || ext.Manifest.Capabilities == nil { - return "" - } - id = stripProviderPrefix(strings.TrimSpace(id)) - if id == "" { - return "" - } - - templates, ok := ext.Manifest.Capabilities["shareUrlTemplates"].(map[string]any) - if !ok { - return "" - } - rawTemplate, ok := templates[itemType].(string) - if !ok { - return "" - } - rawTemplate = strings.TrimSpace(rawTemplate) - if rawTemplate == "" { - return "" - } - return strings.ReplaceAll(rawTemplate, "{id}", id) -} - -func stripProviderPrefix(id string) string { - if index := strings.Index(id, ":"); index > 0 && index < len(id)-1 { - return id[index+1:] - } - return id -} diff --git a/go_backend/cross_extension_share_test.go b/go_backend/cross_extension_share_test.go deleted file mode 100644 index 588de8a3..00000000 --- a/go_backend/cross_extension_share_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package gobackend - -import "testing" - -func TestCrossExtensionShareUsesAlbumCollectionItems(t *testing.T) { - ext := &loadedExtension{ - Manifest: &ExtensionManifest{ - Capabilities: map[string]any{ - "shareUrlTemplates": map[string]any{ - "album": "https://media.example/album/{id}", - }, - }, - }, - } - tracks := []ExtTrackMetadata{ - { - ID: "1440783617", - Name: "Nevermind", - Artists: "Nirvana", - ItemType: "album", - }, - } - - best := bestAlbumTrack(tracks, "Nevermind", "Nirvana") - if best == nil { - t.Fatal("expected album collection item to match") - } - if url := resolveCollectionShareURL(ext, "album", best); url != "https://media.example/album/1440783617" { - t.Fatalf("album share URL = %q", url) - } -} - -func TestCrossExtensionShareUsesArtistCollectionItems(t *testing.T) { - ext := &loadedExtension{ - Manifest: &ExtensionManifest{ - Capabilities: map[string]any{ - "shareUrlTemplates": map[string]any{ - "artist": "https://media.example/artist/{id}", - }, - }, - }, - } - tracks := []ExtTrackMetadata{ - { - ID: "UCrPe3hLA51968GwxHSZ1llw", - Name: "Nirvana", - ItemType: "artist", - }, - } - - best := bestArtistTrack(tracks, "Nirvana") - if best == nil { - t.Fatal("expected artist collection item to match") - } - if url := resolveCollectionShareURL(ext, "artist", best); url != "https://media.example/artist/UCrPe3hLA51968GwxHSZ1llw" { - t.Fatalf("artist share URL = %q", url) - } -} - -func TestCrossExtensionShareCacheKeyIsProviderOrderStable(t *testing.T) { - providerA := &extensionProviderWrapper{ - extension: &loadedExtension{ - ID: "provider-a", - SourceDir: "/extensions/provider-a", - Manifest: &ExtensionManifest{DisplayName: "Provider A"}, - }, - } - providerB := &extensionProviderWrapper{ - extension: &loadedExtension{ - ID: "provider-b", - SourceDir: "/extensions/provider-b", - Manifest: &ExtensionManifest{DisplayName: "Provider B"}, - }, - } - - first := crossExtensionShareCacheKey("Nevermind", "Nirvana", "album", "metadata-source", []*extensionProviderWrapper{providerA, providerB}) - second := crossExtensionShareCacheKey("Nevermind", "Nirvana", "album", "metadata-source", []*extensionProviderWrapper{providerB, providerA}) - if first != second { - t.Fatalf("cache key should not depend on provider order:\n%s\n%s", first, second) - } -} - -func TestCrossExtensionShareCacheableSkipsTransientErrors(t *testing.T) { - cacheable := []CrossExtensionShareResult{ - {ExtensionID: "provider-a", Found: true, URL: "https://media.example/album/1"}, - {ExtensionID: "provider-b", Error: "album not found"}, - {ExtensionID: "provider-c", Error: "no results"}, - } - if !crossExtensionShareResultsCacheable(cacheable) { - t.Fatal("expected found and deterministic not-found results to be cacheable") - } - - transient := []CrossExtensionShareResult{ - {ExtensionID: "provider-a", Found: true, URL: "https://media.example/album/1"}, - {ExtensionID: "provider-b", Error: "request failed: timeout"}, - } - if crossExtensionShareResultsCacheable(transient) { - t.Fatal("expected transient extension errors to skip cache") - } -} diff --git a/go_backend/cue_duplicate_supplement_test.go b/go_backend/cue_duplicate_supplement_test.go deleted file mode 100644 index 9433d2b8..00000000 --- a/go_backend/cue_duplicate_supplement_test.go +++ /dev/null @@ -1,301 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - "time" -) - -func TestCueParserEndToEnd(t *testing.T) { - dir := t.TempDir() - audioPath := filepath.Join(dir, "album.wav") - if err := os.WriteFile(audioPath, []byte("audio"), 0600); err != nil { - t.Fatalf("write audio: %v", err) - } - cuePath := filepath.Join(dir, "album.cue") - cue := "\ufeffREM GENRE \"Pop\"\n" + - "REM DATE 2026\n" + - "REM COMMENT \"comment\"\n" + - "REM COMPOSER \"Album Composer\"\n" + - "PERFORMER \"Album Artist\"\n" + - "TITLE \"Album Title\"\n" + - "FILE \"album.wav\" WAVE\n" + - " TRACK 01 AUDIO\n" + - " TITLE \"First\"\n" + - " PERFORMER \"Track Artist\"\n" + - " ISRC USRC17607839\n" + - " INDEX 01 00:00:00\n" + - " TRACK 02 AUDIO\n" + - " TITLE \"Second\"\n" + - " SONGWRITER \"Track Composer\"\n" + - " INDEX 00 03:00:00\n" + - " INDEX 01 03:05:00\n" - if err := os.WriteFile(cuePath, []byte(cue), 0600); err != nil { - t.Fatalf("write cue: %v", err) - } - - sheet, err := ParseCueFile(cuePath) - if err != nil { - t.Fatalf("ParseCueFile: %v", err) - } - if sheet.Performer != "Album Artist" || sheet.Title != "Album Title" || len(sheet.Tracks) != 2 { - t.Fatalf("sheet = %#v", sheet) - } - if got := parseCueTimestamp("01:02:37"); got <= 62 || got >= 63 { - t.Fatalf("timestamp = %f", got) - } - if got := formatCueTimestamp(3723.5); got != "01:02:03.500" { - t.Fatalf("format timestamp = %q", got) - } - if got := unquoteCue(" \"quoted\" "); got != "quoted" { - t.Fatalf("unquote = %q", got) - } - fileName, fileType := parseCueFileLine("unquoted album.flac FLAC") - if fileName != "unquoted album.flac" || fileType != "FLAC" { - t.Fatalf("file line = %q/%q", fileName, fileType) - } - - if resolved := ResolveCueAudioPath(cuePath, "album.flac"); resolved != audioPath { - t.Fatalf("resolved = %q want %q", resolved, audioPath) - } - info, err := BuildCueSplitInfo(cuePath, sheet, "") - if err != nil { - t.Fatalf("BuildCueSplitInfo: %v", err) - } - if info.Tracks[0].EndSec != 180 || info.Tracks[1].Composer != "Track Composer" { - t.Fatalf("split info = %#v", info.Tracks) - } - - jsonText, err := ParseCueFileJSON(cuePath, "") - if err != nil { - t.Fatalf("ParseCueFileJSON: %v", err) - } - var decoded CueSplitInfo - if err := json.Unmarshal([]byte(jsonText), &decoded); err != nil { - t.Fatalf("decode cue json: %v", err) - } - if decoded.AudioPath != audioPath { - t.Fatalf("decoded audio path = %q", decoded.AudioPath) - } - - results, err := ScanCueFileForLibraryExt(cuePath, "", "virtual/album.cue", 1234, "scan-time") - if err != nil { - t.Fatalf("ScanCueFileForLibraryExt: %v", err) - } - if len(results) != 2 || results[0].TrackName != "First" || results[0].Duration != 180 { - t.Fatalf("scan results = %#v", results) - } - if results[0].FilePath != "virtual/album.cue#track01" || results[0].Format != "cue+wav" { - t.Fatalf("scan path/format = %q/%q", results[0].FilePath, results[0].Format) - } - - if _, err := ParseCueFile(filepath.Join(dir, "missing.cue")); err == nil { - t.Fatal("expected missing cue error") - } - emptyCue := filepath.Join(dir, "empty.cue") - if err := os.WriteFile(emptyCue, []byte("TITLE \"No tracks\""), 0600); err != nil { - t.Fatal(err) - } - if _, err := ParseCueFile(emptyCue); err == nil { - t.Fatal("expected no tracks error") - } - missingDir := t.TempDir() - missingCuePath := filepath.Join(missingDir, "missing.cue") - if err := os.WriteFile(missingCuePath, []byte(cue), 0600); err != nil { - t.Fatal(err) - } - if _, err := BuildCueSplitInfo(missingCuePath, &CueSheet{FileName: "missing.wav"}, ""); err == nil { - t.Fatal("expected missing audio error") - } - if _, err := resolveCueAudioPathForLibrary(cuePath, nil, ""); err == nil { - t.Fatal("expected nil sheet error") - } - if _, err := scanCueSheetForLibrary(cuePath, nil, audioPath, "", 0, "", ""); err == nil { - t.Fatal("expected nil scan sheet error") - } -} - -func writeTestFlacWithISRC(t *testing.T, path, isrc string) { - t.Helper() - le32 := func(v int) []byte { - return []byte{byte(v), byte(v >> 8), byte(v >> 16), byte(v >> 24)} - } - vendor := "test-vendor" - comments := [][]byte{ - []byte("TITLE=Song"), - []byte("ISRC=" + isrc), - } - var vorbis []byte - vorbis = append(vorbis, le32(len(vendor))...) - vorbis = append(vorbis, vendor...) - vorbis = append(vorbis, le32(len(comments))...) - for _, comment := range comments { - vorbis = append(vorbis, le32(len(comment))...) - vorbis = append(vorbis, comment...) - } - - blockHeader := func(last bool, blockType byte, length int) []byte { - first := blockType - if last { - first |= 0x80 - } - return []byte{first, byte(length >> 16), byte(length >> 8), byte(length)} - } - - var data []byte - data = append(data, "fLaC"...) - data = append(data, blockHeader(false, 0, 34)...) // STREAMINFO - data = append(data, make([]byte, 34)...) - picture := make([]byte, 4096) // stands in for embedded cover art - data = append(data, blockHeader(false, 6, len(picture))...) - data = append(data, picture...) - data = append(data, blockHeader(true, 4, len(vorbis))...) - data = append(data, vorbis...) - - if err := os.WriteFile(path, data, 0600); err != nil { - t.Fatal(err) - } -} - -func TestISRCIndexCoversNonFlacFormats(t *testing.T) { - dir := t.TempDir() - flacPath := filepath.Join(dir, "a.flac") - mp3Path := filepath.Join(dir, "b.mp3") - writeTestFlacWithISRC(t, flacPath, "USAA00000011") - mp3Data := buildID3v23Tag( - id3TextFrame("TIT2", "Song"), - id3TextFrame("TSRC", "usbb00000022"), - ) - if err := os.WriteFile(mp3Path, mp3Data, 0600); err != nil { - t.Fatal(err) - } - // Unsupported/untagged formats must stay invisible to the index. - if err := os.WriteFile(filepath.Join(dir, "c.wav"), []byte("RIFF"), 0600); err != nil { - t.Fatal(err) - } - defer InvalidateISRCCache(dir) - - if got := readFileISRC(mp3Path); got != "usbb00000022" { - t.Fatalf("readFileISRC mp3 = %q", got) - } - if got := readFileISRC(filepath.Join(dir, "c.wav")); got != "" { - t.Fatalf("readFileISRC wav = %q", got) - } - - idx := buildISRCIndex(dir) - if path, ok := idx.lookup("USAA00000011"); !ok || path != flacPath { - t.Fatalf("flac lookup = %q/%v", path, ok) - } - if path, ok := idx.lookup("USBB00000022"); !ok || path != mp3Path { - t.Fatalf("mp3 lookup = %q/%v", path, ok) - } -} - -func TestISRCIndexIncrementalRebuild(t *testing.T) { - dir := t.TempDir() - trackA := filepath.Join(dir, "a.flac") - trackB := filepath.Join(dir, "b.flac") - writeTestFlacWithISRC(t, trackA, "USAA00000001") - writeTestFlacWithISRC(t, trackB, "USBB00000002") - defer InvalidateISRCCache(dir) - - if got := readFlacISRC(trackA); got != "USAA00000001" { - t.Fatalf("readFlacISRC = %q", got) - } - if got := readFlacISRC(filepath.Join(dir, "missing.flac")); got != "" { - t.Fatalf("expected empty ISRC for missing file, got %q", got) - } - - idx := buildISRCIndex(dir) - if path, ok := idx.lookup("usaa00000001"); !ok || path != trackA { - t.Fatalf("lookup A = %q/%v", path, ok) - } - if path, ok := idx.lookup("USBB00000002"); !ok || path != trackB { - t.Fatalf("lookup B = %q/%v", path, ok) - } - - // Change one file's tag (and mtime); an incremental rebuild must pick up - // the change while adopting the untouched file from the cache. - writeTestFlacWithISRC(t, trackB, "USBB00000099") - future := time.Now().Add(2 * time.Second) - if err := os.Chtimes(trackB, future, future); err != nil { - t.Fatal(err) - } - rebuilt := buildISRCIndex(dir) - if _, ok := rebuilt.lookup("USBB00000002"); ok { - t.Fatal("expected stale ISRC to disappear after rebuild") - } - if path, ok := rebuilt.lookup("USBB00000099"); !ok || path != trackB { - t.Fatalf("rebuilt lookup = %q/%v", path, ok) - } - if path, ok := rebuilt.lookup("USAA00000001"); !ok || path != trackA { - t.Fatalf("cached entry lost on rebuild: %q/%v", path, ok) - } - - // Add() keeps the index fresh so the TTL never forces a rebuild while - // downloads are actively maintaining it. - rebuilt.buildTime.Store(time.Now().Add(-isrcIndexTTL - time.Minute).UnixNano()) - if rebuilt.isFresh() { - t.Fatal("expected stale index") - } - rebuilt.Add("USCC00000003", trackA) - if !rebuilt.isFresh() { - t.Fatal("expected Add to refresh the index timestamp") - } -} - -func TestDuplicateIndexAndParallelExistence(t *testing.T) { - dir := t.TempDir() - filePath := filepath.Join(dir, "song.flac") - if err := os.WriteFile(filePath, []byte("audio"), 0600); err != nil { - t.Fatal(err) - } - - idx := &ISRCIndex{index: map[string]string{}, outputDir: dir} - idx.buildTime.Store(time.Now().UnixNano()) - idx.Add("usrc17607839", filePath) - if got, ok := idx.lookup("USRC17607839"); !ok || got != filePath { - t.Fatalf("lookup = %q/%v", got, ok) - } - if got, err := idx.Lookup("usrc17607839"); err != nil || got != filePath { - t.Fatalf("Lookup = %q/%v", got, err) - } - idx.remove("usrc17607839") - if _, ok := idx.lookup("usrc17607839"); ok { - t.Fatal("expected removed ISRC") - } - - isrcIndexCacheMu.Lock() - isrcIndexCache[dir] = idx - isrcIndexCacheMu.Unlock() - defer InvalidateISRCCache(dir) - - AddToISRCIndex(dir, "USRC17607839", filePath) - if found, err := CheckISRCExists(dir, "USRC17607839"); err != nil || found != filePath { - t.Fatalf("CheckISRCExists = %q/%v", found, err) - } - if !CheckFileExists(filePath) || CheckFileExists(dir) || CheckFileExists(filepath.Join(dir, "missing.flac")) { - t.Fatal("unexpected file existence result") - } - - tracksJSON := `[{"isrc":"USRC17607839","track_name":"Song","artist_name":"Artist"},{"isrc":"MISSING","track_name":"Other","artist_name":"Artist"}]` - resultJSON, err := CheckFilesExistParallel(dir, tracksJSON) - if err != nil { - t.Fatalf("CheckFilesExistParallel: %v", err) - } - var results []FileExistenceResult - if err := json.Unmarshal([]byte(resultJSON), &results); err != nil { - t.Fatalf("decode results: %v", err) - } - if !results[0].Exists || results[0].FilePath != filePath || results[1].Exists { - t.Fatalf("results = %#v", results) - } - if _, err := CheckFilesExistParallel(dir, `not-json`); err == nil { - t.Fatal("expected invalid json error") - } - if err := PreBuildISRCIndex(""); err == nil { - t.Fatal("expected empty dir error") - } -} diff --git a/go_backend/cue_parser.go b/go_backend/cue_parser.go deleted file mode 100644 index 47209f6c..00000000 --- a/go_backend/cue_parser.go +++ /dev/null @@ -1,565 +0,0 @@ -package gobackend - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" -) - -type CueSheet struct { - Performer string `json:"performer"` - Title string `json:"title"` - FileName string `json:"file_name"` - FileType string `json:"file_type"` // WAVE, FLAC, MP3, AIFF, etc. - Genre string `json:"genre,omitempty"` - Date string `json:"date,omitempty"` - Comment string `json:"comment,omitempty"` - Composer string `json:"composer,omitempty"` - Tracks []CueTrack `json:"tracks"` -} - -type CueTrack struct { - Number int `json:"number"` - Title string `json:"title"` - Performer string `json:"performer"` - ISRC string `json:"isrc,omitempty"` - Composer string `json:"composer,omitempty"` - StartTime float64 `json:"start_time"` // INDEX 01 in seconds - PreGap float64 `json:"pre_gap"` // INDEX 00 in seconds (or -1 if not present) -} - -type CueSplitInfo struct { - CuePath string `json:"cue_path"` - AudioPath string `json:"audio_path"` - Album string `json:"album"` - Artist string `json:"artist"` - Genre string `json:"genre,omitempty"` - Date string `json:"date,omitempty"` - Tracks []CueSplitTrack `json:"tracks"` -} - -type CueSplitTrack struct { - Number int `json:"number"` - Title string `json:"title"` - Artist string `json:"artist"` - ISRC string `json:"isrc,omitempty"` - Composer string `json:"composer,omitempty"` - StartSec float64 `json:"start_sec"` - EndSec float64 `json:"end_sec"` // -1 means until end of file -} - -var ( - reRemCommand = regexp.MustCompile(`^REM\s+(\S+)\s+(.+)$`) - reQuoted = regexp.MustCompile(`"([^"]*)"`) -) - -func ParseCueFile(cuePath string) (*CueSheet, error) { - f, err := os.Open(cuePath) - if err != nil { - return nil, fmt.Errorf("failed to open cue file: %w", err) - } - defer f.Close() - - sheet := &CueSheet{} - var currentTrack *CueTrack - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - - if strings.HasPrefix(line, "\xef\xbb\xbf") { - line = strings.TrimPrefix(line, "\xef\xbb\xbf") - line = strings.TrimSpace(line) - } - - upper := strings.ToUpper(line) - - if strings.HasPrefix(upper, "REM ") { - matches := reRemCommand.FindStringSubmatch(line) - if len(matches) == 3 { - key := strings.ToUpper(matches[1]) - value := unquoteCue(matches[2]) - switch key { - case "GENRE": - sheet.Genre = value - case "DATE": - sheet.Date = value - case "COMMENT": - sheet.Comment = value - case "COMPOSER": - if currentTrack != nil { - currentTrack.Composer = value - } else { - sheet.Composer = value - } - } - } - continue - } - - if strings.HasPrefix(upper, "PERFORMER ") { - value := unquoteCue(line[len("PERFORMER "):]) - if currentTrack != nil { - currentTrack.Performer = value - } else { - sheet.Performer = value - } - continue - } - - if strings.HasPrefix(upper, "TITLE ") { - value := unquoteCue(line[len("TITLE "):]) - if currentTrack != nil { - currentTrack.Title = value - } else { - sheet.Title = value - } - continue - } - - if strings.HasPrefix(upper, "FILE ") { - rest := line[len("FILE "):] - fname, ftype := parseCueFileLine(rest) - sheet.FileName = fname - sheet.FileType = ftype - continue - } - - if strings.HasPrefix(upper, "TRACK ") { - if currentTrack != nil { - sheet.Tracks = append(sheet.Tracks, *currentTrack) - } - - parts := strings.Fields(line) - trackNum := 0 - if len(parts) >= 2 { - trackNum, _ = strconv.Atoi(parts[1]) - } - - currentTrack = &CueTrack{ - Number: trackNum, - PreGap: -1, - } - continue - } - - if strings.HasPrefix(upper, "INDEX ") && currentTrack != nil { - parts := strings.Fields(line) - if len(parts) >= 3 { - indexNum, _ := strconv.Atoi(parts[1]) - timeSec := parseCueTimestamp(parts[2]) - switch indexNum { - case 0: - currentTrack.PreGap = timeSec - case 1: - currentTrack.StartTime = timeSec - } - } - continue - } - - if strings.HasPrefix(upper, "ISRC ") && currentTrack != nil { - currentTrack.ISRC = strings.TrimSpace(line[len("ISRC "):]) - continue - } - - if strings.HasPrefix(upper, "SONGWRITER ") { - value := unquoteCue(line[len("SONGWRITER "):]) - if currentTrack != nil { - currentTrack.Composer = value - } else { - sheet.Composer = value - } - continue - } - } - - if currentTrack != nil { - sheet.Tracks = append(sheet.Tracks, *currentTrack) - } - - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading cue file: %w", err) - } - - if len(sheet.Tracks) == 0 { - return nil, fmt.Errorf("no tracks found in cue file") - } - - return sheet, nil -} - -func parseCueTimestamp(ts string) float64 { - parts := strings.Split(ts, ":") - if len(parts) != 3 { - return 0 - } - - minutes, _ := strconv.Atoi(parts[0]) - seconds, _ := strconv.Atoi(parts[1]) - frames, _ := strconv.Atoi(parts[2]) - - return float64(minutes)*60 + float64(seconds) + float64(frames)/75.0 -} - -func formatCueTimestamp(seconds float64) string { - if seconds < 0 { - return "0" - } - hours := int(seconds) / 3600 - mins := (int(seconds) % 3600) / 60 - secs := seconds - float64(hours*3600) - float64(mins*60) - return fmt.Sprintf("%02d:%02d:%06.3f", hours, mins, secs) -} - -func unquoteCue(s string) string { - s = strings.TrimSpace(s) - if matches := reQuoted.FindStringSubmatch(s); len(matches) == 2 { - return matches[1] - } - return s -} - -func parseCueFileLine(rest string) (string, string) { - rest = strings.TrimSpace(rest) - - var filename, ftype string - - if strings.HasPrefix(rest, "\"") { - endQuote := strings.Index(rest[1:], "\"") - if endQuote >= 0 { - filename = rest[1 : endQuote+1] - remaining := strings.TrimSpace(rest[endQuote+2:]) - ftype = remaining - } else { - filename = rest - } - } else { - parts := strings.Fields(rest) - if len(parts) >= 2 { - ftype = parts[len(parts)-1] - filename = strings.Join(parts[:len(parts)-1], " ") - } else if len(parts) == 1 { - filename = parts[0] - } - } - - return filename, strings.TrimSpace(ftype) -} - -func ResolveCueAudioPath(cuePath string, cueFileName string) string { - cueDir := filepath.Dir(cuePath) - - candidate := filepath.Join(cueDir, cueFileName) - if _, err := os.Stat(candidate); err == nil { - return candidate - } - - baseName := strings.TrimSuffix(cueFileName, filepath.Ext(cueFileName)) - commonExts := []string{".flac", ".wav", ".aiff", ".aif", ".ape", ".mp3", ".ogg", ".wv", ".m4a"} - for _, ext := range commonExts { - candidate = filepath.Join(cueDir, baseName+ext) - if _, err := os.Stat(candidate); err == nil { - return candidate - } - candidate = filepath.Join(cueDir, baseName+strings.ToUpper(ext)) - if _, err := os.Stat(candidate); err == nil { - return candidate - } - } - - cueBase := strings.TrimSuffix(filepath.Base(cuePath), filepath.Ext(cuePath)) - for _, ext := range commonExts { - candidate = filepath.Join(cueDir, cueBase+ext) - if _, err := os.Stat(candidate); err == nil { - return candidate - } - } - - entries, err := os.ReadDir(cueDir) - if err == nil { - audioExts := map[string]bool{ - ".flac": true, ".wav": true, ".ape": true, ".mp3": true, - ".ogg": true, ".wv": true, ".m4a": true, ".aiff": true, - } - var audioFiles []string - for _, entry := range entries { - if entry.IsDir() { - continue - } - ext := strings.ToLower(filepath.Ext(entry.Name())) - if audioExts[ext] { - audioFiles = append(audioFiles, filepath.Join(cueDir, entry.Name())) - } - } - if len(audioFiles) == 1 { - return audioFiles[0] - } - } - - return "" -} - -func BuildCueSplitInfo(cuePath string, sheet *CueSheet, audioDir string) (*CueSplitInfo, error) { - resolveDir := cuePath - if audioDir != "" { - resolveDir = filepath.Join(audioDir, filepath.Base(cuePath)) - } - audioPath := ResolveCueAudioPath(resolveDir, sheet.FileName) - if audioPath == "" { - return nil, fmt.Errorf("audio file not found for cue sheet: %s (referenced: %s)", cuePath, sheet.FileName) - } - - info := &CueSplitInfo{ - CuePath: cuePath, - AudioPath: audioPath, - Album: sheet.Title, - Artist: sheet.Performer, - Genre: sheet.Genre, - Date: sheet.Date, - } - - for i, track := range sheet.Tracks { - performer := track.Performer - if performer == "" { - performer = sheet.Performer - } - - composer := track.Composer - if composer == "" { - composer = sheet.Composer - } - - endSec := float64(-1) - if i+1 < len(sheet.Tracks) { - nextTrack := sheet.Tracks[i+1] - if nextTrack.PreGap >= 0 { - endSec = nextTrack.PreGap - } else { - endSec = nextTrack.StartTime - } - } - - info.Tracks = append(info.Tracks, CueSplitTrack{ - Number: track.Number, - Title: track.Title, - Artist: performer, - ISRC: track.ISRC, - Composer: composer, - StartSec: track.StartTime, - EndSec: endSec, - }) - } - - return info, nil -} - -func ParseCueFileJSON(cuePath string, audioDir string) (string, error) { - sheet, err := ParseCueFile(cuePath) - if err != nil { - return "", fmt.Errorf("failed to parse cue file: %w", err) - } - - info, err := BuildCueSplitInfo(cuePath, sheet, audioDir) - if err != nil { - return "", err - } - - jsonBytes, err := json.Marshal(info) - if err != nil { - return "", fmt.Errorf("failed to marshal cue split info: %w", err) - } - - return string(jsonBytes), nil -} - -func ScanCueFileForLibrary(cuePath string, scanTime string) ([]LibraryScanResult, error) { - sheet, err := ParseCueFile(cuePath) - if err != nil { - return nil, err - } - audioPath, err := resolveCueAudioPathForLibrary(cuePath, sheet, "") - if err != nil { - return nil, err - } - return scanCueSheetForLibrary(cuePath, sheet, audioPath, "", 0, "", scanTime) -} - -func ScanCueFileForLibraryExt(cuePath, audioDir, virtualPathPrefix string, fileModTime int64, scanTime string) ([]LibraryScanResult, error) { - return ScanCueFileForLibraryExtWithCoverCacheKey( - cuePath, - audioDir, - virtualPathPrefix, - fileModTime, - "", - scanTime, - ) -} - -func ScanCueFileForLibraryExtWithCoverCacheKey(cuePath, audioDir, virtualPathPrefix string, fileModTime int64, coverCacheKey, scanTime string) ([]LibraryScanResult, error) { - sheet, err := ParseCueFile(cuePath) - if err != nil { - return nil, err - } - audioPath, err := resolveCueAudioPathForLibrary(cuePath, sheet, audioDir) - if err != nil { - return nil, err - } - return scanCueSheetForLibrary( - cuePath, - sheet, - audioPath, - virtualPathPrefix, - fileModTime, - coverCacheKey, - scanTime, - ) -} - -func resolveCueAudioPathForLibrary(cuePath string, sheet *CueSheet, audioDir string) (string, error) { - if sheet == nil { - return "", fmt.Errorf("cue sheet is nil for %s", cuePath) - } - resolveBase := cuePath - if audioDir != "" { - resolveBase = filepath.Join(audioDir, filepath.Base(cuePath)) - } - audioPath := ResolveCueAudioPath(resolveBase, sheet.FileName) - if audioPath == "" { - return "", fmt.Errorf("audio file not found for cue: %s (referenced: %s)", cuePath, sheet.FileName) - } - return audioPath, nil -} - -func scanCueSheetForLibrary(cuePath string, sheet *CueSheet, audioPath, virtualPathPrefix string, fileModTime int64, coverCacheKey, scanTime string) ([]LibraryScanResult, error) { - if sheet == nil { - return nil, fmt.Errorf("cue sheet is nil for %s", cuePath) - } - - var bitDepth, sampleRate int - var totalDurationSec float64 - audioExt := strings.ToLower(filepath.Ext(audioPath)) - switch audioExt { - case ".flac": - quality, qErr := GetAudioQuality(audioPath) - if qErr == nil { - bitDepth = quality.BitDepth - sampleRate = quality.SampleRate - if quality.SampleRate > 0 && quality.TotalSamples > 0 { - totalDurationSec = float64(quality.TotalSamples) / float64(quality.SampleRate) - } - } - case ".mp3": - quality, qErr := GetMP3Quality(audioPath) - if qErr == nil { - sampleRate = quality.SampleRate - totalDurationSec = float64(quality.Duration) - } - } - - var coverPath string - libraryCoverCacheMu.RLock() - coverCacheDir := libraryCoverCacheDir - libraryCoverCacheMu.RUnlock() - if coverCacheDir != "" { - cp, err := SaveCoverToCacheWithHintAndKey( - audioPath, - "", - coverCacheDir, - coverCacheKey, - ) - if err == nil && cp != "" { - coverPath = cp - } - } - - pathBase := cuePath - if virtualPathPrefix != "" { - pathBase = virtualPathPrefix - } - - modTime := fileModTime - if modTime <= 0 { - if info, err := os.Stat(cuePath); err == nil { - modTime = info.ModTime().UnixMilli() - } - } - - var results []LibraryScanResult - for i, track := range sheet.Tracks { - performer := track.Performer - if performer == "" { - performer = sheet.Performer - } - if performer == "" { - performer = "Unknown Artist" - } - - title := track.Title - if title == "" { - title = fmt.Sprintf("Track %02d", track.Number) - } - - album := sheet.Title - if album == "" { - album = "Unknown Album" - } - - composer := track.Composer - if composer == "" { - composer = sheet.Composer - } - - var duration int - if i+1 < len(sheet.Tracks) { - nextStart := sheet.Tracks[i+1].StartTime - if sheet.Tracks[i+1].PreGap >= 0 { - nextStart = sheet.Tracks[i+1].PreGap - } - duration = int(nextStart - track.StartTime) - } else if totalDurationSec > 0 { - duration = int(totalDurationSec - track.StartTime) - } - - id := generateLibraryID(fmt.Sprintf("%s#track%d", pathBase, track.Number)) - - virtualFilePath := fmt.Sprintf("%s#track%02d", pathBase, track.Number) - - result := LibraryScanResult{ - ID: id, - TrackName: title, - ArtistName: performer, - AlbumName: album, - AlbumArtist: sheet.Performer, - FilePath: virtualFilePath, - CoverPath: coverPath, - ScannedAt: scanTime, - ISRC: track.ISRC, - TrackNumber: track.Number, - TotalTracks: len(sheet.Tracks), - DiscNumber: 1, - TotalDiscs: 1, - Duration: duration, - ReleaseDate: sheet.Date, - BitDepth: bitDepth, - SampleRate: sampleRate, - Genre: sheet.Genre, - Composer: composer, - Format: "cue+" + strings.TrimPrefix(audioExt, "."), - } - - result.FileModTime = modTime - - results = append(results, result) - } - - return results, nil -} diff --git a/go_backend/deezer.go b/go_backend/deezer.go deleted file mode 100644 index be8a54fb..00000000 --- a/go_backend/deezer.go +++ /dev/null @@ -1,702 +0,0 @@ -package gobackend - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "sync" - "time" - - "golang.org/x/sync/singleflight" -) - -const ( - deezerBaseURL = "https://api.deezer.com/2.0" - deezerSearchURL = deezerBaseURL + "/search" - deezerTrackURL = deezerBaseURL + "/track/%s" - deezerAlbumURL = deezerBaseURL + "/album/%s" - deezerArtistURL = deezerBaseURL + "/artist/%s" - deezerPlaylistURL = deezerBaseURL + "/playlist/%s" - - deezerCacheTTL = 10 * time.Minute - - deezerMaxParallelISRC = 10 - - // Deezer API timeout and retry configuration for mobile networks - deezerAPITimeoutMobile = 25 * time.Second - deezerMaxRetries = 2 - deezerRetryDelay = 500 * time.Millisecond - - deezerMaxSearchCacheEntries = 300 - deezerMaxAlbumCacheEntries = 200 - deezerMaxArtistCacheEntries = 200 - deezerMaxISRCCacheEntries = 4000 - deezerCacheCleanupInterval = 5 * time.Minute -) - -type DeezerClient struct { - httpClient *http.Client - searchCache map[string]*cacheEntry - albumCache map[string]*cacheEntry - artistCache map[string]*cacheEntry - isrcCache map[string]string - cacheMu sync.RWMutex - metadataFlight singleflight.Group - lastCacheCleanup time.Time - cacheCleanupInterval time.Duration -} - -var ( - deezerClient *DeezerClient - deezerClientOnce sync.Once -) - -func GetDeezerClient() *DeezerClient { - deezerClientOnce.Do(func() { - deezerClient = &DeezerClient{ - httpClient: NewMetadataHTTPClient(deezerAPITimeoutMobile), - searchCache: make(map[string]*cacheEntry), - albumCache: make(map[string]*cacheEntry), - artistCache: make(map[string]*cacheEntry), - isrcCache: make(map[string]string), - cacheCleanupInterval: deezerCacheCleanupInterval, - } - }) - return deezerClient -} - -func (c *DeezerClient) SearchAll(ctx context.Context, query string, trackLimit, artistLimit int, filter string) (*SearchAllResult, error) { - GoLog("[Deezer] SearchAll: query=%q, trackLimit=%d, artistLimit=%d, filter=%q\n", query, trackLimit, artistLimit, filter) - - albumLimit := 5 - playlistLimit := 5 - - if filter != "" { - switch filter { - case "track": - trackLimit = 50 - artistLimit = 0 - albumLimit = 0 - playlistLimit = 0 - case "artist": - trackLimit = 0 - artistLimit = 20 - albumLimit = 0 - playlistLimit = 0 - case "album": - trackLimit = 0 - artistLimit = 0 - albumLimit = 20 - playlistLimit = 0 - case "playlist": - trackLimit = 0 - artistLimit = 0 - albumLimit = 0 - playlistLimit = 20 - } - } - - cacheKey := fmt.Sprintf("deezer:all:%s:%d:%d:%d:%d:%s", query, trackLimit, artistLimit, albumLimit, playlistLimit, filter) - - c.cacheMu.RLock() - if entry, ok := c.searchCache[cacheKey]; ok && !entry.isExpired() { - c.cacheMu.RUnlock() - GoLog("[Deezer] SearchAll: returning cached result\n") - return entry.data.(*SearchAllResult), nil - } - c.cacheMu.RUnlock() - - result := &SearchAllResult{ - Tracks: make([]TrackMetadata, 0, trackLimit), - Artists: make([]SearchArtistResult, 0, artistLimit), - Albums: make([]SearchAlbumResult, 0, albumLimit), - Playlists: make([]SearchPlaylistResult, 0, playlistLimit), - } - - if trackLimit > 0 { - trackURL := fmt.Sprintf("%s/track?q=%s&limit=%d", deezerSearchURL, url.QueryEscape(query), trackLimit) - GoLog("[Deezer] Fetching tracks from: %s\n", trackURL) - - var trackResp struct { - Data []deezerTrack `json:"data"` - Error *struct { - Type string `json:"type"` - Message string `json:"message"` - Code int `json:"code"` - } `json:"error"` - } - if err := c.getJSON(ctx, trackURL, &trackResp); err != nil { - GoLog("[Deezer] Track search failed: %v\n", err) - return nil, fmt.Errorf("deezer track search failed: %w", err) - } - - if trackResp.Error != nil { - GoLog("[Deezer] API error: type=%s, code=%d, message=%s\n", trackResp.Error.Type, trackResp.Error.Code, trackResp.Error.Message) - return nil, fmt.Errorf("deezer API error: %s (code %d)", trackResp.Error.Message, trackResp.Error.Code) - } - - GoLog("[Deezer] Got %d tracks from API\n", len(trackResp.Data)) - - for _, track := range trackResp.Data { - result.Tracks = append(result.Tracks, c.convertTrack(track)) - } - } - - if artistLimit > 0 { - artistURL := fmt.Sprintf("%s/artist?q=%s&limit=%d", deezerSearchURL, url.QueryEscape(query), artistLimit) - GoLog("[Deezer] Fetching artists from: %s\n", artistURL) - - var artistResp struct { - Data []deezerArtist `json:"data"` - Error *struct { - Type string `json:"type"` - Message string `json:"message"` - Code int `json:"code"` - } `json:"error"` - } - if err := c.getJSON(ctx, artistURL, &artistResp); err == nil { - if artistResp.Error != nil { - GoLog("[Deezer] Artist API error: type=%s, code=%d, message=%s\n", artistResp.Error.Type, artistResp.Error.Code, artistResp.Error.Message) - } else { - GoLog("[Deezer] Got %d artists from API\n", len(artistResp.Data)) - for _, artist := range artistResp.Data { - result.Artists = append(result.Artists, SearchArtistResult{ - ID: fmt.Sprintf("deezer:%d", artist.ID), - Name: artist.Name, - Images: c.getBestArtistImage(artist), - Followers: artist.NbFan, - Popularity: 0, - }) - } - } - } else { - GoLog("[Deezer] Artist search failed: %v\n", err) - } - } - - if albumLimit > 0 { - albumURL := fmt.Sprintf("%s/album?q=%s&limit=%d", deezerSearchURL, url.QueryEscape(query), albumLimit) - GoLog("[Deezer] Fetching albums from: %s\n", albumURL) - - var albumResp struct { - Data []struct { - ID int64 `json:"id"` - Title string `json:"title"` - Cover string `json:"cover"` - CoverMedium string `json:"cover_medium"` - CoverBig string `json:"cover_big"` - CoverXL string `json:"cover_xl"` - NbTracks int `json:"nb_tracks"` - ReleaseDate string `json:"release_date"` - RecordType string `json:"record_type"` - Artist deezerArtist `json:"artist"` - } `json:"data"` - Error *struct { - Type string `json:"type"` - Message string `json:"message"` - Code int `json:"code"` - } `json:"error"` - } - if err := c.getJSON(ctx, albumURL, &albumResp); err == nil { - if albumResp.Error != nil { - GoLog("[Deezer] Album API error: type=%s, code=%d, message=%s\n", albumResp.Error.Type, albumResp.Error.Code, albumResp.Error.Message) - } else { - GoLog("[Deezer] Got %d albums from API\n", len(albumResp.Data)) - for _, album := range albumResp.Data { - coverURL := album.CoverXL - if coverURL == "" { - coverURL = album.CoverBig - } - if coverURL == "" { - coverURL = album.CoverMedium - } - if coverURL == "" { - coverURL = album.Cover - } - - albumType := album.RecordType - if albumType == "compile" { - albumType = "compilation" - } - - result.Albums = append(result.Albums, SearchAlbumResult{ - ID: fmt.Sprintf("deezer:%d", album.ID), - Name: album.Title, - Artists: album.Artist.Name, - Images: coverURL, - ReleaseDate: album.ReleaseDate, - TotalTracks: album.NbTracks, - AlbumType: albumType, - }) - } - } - } else { - GoLog("[Deezer] Album search failed: %v\n", err) - } - } - - if playlistLimit > 0 { - playlistURL := fmt.Sprintf("%s/playlist?q=%s&limit=%d", deezerSearchURL, url.QueryEscape(query), playlistLimit) - GoLog("[Deezer] Fetching playlists from: %s\n", playlistURL) - - var playlistResp struct { - Data []struct { - ID int64 `json:"id"` - Title string `json:"title"` - Picture string `json:"picture"` - PictureMedium string `json:"picture_medium"` - PictureBig string `json:"picture_big"` - PictureXL string `json:"picture_xl"` - NbTracks int `json:"nb_tracks"` - User struct { - Name string `json:"name"` - } `json:"user"` - } `json:"data"` - Error *struct { - Type string `json:"type"` - Message string `json:"message"` - Code int `json:"code"` - } `json:"error"` - } - if err := c.getJSON(ctx, playlistURL, &playlistResp); err == nil { - if playlistResp.Error != nil { - GoLog("[Deezer] Playlist API error: type=%s, code=%d, message=%s\n", playlistResp.Error.Type, playlistResp.Error.Code, playlistResp.Error.Message) - } else { - GoLog("[Deezer] Got %d playlists from API\n", len(playlistResp.Data)) - for _, playlist := range playlistResp.Data { - pictureURL := playlist.PictureXL - if pictureURL == "" { - pictureURL = playlist.PictureBig - } - if pictureURL == "" { - pictureURL = playlist.PictureMedium - } - if pictureURL == "" { - pictureURL = playlist.Picture - } - - result.Playlists = append(result.Playlists, SearchPlaylistResult{ - ID: fmt.Sprintf("deezer:%d", playlist.ID), - Name: playlist.Title, - Owner: playlist.User.Name, - Images: pictureURL, - TotalTracks: playlist.NbTracks, - }) - } - } - } else { - GoLog("[Deezer] Playlist search failed: %v\n", err) - } - } - - GoLog("[Deezer] SearchAll complete: %d tracks, %d artists, %d albums, %d playlists\n", len(result.Tracks), len(result.Artists), len(result.Albums), len(result.Playlists)) - - c.cacheMu.Lock() - now := time.Now() - c.searchCache[cacheKey] = &cacheEntry{ - data: result, - expiresAt: now.Add(deezerCacheTTL), - } - c.maybeCleanupCachesLocked(now) - c.cacheMu.Unlock() - - return result, nil -} - -func (c *DeezerClient) GetTrack(ctx context.Context, trackID string) (*TrackResponse, error) { - trackURL := fmt.Sprintf(deezerTrackURL, trackID) - - var track deezerTrack - if err := c.getJSON(ctx, trackURL, &track); err != nil { - return nil, err - } - - return &TrackResponse{ - Track: c.convertTrack(track), - }, nil -} - -func (c *DeezerClient) SearchByISRC(ctx context.Context, isrc string) (*TrackMetadata, error) { - directURL := fmt.Sprintf("%s/track/isrc:%s", deezerBaseURL, isrc) - - var track deezerTrack - if err := c.getJSON(ctx, directURL, &track); err != nil { - searchURL := fmt.Sprintf("%s/track?q=isrc:%s&limit=1", deezerSearchURL, isrc) - var resp struct { - Data []deezerTrack `json:"data"` - } - if err := c.getJSON(ctx, searchURL, &resp); err != nil { - return nil, err - } - if len(resp.Data) == 0 { - return nil, fmt.Errorf("no track found for ISRC: %s", isrc) - } - result := c.convertTrack(resp.Data[0]) - return &result, nil - } - - if track.ID == 0 { - return nil, fmt.Errorf("no track found for ISRC: %s", isrc) - } - - result := c.convertTrack(track) - return &result, nil -} - -func (c *DeezerClient) fetchFullTrack(ctx context.Context, trackID string) (*deezerTrack, error) { - trackURL := fmt.Sprintf(deezerTrackURL, trackID) - var track deezerTrack - if err := c.getJSON(ctx, trackURL, &track); err != nil { - return nil, err - } - return &track, nil -} - -func (c *DeezerClient) fetchISRCsParallel(ctx context.Context, tracks []deezerTrack) map[string]string { - result := make(map[string]string, len(tracks)) - var resultMu sync.Mutex - - var tracksToFetch []deezerTrack - var directISRCs map[string]string - c.cacheMu.RLock() - for _, track := range tracks { - trackIDStr := fmt.Sprintf("%d", track.ID) - if track.ISRC != "" { - result[trackIDStr] = track.ISRC - if _, ok := c.isrcCache[trackIDStr]; !ok { - if directISRCs == nil { - directISRCs = make(map[string]string) - } - directISRCs[trackIDStr] = track.ISRC - } - continue - } - if isrc, ok := c.isrcCache[trackIDStr]; ok { - result[trackIDStr] = isrc - } else { - tracksToFetch = append(tracksToFetch, track) - } - } - c.cacheMu.RUnlock() - if len(directISRCs) > 0 { - c.cacheMu.Lock() - for trackIDStr, isrc := range directISRCs { - c.isrcCache[trackIDStr] = isrc - } - c.maybeCleanupCachesLocked(time.Now()) - c.cacheMu.Unlock() - } - - if len(tracksToFetch) == 0 { - return result - } - - sem := make(chan struct{}, deezerMaxParallelISRC) - var wg sync.WaitGroup - - for _, track := range tracksToFetch { - wg.Add(1) - go func(t deezerTrack) { - defer wg.Done() - - select { - case sem <- struct{}{}: - defer func() { <-sem }() - case <-ctx.Done(): - return - } - - trackIDStr := fmt.Sprintf("%d", t.ID) - fullTrack, err := c.fetchFullTrack(ctx, trackIDStr) - if err != nil || fullTrack == nil { - return - } - - resultMu.Lock() - result[trackIDStr] = fullTrack.ISRC - resultMu.Unlock() - - c.cacheMu.Lock() - c.isrcCache[trackIDStr] = fullTrack.ISRC - c.maybeCleanupCachesLocked(time.Now()) - c.cacheMu.Unlock() - }(track) - } - - wg.Wait() - return result -} - -func (c *DeezerClient) GetTrackISRC(ctx context.Context, trackID string) (string, error) { - c.cacheMu.RLock() - if isrc, ok := c.isrcCache[trackID]; ok { - c.cacheMu.RUnlock() - return isrc, nil - } - c.cacheMu.RUnlock() - - fullTrack, err := c.fetchFullTrack(ctx, trackID) - if err != nil { - return "", err - } - - c.cacheMu.Lock() - c.isrcCache[trackID] = fullTrack.ISRC - c.maybeCleanupCachesLocked(time.Now()) - c.cacheMu.Unlock() - - return fullTrack.ISRC, nil -} - -type AlbumExtendedMetadata struct { - Genre string - Label string - Copyright string -} - -func (c *DeezerClient) GetAlbumExtendedMetadata(ctx context.Context, albumID string) (*AlbumExtendedMetadata, error) { - if albumID == "" { - return nil, fmt.Errorf("empty album ID") - } - - cacheKey := fmt.Sprintf("album_meta:%s", albumID) - c.cacheMu.RLock() - if entry, ok := c.searchCache[cacheKey]; ok && !entry.isExpired() { - c.cacheMu.RUnlock() - return entry.data.(*AlbumExtendedMetadata), nil - } - c.cacheMu.RUnlock() - - value, err, _ := c.metadataFlight.Do(cacheKey, func() (any, error) { - // Re-check after joining the flight; another request may have filled - // the cache between the optimistic lookup above and Do acquiring it. - c.cacheMu.RLock() - if entry, ok := c.searchCache[cacheKey]; ok && !entry.isExpired() { - c.cacheMu.RUnlock() - return entry.data.(*AlbumExtendedMetadata), nil - } - c.cacheMu.RUnlock() - - albumURL := fmt.Sprintf(deezerAlbumURL, albumID) - var album deezerAlbumFull - if err := c.getJSON(ctx, albumURL, &album); err != nil { - return nil, fmt.Errorf("failed to fetch album: %w", err) - } - - var genres []string - for _, genre := range album.Genres.Data { - if genre.Name != "" { - genres = append(genres, genre.Name) - } - } - result := &AlbumExtendedMetadata{ - Genre: strings.Join(genres, ", "), - Label: album.Label, - Copyright: album.Copyright, - } - - c.cacheMu.Lock() - now := time.Now() - c.searchCache[cacheKey] = &cacheEntry{ - data: result, - expiresAt: now.Add(deezerCacheTTL), - } - c.maybeCleanupCachesLocked(now) - c.cacheMu.Unlock() - GoLog("[Deezer] Album metadata fetched - Genre: %s, Label: %s, Copyright: %s\n", result.Genre, result.Label, result.Copyright) - return result, nil - }) - if err != nil { - return nil, err - } - return value.(*AlbumExtendedMetadata), nil -} - -func (c *DeezerClient) GetTrackAlbumID(ctx context.Context, trackID string) (string, error) { - cacheKey := "track_album:" + trackID - c.cacheMu.RLock() - if entry, ok := c.searchCache[cacheKey]; ok && !entry.isExpired() { - c.cacheMu.RUnlock() - return entry.data.(string), nil - } - c.cacheMu.RUnlock() - - value, err, _ := c.metadataFlight.Do(cacheKey, func() (any, error) { - c.cacheMu.RLock() - if entry, ok := c.searchCache[cacheKey]; ok && !entry.isExpired() { - c.cacheMu.RUnlock() - return entry.data.(string), nil - } - c.cacheMu.RUnlock() - - trackURL := fmt.Sprintf(deezerTrackURL, trackID) - var track deezerTrack - if err := c.getJSON(ctx, trackURL, &track); err != nil { - return "", err - } - albumID := fmt.Sprintf("%d", track.Album.ID) - c.cacheMu.Lock() - now := time.Now() - c.searchCache[cacheKey] = &cacheEntry{ - data: albumID, - expiresAt: now.Add(deezerCacheTTL), - } - c.maybeCleanupCachesLocked(now) - c.cacheMu.Unlock() - return albumID, nil - }) - if err != nil { - return "", err - } - return value.(string), nil -} - -func (c *DeezerClient) GetExtendedMetadataByTrackID(ctx context.Context, trackID string) (*AlbumExtendedMetadata, error) { - albumID, err := c.GetTrackAlbumID(ctx, trackID) - if err != nil { - return nil, fmt.Errorf("failed to get album ID: %w", err) - } - - return c.GetAlbumExtendedMetadata(ctx, albumID) -} - -func (c *DeezerClient) GetExtendedMetadataByISRC(ctx context.Context, isrc string) (*AlbumExtendedMetadata, error) { - if isrc == "" { - return nil, fmt.Errorf("empty ISRC") - } - - track, err := c.SearchByISRC(ctx, isrc) - if err != nil { - return nil, fmt.Errorf("failed to find track by ISRC: %w", err) - } - - deezerID := strings.TrimPrefix(track.SpotifyID, "deezer:") - - if deezerID == "" { - return nil, fmt.Errorf("track found but no Deezer ID") - } - - return c.GetExtendedMetadataByTrackID(ctx, deezerID) -} - -func (c *DeezerClient) getJSON(ctx context.Context, endpoint string, dst any) error { - var lastErr error - - for attempt := 0; attempt <= deezerMaxRetries; attempt++ { - if attempt > 0 { - delay := deezerRetryDelay * time.Duration(1<<(attempt-1)) - var apiErr *deezerAPIError - if errors.As(lastErr, &apiErr) && apiErr.RetryAfter > 0 { - delay = apiErr.RetryAfter - } - GoLog("[Deezer] Retry %d/%d after %v...\n", attempt, deezerMaxRetries, delay) - time.Sleep(delay) - } - - err := c.doGetJSON(ctx, endpoint, dst) - if err == nil { - return nil - } - - lastErr = err - if !isDeezerRetryableError(err) { - return err - } - - GoLog("[Deezer] Attempt %d failed (retryable): %v\n", attempt+1, err) - } - - return fmt.Errorf("all %d attempts failed: %w", deezerMaxRetries+1, lastErr) -} - -type deezerAPIError struct { - StatusCode int - Body string - RetryAfter time.Duration -} - -func (e *deezerAPIError) Error() string { - return fmt.Sprintf("deezer API returned status %d: %s", e.StatusCode, e.Body) -} - -func isDeezerRetryableError(err error) bool { - if isConnectivityFailure(err) || errors.Is(err, io.ErrUnexpectedEOF) { - return true - } - var apiErr *deezerAPIError - if errors.As(err, &apiErr) { - return apiErr.StatusCode == http.StatusTooManyRequests || apiErr.StatusCode >= http.StatusInternalServerError - } - return false -} - -func (c *DeezerClient) doGetJSON(ctx context.Context, endpoint string, dst any) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return err - } - - req.Header.Set("Accept", "application/json") - // Without an explicit language Deezer localizes artist/genre names by - // the caller's IP geolocation (issue #480: Arabic metadata on an - // English device). Follow the app's display language instead. - req.Header.Set("Accept-Language", metadataAcceptLanguage()) - - resp, err := c.httpClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - - if resp.StatusCode != http.StatusOK { - return &deezerAPIError{StatusCode: resp.StatusCode, Body: string(body), RetryAfter: getRetryAfterDuration(resp)} - } - - return json.Unmarshal(body, dst) -} - -func parseDeezerURL(input string) (string, string, error) { - trimmed := strings.TrimSpace(input) - if trimmed == "" { - return "", "", fmt.Errorf("empty URL") - } - - parsed, err := url.Parse(trimmed) - if err != nil { - return "", "", err - } - - if parsed.Host != "www.deezer.com" && parsed.Host != "deezer.com" && parsed.Host != "deezer.page.link" { - return "", "", fmt.Errorf("not a Deezer URL") - } - - parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") - - if len(parts) > 0 && len(parts[0]) == 2 { - parts = parts[1:] - } - - if len(parts) < 2 { - return "", "", fmt.Errorf("invalid Deezer URL format") - } - - resourceType := parts[0] - resourceID := parts[1] - - switch resourceType { - case "track", "album", "artist", "playlist": - return resourceType, resourceID, nil - default: - return "", "", fmt.Errorf("unsupported Deezer resource type: %s", resourceType) - } -} diff --git a/go_backend/deezer_browse.go b/go_backend/deezer_browse.go deleted file mode 100644 index e7b05049..00000000 --- a/go_backend/deezer_browse.go +++ /dev/null @@ -1,386 +0,0 @@ -package gobackend - -import ( - "context" - "fmt" - "strings" - "sync" - "time" -) - -func (c *DeezerClient) GetAlbum(ctx context.Context, albumID string) (*AlbumResponsePayload, error) { - c.cacheMu.RLock() - if entry, ok := c.albumCache[albumID]; ok && !entry.isExpired() { - c.cacheMu.RUnlock() - return entry.data.(*AlbumResponsePayload), nil - } - c.cacheMu.RUnlock() - - albumURL := fmt.Sprintf(deezerAlbumURL, albumID) - - var album deezerAlbumFull - if err := c.getJSON(ctx, albumURL, &album); err != nil { - return nil, err - } - - albumImage := c.getBestAlbumImage(album) - artistName := album.Artist.Name - if len(album.Contributors) > 0 { - names := make([]string, len(album.Contributors)) - for i, a := range album.Contributors { - names[i] = a.Name - } - artistName = strings.Join(names, ", ") - } - - var genres []string - for _, g := range album.Genres.Data { - if g.Name != "" { - genres = append(genres, g.Name) - } - } - genreStr := strings.Join(genres, ", ") - - info := AlbumInfoMetadata{ - TotalTracks: album.NbTracks, - Name: album.Title, - ReleaseDate: album.ReleaseDate, - Artists: artistName, - ArtistId: fmt.Sprintf("deezer:%d", album.Artist.ID), - Images: albumImage, - Genre: genreStr, - Label: album.Label, - } - - allTracks := album.Tracks.Data - - if album.NbTracks > len(allTracks) { - GoLog("[Deezer] Album has %d tracks but only got %d, fetching remaining...", album.NbTracks, len(allTracks)) - - tracksURL := fmt.Sprintf("%s/tracks?limit=100&index=%d", fmt.Sprintf(deezerAlbumURL, albumID), len(allTracks)) - - for len(allTracks) < album.NbTracks { - var tracksResp struct { - Data []deezerTrack `json:"data"` - Next string `json:"next"` - } - - if err := c.getJSON(ctx, tracksURL, &tracksResp); err != nil { - GoLog("[Deezer] Warning: failed to fetch album tracks page: %v", err) - break - } - - if len(tracksResp.Data) == 0 { - break - } - - allTracks = append(allTracks, tracksResp.Data...) - - if tracksResp.Next == "" { - break - } - tracksURL = tracksResp.Next - } - - GoLog("[Deezer] Fetched total %d tracks for album", len(allTracks)) - } - - isrcMap := c.fetchISRCsParallel(ctx, allTracks) - totalDiscs := 0 - for _, track := range allTracks { - if track.DiskNumber > totalDiscs { - totalDiscs = track.DiskNumber - } - } - - tracks := make([]AlbumTrackMetadata, 0, len(allTracks)) - albumType := album.RecordType - if albumType == "compile" { - albumType = "compilation" - } - - for i, track := range allTracks { - trackIDStr := fmt.Sprintf("%d", track.ID) - isrc := isrcMap[trackIDStr] - - trackNum := track.TrackPosition - if trackNum == 0 { - trackNum = i + 1 - } - - tracks = append(tracks, AlbumTrackMetadata{ - SpotifyID: fmt.Sprintf("deezer:%d", track.ID), - Artists: deezerTrackArtistDisplay(track), - Name: track.Title, - AlbumName: album.Title, - AlbumArtist: artistName, - DurationMS: track.Duration * 1000, - Images: albumImage, - ReleaseDate: album.ReleaseDate, - TrackNumber: trackNum, - TotalTracks: album.NbTracks, - DiscNumber: track.DiskNumber, - TotalDiscs: totalDiscs, - ExternalURL: track.Link, - ISRC: isrc, - AlbumID: fmt.Sprintf("deezer:%d", album.ID), - AlbumType: albumType, - Explicit: deezerTrackIsExplicit(track), - }) - } - - result := &AlbumResponsePayload{ - AlbumInfo: info, - TrackList: tracks, - } - - c.cacheMu.Lock() - now := time.Now() - c.albumCache[albumID] = &cacheEntry{ - data: result, - expiresAt: now.Add(deezerCacheTTL), - } - c.maybeCleanupCachesLocked(now) - c.cacheMu.Unlock() - - return result, nil -} - -func (c *DeezerClient) GetArtist(ctx context.Context, artistID string) (*ArtistResponsePayload, error) { - c.cacheMu.RLock() - if entry, ok := c.artistCache[artistID]; ok && !entry.isExpired() { - c.cacheMu.RUnlock() - return entry.data.(*ArtistResponsePayload), nil - } - c.cacheMu.RUnlock() - - artistURL := fmt.Sprintf(deezerArtistURL, artistID) - var artist deezerArtistFull - if err := c.getJSON(ctx, artistURL, &artist); err != nil { - return nil, err - } - - artistInfo := ArtistInfoMetadata{ - ID: fmt.Sprintf("deezer:%d", artist.ID), - Name: artist.Name, - Images: c.getBestArtistImageFull(artist), - Followers: artist.NbFan, - Popularity: 0, - } - - albumsURL := fmt.Sprintf("%s/albums?limit=100", fmt.Sprintf(deezerArtistURL, artistID)) - var albumsResp struct { - Data []struct { - ID int64 `json:"id"` - Title string `json:"title"` - ReleaseDate string `json:"release_date"` - NbTracks int `json:"nb_tracks"` - Cover string `json:"cover"` - CoverMedium string `json:"cover_medium"` - CoverBig string `json:"cover_big"` - CoverXL string `json:"cover_xl"` - RecordType string `json:"record_type"` - } `json:"data"` - } - - albums := make([]ArtistAlbumMetadata, 0) - if err := c.getJSON(ctx, albumsURL, &albumsResp); err == nil { - for _, album := range albumsResp.Data { - albumType := album.RecordType - if albumType == "compile" { - albumType = "compilation" - } - - coverURL := album.CoverXL - if coverURL == "" { - coverURL = album.CoverBig - } - if coverURL == "" { - coverURL = album.CoverMedium - } - if coverURL == "" { - coverURL = album.Cover - } - - albums = append(albums, ArtistAlbumMetadata{ - ID: fmt.Sprintf("deezer:%d", album.ID), - Name: album.Title, - ReleaseDate: album.ReleaseDate, - TotalTracks: album.NbTracks, - Images: coverURL, - AlbumType: albumType, - Artists: artist.Name, - }) - } - - // The Deezer /artist/{id}/albums endpoint does not return nb_tracks. - // Fetch track counts in parallel from individual /album/{id} endpoints. - c.fetchAlbumTrackCounts(ctx, albums) - } - - result := &ArtistResponsePayload{ - ArtistInfo: artistInfo, - Albums: albums, - } - - c.cacheMu.Lock() - now := time.Now() - c.artistCache[artistID] = &cacheEntry{ - data: result, - expiresAt: now.Add(deezerCacheTTL), - } - c.maybeCleanupCachesLocked(now) - c.cacheMu.Unlock() - - return result, nil -} - -// fetchAlbumTrackCounts fetches nb_tracks for each album in parallel using -// individual /album/{id} calls, since the /artist/{id}/albums endpoint does -// not include this field. Albums whose track count is already known (non-zero) -// are skipped. -func (c *DeezerClient) fetchAlbumTrackCounts(ctx context.Context, albums []ArtistAlbumMetadata) { - type indexedID struct { - idx int - albumID string - } - var toFetch []indexedID - for i, a := range albums { - if a.TotalTracks == 0 { - rawID := strings.TrimPrefix(a.ID, "deezer:") - if rawID != "" { - toFetch = append(toFetch, indexedID{idx: i, albumID: rawID}) - } - } - } - if len(toFetch) == 0 { - return - } - - const maxParallel = 10 - sem := make(chan struct{}, maxParallel) - var mu sync.Mutex - var wg sync.WaitGroup - - for _, item := range toFetch { - wg.Add(1) - go func(it indexedID) { - defer wg.Done() - - select { - case sem <- struct{}{}: - defer func() { <-sem }() - case <-ctx.Done(): - return - } - - albumURL := fmt.Sprintf(deezerAlbumURL, it.albumID) - var resp struct { - NbTracks int `json:"nb_tracks"` - } - if err := c.getJSON(ctx, albumURL, &resp); err != nil { - return - } - - mu.Lock() - albums[it.idx].TotalTracks = resp.NbTracks - mu.Unlock() - }(item) - } - - wg.Wait() -} - -func (c *DeezerClient) GetPlaylist(ctx context.Context, playlistID string) (*PlaylistResponsePayload, error) { - playlistURL := fmt.Sprintf(deezerPlaylistURL, playlistID) - - var playlist deezerPlaylistFull - if err := c.getJSON(ctx, playlistURL, &playlist); err != nil { - return nil, err - } - - playlistImage := playlist.PictureXL - if playlistImage == "" { - playlistImage = playlist.PictureBig - } - if playlistImage == "" { - playlistImage = playlist.PictureMedium - } - - var info PlaylistInfoMetadata - info.Tracks.Total = playlist.NbTracks - info.Owner.DisplayName = playlist.Creator.Name - info.Owner.Name = playlist.Title - info.Owner.Images = playlistImage - - allTracks := playlist.Tracks.Data - - if playlist.NbTracks > len(allTracks) { - GoLog("[Deezer] Playlist has %d tracks but only got %d, fetching remaining...", playlist.NbTracks, len(allTracks)) - - tracksURL := fmt.Sprintf("%s/tracks?limit=100&index=%d", fmt.Sprintf(deezerPlaylistURL, playlistID), len(allTracks)) - - for len(allTracks) < playlist.NbTracks { - var tracksResp struct { - Data []deezerTrack `json:"data"` - Next string `json:"next"` - } - - if err := c.getJSON(ctx, tracksURL, &tracksResp); err != nil { - GoLog("[Deezer] Warning: failed to fetch playlist tracks page: %v", err) - break - } - - if len(tracksResp.Data) == 0 { - break - } - - allTracks = append(allTracks, tracksResp.Data...) - - if tracksResp.Next == "" { - break - } - tracksURL = tracksResp.Next - } - - GoLog("[Deezer] Fetched total %d tracks for playlist", len(allTracks)) - } - - isrcMap := c.fetchISRCsParallel(ctx, allTracks) - - tracks := make([]AlbumTrackMetadata, 0, len(allTracks)) - for _, track := range allTracks { - albumImage := track.Album.CoverXL - if albumImage == "" { - albumImage = track.Album.CoverBig - } - if albumImage == "" { - albumImage = track.Album.CoverMedium - } - - trackIDStr := fmt.Sprintf("%d", track.ID) - isrc := isrcMap[trackIDStr] - - tracks = append(tracks, AlbumTrackMetadata{ - SpotifyID: fmt.Sprintf("deezer:%d", track.ID), - Artists: deezerTrackArtistDisplay(track), - Name: track.Title, - AlbumName: track.Album.Title, - AlbumArtist: track.Artist.Name, - DurationMS: track.Duration * 1000, - Images: albumImage, - ReleaseDate: "", - TrackNumber: track.TrackPosition, - DiscNumber: track.DiskNumber, - ExternalURL: track.Link, - ISRC: isrc, - AlbumID: fmt.Sprintf("deezer:%d", track.Album.ID), - Explicit: deezerTrackIsExplicit(track), - }) - } - - return &PlaylistResponsePayload{ - PlaylistInfo: info, - TrackList: tracks, - }, nil -} diff --git a/go_backend/deezer_cache.go b/go_backend/deezer_cache.go deleted file mode 100644 index 9e60278b..00000000 --- a/go_backend/deezer_cache.go +++ /dev/null @@ -1,97 +0,0 @@ -package gobackend - -import "time" - -func (c *DeezerClient) pruneExpiredCacheEntriesLocked( - cache map[string]*cacheEntry, - now time.Time, -) { - for key, entry := range cache { - if entry == nil || now.After(entry.expiresAt) { - delete(cache, key) - } - } -} - -func (c *DeezerClient) trimCacheEntriesLocked( - cache map[string]*cacheEntry, - maxEntries int, -) { - if maxEntries <= 0 || len(cache) <= maxEntries { - return - } - - for len(cache) > maxEntries { - var oldestKey string - var oldestExpiry time.Time - first := true - for key, entry := range cache { - expiry := time.Time{} - if entry != nil { - expiry = entry.expiresAt - } - if first || expiry.Before(oldestExpiry) { - first = false - oldestKey = key - oldestExpiry = expiry - } - } - if oldestKey == "" { - return - } - delete(cache, oldestKey) - } -} - -func (c *DeezerClient) trimStringCacheEntriesLocked( - cache map[string]string, - maxEntries int, -) { - if maxEntries <= 0 || len(cache) <= maxEntries { - return - } - - toRemove := len(cache) - maxEntries - for key := range cache { - delete(cache, key) - toRemove-- - if toRemove <= 0 { - return - } - } -} - -func (c *DeezerClient) maybeCleanupCachesLocked(now time.Time) { - periodicCleanupDue := c.cacheCleanupInterval > 0 && - (c.lastCacheCleanup.IsZero() || - now.Sub(c.lastCacheCleanup) >= c.cacheCleanupInterval) - - if periodicCleanupDue { - c.pruneExpiredCacheEntriesLocked(c.searchCache, now) - c.pruneExpiredCacheEntriesLocked(c.albumCache, now) - c.pruneExpiredCacheEntriesLocked(c.artistCache, now) - c.lastCacheCleanup = now - } - - if len(c.searchCache) > deezerMaxSearchCacheEntries { - if !periodicCleanupDue { - c.pruneExpiredCacheEntriesLocked(c.searchCache, now) - } - c.trimCacheEntriesLocked(c.searchCache, deezerMaxSearchCacheEntries) - } - if len(c.albumCache) > deezerMaxAlbumCacheEntries { - if !periodicCleanupDue { - c.pruneExpiredCacheEntriesLocked(c.albumCache, now) - } - c.trimCacheEntriesLocked(c.albumCache, deezerMaxAlbumCacheEntries) - } - if len(c.artistCache) > deezerMaxArtistCacheEntries { - if !periodicCleanupDue { - c.pruneExpiredCacheEntriesLocked(c.artistCache, now) - } - c.trimCacheEntriesLocked(c.artistCache, deezerMaxArtistCacheEntries) - } - if len(c.isrcCache) > deezerMaxISRCCacheEntries { - c.trimStringCacheEntriesLocked(c.isrcCache, deezerMaxISRCCacheEntries) - } -} diff --git a/go_backend/deezer_models.go b/go_backend/deezer_models.go deleted file mode 100644 index b3c2ea9f..00000000 --- a/go_backend/deezer_models.go +++ /dev/null @@ -1,195 +0,0 @@ -package gobackend - -import ( - "fmt" - "strings" -) - -type deezerTrack struct { - ID int64 `json:"id"` - Title string `json:"title"` - Duration int `json:"duration"` - TrackPosition int `json:"track_position"` - DiskNumber int `json:"disk_number"` - ISRC string `json:"isrc"` - Link string `json:"link"` - ReleaseDate string `json:"release_date"` - ExplicitLyrics bool `json:"explicit_lyrics"` - ExplicitContentLyrics int `json:"explicit_content_lyrics"` - Artist deezerArtist `json:"artist"` - Album deezerAlbumSimple `json:"album"` - Contributors []deezerArtist `json:"contributors"` -} - -// deezerTrackIsExplicit maps Deezer's parental-advisory fields to a boolean: -// explicit_lyrics is the boolean flag, explicit_content_lyrics uses 1 to mean -// explicit (0 = clean, 2 = unknown). -func deezerTrackIsExplicit(track deezerTrack) bool { - return track.ExplicitLyrics || track.ExplicitContentLyrics == 1 -} - -type deezerArtist struct { - ID int64 `json:"id"` - Name string `json:"name"` - Picture string `json:"picture"` - PictureMedium string `json:"picture_medium"` - PictureBig string `json:"picture_big"` - PictureXL string `json:"picture_xl"` - NbFan int `json:"nb_fan"` -} - -type deezerAlbumSimple struct { - ID int64 `json:"id"` - Title string `json:"title"` - Cover string `json:"cover"` - CoverMedium string `json:"cover_medium"` - CoverBig string `json:"cover_big"` - CoverXL string `json:"cover_xl"` - ReleaseDate string `json:"release_date"` - RecordType string `json:"record_type"` -} - -// deezerTrackArtistDisplay returns the display artist string for a track, -// preferring the Contributors list (comma-joined) when available, falling -// back to the primary Artist.Name. -func deezerTrackArtistDisplay(track deezerTrack) string { - if len(track.Contributors) > 0 { - names := make([]string, len(track.Contributors)) - for i, a := range track.Contributors { - names[i] = a.Name - } - return strings.Join(names, ", ") - } - return track.Artist.Name -} - -func (c *DeezerClient) convertTrack(track deezerTrack) TrackMetadata { - artistName := deezerTrackArtistDisplay(track) - - albumImage := track.Album.CoverXL - if albumImage == "" { - albumImage = track.Album.CoverBig - } - if albumImage == "" { - albumImage = track.Album.CoverMedium - } - if albumImage == "" { - albumImage = track.Album.Cover - } - - releaseDate := track.ReleaseDate - if releaseDate == "" { - releaseDate = track.Album.ReleaseDate - } - - return TrackMetadata{ - SpotifyID: fmt.Sprintf("deezer:%d", track.ID), - Artists: artistName, - Name: track.Title, - AlbumName: track.Album.Title, - AlbumArtist: track.Artist.Name, - DurationMS: track.Duration * 1000, - Images: albumImage, - ReleaseDate: releaseDate, - TrackNumber: track.TrackPosition, - DiscNumber: track.DiskNumber, - ExternalURL: track.Link, - ISRC: track.ISRC, - AlbumID: fmt.Sprintf("deezer:%d", track.Album.ID), - ArtistID: fmt.Sprintf("deezer:%d", track.Artist.ID), - Explicit: deezerTrackIsExplicit(track), - } -} - -type deezerGenre struct { - ID int `json:"id"` - Name string `json:"name"` -} - -type deezerAlbumFull struct { - ID int64 `json:"id"` - Title string `json:"title"` - Cover string `json:"cover"` - CoverMedium string `json:"cover_medium"` - CoverBig string `json:"cover_big"` - CoverXL string `json:"cover_xl"` - ReleaseDate string `json:"release_date"` - NbTracks int `json:"nb_tracks"` - RecordType string `json:"record_type"` - Label string `json:"label"` - Copyright string `json:"copyright"` - Genres struct { - Data []deezerGenre `json:"data"` - } `json:"genres"` - Artist deezerArtist `json:"artist"` - Contributors []deezerArtist `json:"contributors"` - Tracks struct { - Data []deezerTrack `json:"data"` - } `json:"tracks"` -} - -type deezerArtistFull struct { - ID int64 `json:"id"` - Name string `json:"name"` - Picture string `json:"picture"` - PictureMedium string `json:"picture_medium"` - PictureBig string `json:"picture_big"` - PictureXL string `json:"picture_xl"` - NbFan int `json:"nb_fan"` - NbAlbum int `json:"nb_album"` -} - -type deezerPlaylistFull struct { - ID int64 `json:"id"` - Title string `json:"title"` - Picture string `json:"picture"` - PictureMedium string `json:"picture_medium"` - PictureBig string `json:"picture_big"` - PictureXL string `json:"picture_xl"` - NbTracks int `json:"nb_tracks"` - Creator struct { - Name string `json:"name"` - } `json:"creator"` - Tracks struct { - Data []deezerTrack `json:"data"` - } `json:"tracks"` -} - -func (c *DeezerClient) getBestArtistImage(artist deezerArtist) string { - if artist.PictureXL != "" { - return artist.PictureXL - } - if artist.PictureBig != "" { - return artist.PictureBig - } - if artist.PictureMedium != "" { - return artist.PictureMedium - } - return artist.Picture -} - -func (c *DeezerClient) getBestArtistImageFull(artist deezerArtistFull) string { - if artist.PictureXL != "" { - return artist.PictureXL - } - if artist.PictureBig != "" { - return artist.PictureBig - } - if artist.PictureMedium != "" { - return artist.PictureMedium - } - return artist.Picture -} - -func (c *DeezerClient) getBestAlbumImage(album deezerAlbumFull) string { - if album.CoverXL != "" { - return album.CoverXL - } - if album.CoverBig != "" { - return album.CoverBig - } - if album.CoverMedium != "" { - return album.CoverMedium - } - return album.Cover -} diff --git a/go_backend/deezer_supplement_test.go b/go_backend/deezer_supplement_test.go deleted file mode 100644 index 02a907a8..00000000 --- a/go_backend/deezer_supplement_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package gobackend - -import ( - "context" - "io" - "net/http" - "strings" - "testing" - "time" -) - -func TestDeezerClientWithFakeHTTP(t *testing.T) { - client := &DeezerClient{ - httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - body := fakeDeezerResponse(req.URL.Path, req.URL.RawQuery) - status := http.StatusOK - if body == "" { - status = http.StatusNotFound - body = `{"error":"missing"}` - } - return &http.Response{ - StatusCode: status, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(body)), - Request: req, - }, nil - })}, - searchCache: map[string]*cacheEntry{}, - albumCache: map[string]*cacheEntry{}, - artistCache: map[string]*cacheEntry{}, - isrcCache: map[string]string{}, - cacheCleanupInterval: time.Millisecond, - } - ctx := context.Background() - - search, err := client.SearchAll(ctx, "artist song", 2, 2, "") - if err != nil { - t.Fatalf("SearchAll: %v", err) - } - if len(search.Tracks) != 1 || len(search.Artists) != 1 || len(search.Albums) != 1 || len(search.Playlists) != 1 { - t.Fatalf("search = %#v", search) - } - cached, err := client.SearchAll(ctx, "artist song", 2, 2, "") - if err != nil || cached != search { - t.Fatalf("cached SearchAll = %#v/%v", cached, err) - } - if filtered, err := client.SearchAll(ctx, "artist song", 1, 1, "track"); err != nil || len(filtered.Tracks) != 1 || len(filtered.Artists) != 0 { - t.Fatalf("filtered search = %#v/%v", filtered, err) - } - - track, err := client.GetTrack(ctx, "101") - if err != nil { - t.Fatalf("GetTrack: %v", err) - } - if track.Track.SpotifyID != "deezer:101" || track.Track.Artists != "Contributor A, Contributor B" { - t.Fatalf("track = %#v", track) - } - - album, err := client.GetAlbum(ctx, "201") - if err != nil { - t.Fatalf("GetAlbum: %v", err) - } - if album.AlbumInfo.Name != "Album" || len(album.TrackList) != 2 || album.TrackList[1].ISRC == "" { - t.Fatalf("album = %#v", album) - } - if cachedAlbum, err := client.GetAlbum(ctx, "201"); err != nil || cachedAlbum != album { - t.Fatalf("cached album = %#v/%v", cachedAlbum, err) - } - - artist, err := client.GetArtist(ctx, "301") - if err != nil { - t.Fatalf("GetArtist: %v", err) - } - if artist.ArtistInfo.Name != "Artist" || len(artist.Albums) != 1 || artist.Albums[0].TotalTracks == 0 { - t.Fatalf("artist = %#v", artist) - } - if cachedArtist, err := client.GetArtist(ctx, "301"); err != nil || cachedArtist != artist { - t.Fatalf("cached artist = %#v/%v", cachedArtist, err) - } - - playlist, err := client.GetPlaylist(ctx, "401") - if err != nil { - t.Fatalf("GetPlaylist: %v", err) - } - if playlist.PlaylistInfo.Tracks.Total != 2 || len(playlist.TrackList) != 2 { - t.Fatalf("playlist = %#v", playlist) - } - - byISRC, err := client.SearchByISRC(ctx, "USRC17607839") - if err != nil { - t.Fatalf("SearchByISRC: %v", err) - } - if byISRC.SpotifyID != "deezer:101" { - t.Fatalf("by ISRC = %#v", byISRC) - } - if _, err := client.SearchByISRC(ctx, "MISSING"); err == nil { - t.Fatal("expected missing ISRC error") - } - - isrc, err := client.GetTrackISRC(ctx, "102") - if err != nil || isrc != "USRC17607840" { - t.Fatalf("GetTrackISRC = %q/%v", isrc, err) - } - albumID, err := client.GetTrackAlbumID(ctx, "101") - if err != nil || albumID != "201" { - t.Fatalf("GetTrackAlbumID = %q/%v", albumID, err) - } - extended, err := client.GetAlbumExtendedMetadata(ctx, "201") - if err != nil { - t.Fatalf("GetAlbumExtendedMetadata: %v", err) - } - if extended.Genre != "Pop, Dance" || extended.Label != "Label" { - t.Fatalf("extended = %#v", extended) - } - if byTrack, err := client.GetExtendedMetadataByTrackID(ctx, "101"); err != nil || byTrack.Label != "Label" { - t.Fatalf("metadata by track = %#v/%v", byTrack, err) - } - if byISRCMeta, err := client.GetExtendedMetadataByISRC(ctx, "USRC17607839"); err != nil || byISRCMeta.Label != "Label" { - t.Fatalf("metadata by isrc = %#v/%v", byISRCMeta, err) - } - if _, err := client.GetExtendedMetadataByISRC(ctx, ""); err == nil { - t.Fatal("expected empty ISRC metadata error") - } - - if typ, id, err := parseDeezerURL("https://www.deezer.com/us/track/101"); err != nil || typ != "track" || id != "101" { - t.Fatalf("parseDeezerURL = %q/%q/%v", typ, id, err) - } - if _, _, err := parseDeezerURL("https://example.com/track/101"); err == nil { - t.Fatal("expected non-Deezer URL error") - } - - client.cacheMu.Lock() - client.searchCache["expired"] = &cacheEntry{expiresAt: time.Now().Add(-time.Hour)} - client.searchCache["keep1"] = &cacheEntry{expiresAt: time.Now().Add(time.Hour)} - client.searchCache["keep2"] = &cacheEntry{expiresAt: time.Now().Add(2 * time.Hour)} - client.pruneExpiredCacheEntriesLocked(client.searchCache, time.Now()) - client.trimCacheEntriesLocked(client.searchCache, 1) - client.isrcCache["1"] = "A" - client.isrcCache["2"] = "B" - client.trimStringCacheEntriesLocked(client.isrcCache, 1) - client.cacheMu.Unlock() -} diff --git a/go_backend/dns_doh.go b/go_backend/dns_doh.go deleted file mode 100644 index 4850acf1..00000000 --- a/go_backend/dns_doh.go +++ /dev/null @@ -1,438 +0,0 @@ -package gobackend - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net" - "net/http" - "sync" - "time" - - "golang.org/x/net/dns/dnsmessage" -) - -// DNS-over-HTTPS fallback for DNS-level ISP blocking. The OS resolver stays -// the primary path; only when it fails with a DNS error (NXDOMAIN, SERVFAIL, -// refused, resolver timeout) is the host re-resolved over DoH to hardcoded -// resolver IPs and dialed directly. TLS verification still runs against the -// original hostname, so a bad answer cannot silently redirect traffic. - -var dohUpstreams = []string{ - "https://1.1.1.1/dns-query", - "https://8.8.8.8/dns-query", -} - -// Upstream URLs are literal IPs, so this client never needs DNS itself. -var dohClient = &http.Client{ - Transport: &http.Transport{ - DialContext: (&net.Dialer{Timeout: 5 * time.Second}).DialContext, - MaxIdleConnsPerHost: 1, - IdleConnTimeout: 60 * time.Second, - TLSHandshakeTimeout: 5 * time.Second, - ForceAttemptHTTP2: true, - TLSClientConfig: newTLSCompatibilityConfig(false), - }, - Timeout: 10 * time.Second, -} - -const ( - dohCacheMaxEntries = 256 - dohCacheMinTTL = time.Minute - dohCacheMaxTTL = 30 * time.Minute - dohCacheErrorTTL = 30 * time.Second - // Match Go's net.Dialer fallback cadence: give the preferred address family - // a brief head start, then race the remaining vetted answers. We cannot hand - // the hostname back to net.Dialer because the socket must stay pinned to an - // address that already passed the private-network filter. - happyEyeballsFallbackDelay = 300 * time.Millisecond -) - -type dohCacheEntry struct { - ips []net.IP - expiresAt time.Time -} - -type resolvedDialResult struct { - conn net.Conn - err error -} - -type dialContextFunc func(context.Context, string, string) (net.Conn, error) - -var ( - dohMu sync.Mutex - dohCache = map[string]dohCacheEntry{} -) - -// dialWithDoHFallback resolves once, filters every answer, and then dials the -// vetted IP directly. This closes the validation-to-dial DNS rebinding window: -// TLS still receives the original hostname from net/http for SNI and hostname -// verification, while the socket cannot be redirected to a private address. -func dialWithDoHFallback(ctx context.Context, dialer *net.Dialer, network, addr string) (net.Conn, error) { - host, port, splitErr := net.SplitHostPort(addr) - if splitErr != nil { - return nil, splitErr - } - if literal := net.ParseIP(host); literal != nil { - if !IsPrivateNetworkAllowed() && isPrivateIPAddr(literal) { - return nil, fmt.Errorf("network access denied: private/local address %s", host) - } - return dialer.DialContext(ctx, network, addr) - } - - ips, lookupErr := net.DefaultResolver.LookupIP(ctx, "ip", host) - if lookupErr == nil { - ips = filterDialableIPs(ips) - if len(ips) == 0 { - return nil, fmt.Errorf("network access denied: %s resolved only to private/local addresses", host) - } - return dialResolvedIPs(ctx, dialer, network, host, port, ips, nil) - } - - var dnsErr *net.DNSError - if !errors.As(lookupErr, &dnsErr) { - return nil, lookupErr - } - ips, dohErr := dohResolve(ctx, host) - if dohErr != nil { - // Surface the OS resolver's error, not the fallback's. - return nil, lookupErr - } - GoLog("[DoH] OS resolver failed for %s (%v), dialing DoH answer\n", host, lookupErr) - return dialResolvedIPs(ctx, dialer, network, host, port, ips, lookupErr) -} - -func dialResolvedIPs( - ctx context.Context, - dialer *net.Dialer, - network string, - host string, - port string, - ips []net.IP, - initialErr error, -) (net.Conn, error) { - ordered := interleaveDialIPs(ips, network) - return raceResolvedIPs( - ctx, - network, - host, - port, - ordered, - initialErr, - happyEyeballsFallbackDelay, - dialer.DialContext, - ) -} - -func raceResolvedIPs( - ctx context.Context, - network string, - host string, - port string, - ordered []net.IP, - initialErr error, - fallbackDelay time.Duration, - dial dialContextFunc, -) (net.Conn, error) { - if len(ordered) == 0 { - if initialErr != nil { - return nil, initialErr - } - return nil, fmt.Errorf("no dialable address for %s", host) - } - - raceCtx, cancel := context.WithCancel(ctx) - defer cancel() - results := make(chan resolvedDialResult, len(ordered)) - started := 0 - finished := 0 - lastErr := initialErr - - startNext := func() bool { - if started >= len(ordered) { - return false - } - ip := ordered[started] - started++ - go func() { - conn, err := dial( - raceCtx, - network, - net.JoinHostPort(ip.String(), port), - ) - // The channel is sized for every possible attempt, so each goroutine - // can always report exactly once. The winner path drains and closes any - // late successful connections after cancelling the race. - results <- resolvedDialResult{conn: conn, err: err} - }() - return true - } - - startNext() - timer := time.NewTimer(fallbackDelay) - defer timer.Stop() - for { - select { - case <-ctx.Done(): - cancel() - drainDialResults(results, started-finished) - return nil, ctx.Err() - case result := <-results: - finished++ - if result.err == nil && result.conn != nil { - cancel() - drainDialResults(results, started-finished) - return result.conn, nil - } - if result.err != nil { - lastErr = result.err - } - if finished == len(ordered) { - if lastErr == nil { - lastErr = fmt.Errorf("no dialable address for %s", host) - } - return nil, lastErr - } - // A fast refusal should not wait for the fallback timer when there is - // no other connection attempt currently in flight. - if finished == started && startNext() { - resetTimer(timer, fallbackDelay) - } - case <-timer.C: - if startNext() && started < len(ordered) { - timer.Reset(fallbackDelay) - } - } - } -} - -func drainDialResults(results <-chan resolvedDialResult, count int) { - if count <= 0 { - return - } - go func() { - for i := 0; i < count; i++ { - result := <-results - if result.conn != nil { - result.conn.Close() - } - } - }() -} - -func resetTimer(timer *time.Timer, delay time.Duration) { - if !timer.Stop() { - select { - case <-timer.C: - default: - } - } - timer.Reset(delay) -} - -// interleaveDialIPs preserves the resolver's preferred family while ensuring -// the first fallback uses the other family. This avoids waiting through every -// unreachable IPv6 address before trying IPv4 (and vice versa). -func interleaveDialIPs(ips []net.IP, network string) []net.IP { - ordered := make([]net.IP, 0, len(ips)) - var v4, v6 []net.IP - for _, ip := range ips { - if ip == nil { - continue - } - if ip.To4() != nil { - if network != "tcp6" && network != "udp6" { - v4 = append(v4, ip) - } - } else if network != "tcp4" && network != "udp4" { - v6 = append(v6, ip) - } - } - if len(v4) == 0 { - return append(ordered, v6...) - } - if len(v6) == 0 { - return append(ordered, v4...) - } - - firstV4 := false - for _, ip := range ips { - if ip == nil { - continue - } - firstV4 = ip.To4() != nil - break - } - for len(v4) > 0 || len(v6) > 0 { - if firstV4 { - if len(v4) > 0 { - ordered = append(ordered, v4[0]) - v4 = v4[1:] - } - if len(v6) > 0 { - ordered = append(ordered, v6[0]) - v6 = v6[1:] - } - } else { - if len(v6) > 0 { - ordered = append(ordered, v6[0]) - v6 = v6[1:] - } - if len(v4) > 0 { - ordered = append(ordered, v4[0]) - v4 = v4[1:] - } - } - } - return ordered -} - -// dohResolve resolves host over DoH, IPv4 first. Failures are negative-cached -// briefly so a burst of dials does not hammer the resolvers. -func dohResolve(ctx context.Context, host string) ([]net.IP, error) { - if ips, ok := dohCachedIPs(host); ok { - if len(ips) == 0 { - return nil, fmt.Errorf("doh: cached failure for %s", host) - } - return ips, nil - } - - var lastErr error - for _, upstream := range dohUpstreams { - ips, ttl, err := dohQuery(ctx, upstream, host, dnsmessage.TypeA) - if err == nil && len(ips) == 0 { - ips, ttl, err = dohQuery(ctx, upstream, host, dnsmessage.TypeAAAA) - } - if err != nil { - lastErr = err - continue - } - ips = filterDialableIPs(ips) - if len(ips) == 0 { - break - } - dohCachePut(host, ips, min(max(ttl, dohCacheMinTTL), dohCacheMaxTTL)) - return ips, nil - } - dohCachePut(host, nil, dohCacheErrorTTL) - if lastErr == nil { - lastErr = fmt.Errorf("doh: no address for %s", host) - } - return nil, lastErr -} - -func dohQuery(ctx context.Context, upstream, host string, qtype dnsmessage.Type) ([]net.IP, time.Duration, error) { - name, err := dnsmessage.NewName(host + ".") - if err != nil { - return nil, 0, fmt.Errorf("doh: invalid host %q: %w", host, err) - } - msg := dnsmessage.Message{ - Header: dnsmessage.Header{RecursionDesired: true}, - Questions: []dnsmessage.Question{{ - Name: name, - Type: qtype, - Class: dnsmessage.ClassINET, - }}, - } - packed, err := msg.Pack() - if err != nil { - return nil, 0, err - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstream, bytes.NewReader(packed)) - if err != nil { - return nil, 0, err - } - req.Header.Set("Content-Type", "application/dns-message") - req.Header.Set("Accept", "application/dns-message") - - resp, err := dohClient.Do(req) - if err != nil { - return nil, 0, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, 0, fmt.Errorf("doh: %s answered HTTP %d", upstream, resp.StatusCode) - } - body, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024)) - if err != nil { - return nil, 0, err - } - - var reply dnsmessage.Message - if err := reply.Unpack(body); err != nil { - return nil, 0, err - } - if reply.RCode != dnsmessage.RCodeSuccess { - return nil, 0, fmt.Errorf("doh: rcode %v for %s", reply.RCode, host) - } - - var ips []net.IP - ttl := dohCacheMaxTTL - for _, ans := range reply.Answers { - var ip net.IP - switch r := ans.Body.(type) { - case *dnsmessage.AResource: - ip = net.IP(r.A[:]) - case *dnsmessage.AAAAResource: - ip = net.IP(r.AAAA[:]) - default: - continue - } - ips = append(ips, ip) - if t := time.Duration(ans.Header.TTL) * time.Second; t < ttl { - ttl = t - } - } - return ips, ttl, nil -} - -// filterDialableIPs drops private/loopback/link-local answers unless the user -// opted into private-network access — a DoH answer must not bypass the SSRF -// guard the OS-resolver path enforces. -func filterDialableIPs(ips []net.IP) []net.IP { - if IsPrivateNetworkAllowed() { - return ips - } - kept := ips[:0] - for _, ip := range ips { - if isPrivateIPAddr(ip) { - continue - } - kept = append(kept, ip) - } - return kept -} - -func dohCachedIPs(host string) ([]net.IP, bool) { - dohMu.Lock() - defer dohMu.Unlock() - e, ok := dohCache[host] - if !ok || time.Now().After(e.expiresAt) { - return nil, false - } - return e.ips, true -} - -func dohCachePut(host string, ips []net.IP, ttl time.Duration) { - dohMu.Lock() - defer dohMu.Unlock() - if len(dohCache) >= dohCacheMaxEntries { - now := time.Now() - for k, e := range dohCache { - if now.After(e.expiresAt) { - delete(dohCache, k) - } - } - for k := range dohCache { - if len(dohCache) < dohCacheMaxEntries { - break - } - delete(dohCache, k) - } - } - dohCache[host] = dohCacheEntry{ips: ips, expiresAt: time.Now().Add(ttl)} -} diff --git a/go_backend/dns_doh_security_test.go b/go_backend/dns_doh_security_test.go deleted file mode 100644 index 33cf9cbc..00000000 --- a/go_backend/dns_doh_security_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package gobackend - -import ( - "context" - "errors" - "net" - "strings" - "testing" - "time" -) - -func TestDialWithDoHFallbackRejectsPrivateLiteral(t *testing.T) { - SetAllowPrivateNetwork(false) - dialer := &net.Dialer{Timeout: 50 * time.Millisecond} - for _, address := range []string{ - "127.0.0.1:443", - "10.0.0.1:443", - "[::1]:443", - "[fe80::1]:443", - } { - if _, err := dialWithDoHFallback(context.Background(), dialer, "tcp", address); err == nil || - !strings.Contains(err.Error(), "private/local") { - t.Fatalf("address %q was not rejected as private/local: %v", address, err) - } - } -} - -func TestFilterDialableIPsDropsEveryPrivateAnswer(t *testing.T) { - SetAllowPrivateNetwork(false) - filtered := filterDialableIPs([]net.IP{ - net.ParseIP("127.0.0.1"), - net.ParseIP("192.168.1.4"), - net.ParseIP("169.254.1.2"), - net.ParseIP("::1"), - net.ParseIP("203.0.113.10"), - }) - if len(filtered) != 1 || !filtered[0].Equal(net.ParseIP("203.0.113.10")) { - t.Fatalf("unexpected filtered addresses: %v", filtered) - } -} - -func TestInterleaveDialIPsAlternatesAddressFamilies(t *testing.T) { - ordered := interleaveDialIPs([]net.IP{ - net.ParseIP("2001:db8::1"), - net.ParseIP("2001:db8::2"), - net.ParseIP("192.0.2.1"), - net.ParseIP("192.0.2.2"), - }, "tcp") - want := []string{"2001:db8::1", "192.0.2.1", "2001:db8::2", "192.0.2.2"} - if len(ordered) != len(want) { - t.Fatalf("ordered addresses = %v, want %v", ordered, want) - } - for i, ip := range ordered { - if ip.String() != want[i] { - t.Fatalf("ordered[%d] = %s, want %s", i, ip, want[i]) - } - } -} - -func TestRaceResolvedIPsFallsBackWithoutWaitingForPreferredFamilyTimeout(t *testing.T) { - preferredStarted := make(chan struct{}) - clientPeerClosed := make(chan struct{}) - dial := func(ctx context.Context, _ string, address string) (net.Conn, error) { - host, _, err := net.SplitHostPort(address) - if err != nil { - return nil, err - } - if net.ParseIP(host).To4() == nil { - close(preferredStarted) - <-ctx.Done() - return nil, ctx.Err() - } - client, peer := net.Pipe() - go func() { - <-ctx.Done() - peer.Close() - close(clientPeerClosed) - }() - return client, nil - } - - startedAt := time.Now() - conn, err := raceResolvedIPs( - context.Background(), - "tcp", - "dual-stack.example", - "443", - []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("192.0.2.1")}, - nil, - 10*time.Millisecond, - dial, - ) - if err != nil { - t.Fatalf("raceResolvedIPs returned error: %v", err) - } - defer conn.Close() - if elapsed := time.Since(startedAt); elapsed > 100*time.Millisecond { - t.Fatalf("fallback took %v, want <100ms", elapsed) - } - select { - case <-preferredStarted: - default: - t.Fatal("preferred address family was not attempted first") - } - select { - case <-clientPeerClosed: - case <-time.After(time.Second): - t.Fatal("losing dial was not cancelled") - } -} - -func TestRaceResolvedIPsReturnsLastErrorAfterFastFailures(t *testing.T) { - wantErr := errors.New("refused") - conn, err := raceResolvedIPs( - context.Background(), - "tcp", - "failed.example", - "443", - []net.IP{net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")}, - nil, - time.Second, - func(context.Context, string, string) (net.Conn, error) { - return nil, wantErr - }, - ) - if conn != nil || !errors.Is(err, wantErr) { - t.Fatalf("result = (%v, %v), want (nil, %v)", conn, err, wantErr) - } -} diff --git a/go_backend/download_album_folder.go b/go_backend/download_album_folder.go deleted file mode 100644 index 2036bba1..00000000 --- a/go_backend/download_album_folder.go +++ /dev/null @@ -1,114 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "unicode/utf8" -) - -// Only an album component that was missing when the queue was created may be -// resolved later. The caller supplies its leaf template, preserving artist, -// playlist and year folder choices without replacing literal "Unknown" paths. -func resolvedAlbumFolder(req DownloadRequest, album string) string { - if !strings.Contains(req.AlbumFolderTemplate, "{album}") || strings.TrimSpace(album) == "" { - return "" - } - name := strings.ReplaceAll(req.AlbumFolderTemplate, "{album}", album) - name = strings.Map(func(r rune) rune { - if r < 0x20 || r == 0x7f { - return -1 - } - if strings.ContainsRune(`<>:"/\|?*`, r) { - return ' ' - } - return r - }, name) - name = strings.Join(strings.Fields(strings.Trim(name, ". ")), " ") - for strings.Contains(name, "__") { - name = strings.ReplaceAll(name, "__", "_") - } - name = strings.Trim(name, "_ ") - // Match the app's SAF segment limit without splitting a UTF-8 character. - if len(name) > 120 { - name = name[:120] - for !utf8.ValidString(name) { - name = name[:len(name)-1] - } - } - return strings.Trim(name, "._ ") -} - -func resolvedAlbumOutputDirectory(req DownloadRequest, album string) string { - folder := resolvedAlbumFolder(req, album) - if folder == "" || strings.TrimSpace(req.OutputDir) == "" { - return req.OutputDir - } - return filepath.Join(filepath.Dir(filepath.Clean(req.OutputDir)), folder) -} - -func finalizeDownloadAlbumFolder(req DownloadRequest, result *DownloadResponse) error { - album := firstNonEmptyTrimmed(req.AlbumName, result.Album) - if album == "" && strings.Contains(req.AlbumFolderTemplate, "{album}") && result.FilePath != "" { - // Container tags can remain readable even when the audio payload needs - // host-side decryption. Do not require optional provider enrichment. - if payload, err := ReadFileMetadata(result.FilePath); err == nil { - var metadata struct { - Album string `json:"album"` - } - if json.Unmarshal([]byte(payload), &metadata) == nil { - album = strings.TrimSpace(metadata.Album) - result.Album = album - } - } - } - result.ResolvedAlbumFolder = resolvedAlbumFolder(req, album) - // The Android SAF host publishes its temporary file using the resolved - // leaf. A supplied output path/FD remains owned by that host. - if result.ResolvedAlbumFolder == "" || req.OutputPath != "" || isFDOutput(req.OutputFD) || result.AlreadyExists { - return nil - } - dir := resolvedAlbumOutputDirectory(req, album) - if dir == "" || filepath.Clean(dir) == filepath.Dir(result.FilePath) { - return nil - } - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - destination := filepath.Join(dir, filepath.Base(result.FilePath)) - source, err := os.Open(result.FilePath) - if err != nil { - return err - } - defer source.Close() - // Exclusive creation preserves any existing download at the final path, - // including when another queue item resolves the same album concurrently. - output, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) - if err != nil { - return fmt.Errorf("resolve album folder: %w", err) - } - _, copyErr := io.Copy(output, source) - closeErr := output.Close() - if copyErr != nil || closeErr != nil { - _ = os.Remove(destination) - if copyErr != nil { - return copyErr - } - return closeErr - } - if err := source.Close(); err != nil { - _ = os.Remove(destination) - return err - } - if err := os.Remove(result.FilePath); err != nil { - _ = os.Remove(destination) - return err - } - AddAllowedDownloadDir(dir) - GoLog("[Download] Resolved album folder: %s\n", dir) - result.FilePath = destination - return nil -} diff --git a/go_backend/download_album_folder_test.go b/go_backend/download_album_folder_test.go deleted file mode 100644 index f8b9d923..00000000 --- a/go_backend/download_album_folder_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package gobackend - -import ( - "os" - "path/filepath" - "strings" - "testing" - "unicode/utf8" -) - -func TestResolvedAlbumFolder(t *testing.T) { - for _, tc := range []struct { - name, template, album, want string - }{ - {"missing metadata", "{album}", " ", ""}, - {"no pending folder", "", "Album", ""}, - {"invalid template", "Unknown", "Album", ""}, - {"album", "{album}", "Album", "Album"}, - {"year prefix", "[2024] {album}", "Album", "[2024] Album"}, - {"unsafe characters", "{album}", " ../Album: \"Deluxe\"/Part\\Two\x00 ", "Album Deluxe Part Two"}, - {"literal unknown album", "{album}", "Unknown", "Unknown"}, - } { - t.Run(tc.name, func(t *testing.T) { - got := resolvedAlbumFolder(DownloadRequest{AlbumFolderTemplate: tc.template}, tc.album) - if got != tc.want { - t.Fatalf("got %q, want %q", got, tc.want) - } - }) - } - got := resolvedAlbumFolder(DownloadRequest{AlbumFolderTemplate: "[2024] {album}"}, strings.Repeat("音楽", 40)) - if len(got) > 120 || !utf8.ValidString(got) { - t.Fatalf("invalid bounded UTF-8 folder: %q", got) - } -} - -func TestFinalizeDownloadAlbumFolder(t *testing.T) { - for _, ext := range []string{".mp4", ".m4a", ".opus", ".flac"} { - t.Run(ext, func(t *testing.T) { - root := t.TempDir() - originalDir := filepath.Join(root, "Playlist", "Artist", "Unknown") - if err := os.MkdirAll(originalDir, 0755); err != nil { - t.Fatal(err) - } - original := filepath.Join(originalDir, "Track"+ext) - data := []byte("unchanged audio and metadata") - if err := os.WriteFile(original, data, 0644); err != nil { - t.Fatal(err) - } - req := DownloadRequest{OutputDir: originalDir, AlbumFolderTemplate: "{album}"} - result := DownloadResponse{Success: true, FilePath: original, Album: "Resolved Album"} - if err := finalizeDownloadAlbumFolder(req, &result); err != nil { - t.Fatal(err) - } - want := filepath.Join(root, "Playlist", "Artist", "Resolved Album", "Track"+ext) - if result.FilePath != want || result.ResolvedAlbumFolder != "Resolved Album" { - t.Fatalf("unexpected resolved output: %+v", result) - } - got, err := os.ReadFile(want) - if err != nil || string(got) != string(data) { - t.Fatalf("output changed: %q, %v", got, err) - } - if _, err := os.Stat(original); !os.IsNotExist(err) { - t.Fatalf("old output remains: %v", err) - } - }) - } -} - -func TestFinalizeAlbumFolderPreservesHostOutputAndSourceAlbum(t *testing.T) { - req := DownloadRequest{ - OutputDir: filepath.Join(t.TempDir(), "Unknown"), OutputPath: "/host/cache/track.mp4", - AlbumFolderTemplate: "{album}", AlbumName: "Source Album", - } - result := DownloadResponse{FilePath: req.OutputPath, Album: "Provider Compilation"} - if err := finalizeDownloadAlbumFolder(req, &result); err != nil { - t.Fatal(err) - } - if result.FilePath != req.OutputPath || result.ResolvedAlbumFolder != "Source Album" { - t.Fatalf("host output or source album replaced: %+v", result) - } - req.OutputPath = "" - if got := filepath.Dir(buildOutputPath(req)); filepath.Base(got) != "Source Album" { - t.Fatalf("enriched metadata did not resolve the path before transfer: %q", got) - } -} - -func TestFinalizeAlbumFolderDoesNotOverwriteExistingFile(t *testing.T) { - root := t.TempDir() - original := filepath.Join(root, "track.mp4") - dir := filepath.Join(root, "Album") - if err := os.Mkdir(dir, 0755); err != nil { - t.Fatal(err) - } - destination := filepath.Join(dir, "track.mp4") - for path, value := range map[string]string{original: "new audio", destination: "existing audio"} { - if err := os.WriteFile(path, []byte(value), 0644); err != nil { - t.Fatal(err) - } - } - req := DownloadRequest{OutputDir: filepath.Join(root, "Unknown"), AlbumFolderTemplate: "{album}"} - result := DownloadResponse{FilePath: original, Album: "Album"} - if err := finalizeDownloadAlbumFolder(req, &result); err == nil { - t.Fatal("expected destination collision") - } - for path, want := range map[string]string{original: "new audio", destination: "existing audio"} { - got, err := os.ReadFile(path) - if err != nil || string(got) != want { - t.Fatalf("file damaged: %q, %v", got, err) - } - } -} - -func TestFinalizeAlbumFolderReadsContainerTagsWithoutAudioDecoding(t *testing.T) { - albumData := buildM4AAtom("data", append([]byte{0, 0, 0, 1, 0, 0, 0, 0}, []byte("Embedded Album")...)) - ilst := buildM4AAtom("ilst", buildM4AAtom("\xa9alb", albumData)) - meta := buildM4AAtom("meta", append(make([]byte, 4), ilst...)) - data := buildM4AAtom("moov", buildM4AAtom("udta", meta)) - // The album atom is independent of the encrypted/undecoded audio bytes. - data = append(data, buildM4AAtom("mdat", []byte("undecoded audio payload"))...) - path := filepath.Join(t.TempDir(), "encrypted.mp4") - if err := os.WriteFile(path, data, 0644); err != nil { - t.Fatal(err) - } - req := DownloadRequest{OutputPath: path, AlbumFolderTemplate: "{album}"} - result := DownloadResponse{FilePath: path} - if err := finalizeDownloadAlbumFolder(req, &result); err != nil { - t.Fatal(err) - } - if result.ResolvedAlbumFolder != "Embedded Album" || result.Album != "Embedded Album" { - t.Fatalf("missing container album: %+v", result) - } - got, err := os.ReadFile(path) - if err != nil || string(got) != string(data) { - t.Fatalf("host-owned audio changed: %v", err) - } -} diff --git a/go_backend/download_comment_test.go b/go_backend/download_comment_test.go deleted file mode 100644 index 53829eaf..00000000 --- a/go_backend/download_comment_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package gobackend - -import "testing" - -func TestBuildDownloadedFileCommentKeepsSourceComment(t *testing.T) { - const source = "https://source.example/album/example/123" - got := buildDownloadedFileComment(source, "https://provider.example/albums/example") - if got != source { - t.Fatalf("comment = %q, want %q", got, source) - } -} - -func TestBuildDownloadedFileCommentUsesProviderCommentWhenSourceIsEmpty(t *testing.T) { - const provider = "https://provider.example/albums/example" - got := buildDownloadedFileComment("", provider) - if got != provider { - t.Fatalf("comment = %q, want %q", got, provider) - } -} - -func TestBuildDownloadedFileCommentStaysEmptyWithoutProviderComment(t *testing.T) { - if got := buildDownloadedFileComment("", ""); got != "" { - t.Fatalf("comment = %q, want empty", got) - } -} - -func TestBuildDownloadedFileCommentTrimsWhitespace(t *testing.T) { - const original = "https://example.test/album/1" - if got := buildDownloadedFileComment(" "+original+"\r\n", ""); got != original { - t.Fatalf("comment = %q, want %q", got, original) - } -} diff --git a/go_backend/download_output_staging.go b/go_backend/download_output_staging.go deleted file mode 100644 index 4c8a0b0e..00000000 --- a/go_backend/download_output_staging.go +++ /dev/null @@ -1,33 +0,0 @@ -package gobackend - -import ( - "path/filepath" - "strings" - "sync" -) - -// downloadPathLocks serializes writes per final output path so concurrent -// downloads that resolve to the same file cannot interleave bytes into one -// output or race the staged-promote rename. Keys are normalized case-folded -// cleaned paths; entries live for the process lifetime (bounded by the number -// of distinct output files in a session). -var downloadPathLocks sync.Map - -// lockDownloadOutputPath locks the given final output path and returns the -// unlock function. Different paths keep downloading in parallel; a second -// download of the same path blocks until the first finishes. -func lockDownloadOutputPath(path string) func() { - key := strings.ToLower(filepath.Clean(path)) - value, _ := downloadPathLocks.LoadOrStore(key, &sync.Mutex{}) - mu := value.(*sync.Mutex) - mu.Lock() - return mu.Unlock -} - -// stagedDownloadPath returns the sibling name downloads are streamed into -// before being promoted to the final path with an atomic rename. The suffix -// keeps the staged file invisible to extension-based duplicate checks, which -// match on the final audio extension. -func stagedDownloadPath(finalPath string) string { - return finalPath + ".partial" -} diff --git a/go_backend/download_preparation_cache.go b/go_backend/download_preparation_cache.go deleted file mode 100644 index 0175c21f..00000000 --- a/go_backend/download_preparation_cache.go +++ /dev/null @@ -1,129 +0,0 @@ -package gobackend - -import ( - "strings" - "sync" - "time" -) - -const ( - downloadPreparationCacheTTL = 5 * time.Minute - downloadPreparationCacheMax = 128 -) - -type preparedDownloadRequestEntry struct { - key string - request DownloadRequest - metadataPrepared bool - createdAt time.Time -} - -var ( - preparedDownloadRequests = make(map[string]preparedDownloadRequestEntry) - preparedDownloadRequestsMu sync.Mutex -) - -func downloadPreparationKey(req DownloadRequest) string { - return strings.Join([]string{ - strings.TrimSpace(req.ItemID), - strings.ToLower(strings.TrimSpace(req.Service)), - strings.ToLower(strings.TrimSpace(req.Source)), - strings.TrimSpace(req.SpotifyID), - strings.TrimSpace(req.TidalID), - strings.TrimSpace(req.QobuzID), - strings.TrimSpace(req.DeezerID), - strings.ToLower(strings.TrimSpace(req.TrackName)), - strings.ToLower(strings.TrimSpace(req.ArtistName)), - }, "\n") -} - -func prunePreparedDownloadRequestsLocked(now time.Time) { - for itemID, entry := range preparedDownloadRequests { - if now.Sub(entry.createdAt) >= downloadPreparationCacheTTL { - delete(preparedDownloadRequests, itemID) - } - } - for len(preparedDownloadRequests) >= downloadPreparationCacheMax { - var oldestID string - var oldestAt time.Time - for itemID, entry := range preparedDownloadRequests { - if oldestID == "" || entry.createdAt.Before(oldestAt) { - oldestID = itemID - oldestAt = entry.createdAt - } - } - if oldestID == "" { - break - } - delete(preparedDownloadRequests, oldestID) - } -} - -func cacheDownloadRequestForVerification(key string, req DownloadRequest, metadataPrepared bool) { - itemID := strings.TrimSpace(req.ItemID) - if itemID == "" || strings.TrimSpace(key) == "" { - return - } - - preparedDownloadRequestsMu.Lock() - defer preparedDownloadRequestsMu.Unlock() - now := time.Now() - prunePreparedDownloadRequestsLocked(now) - preparedDownloadRequests[itemID] = preparedDownloadRequestEntry{ - key: key, - request: req, - metadataPrepared: metadataPrepared, - createdAt: now, - } -} - -func cachePreparedDownloadRequest(key string, req DownloadRequest) { - cacheDownloadRequestForVerification(key, req, true) -} - -func cacheUnpreparedDownloadRequest(key string, req DownloadRequest) { - cacheDownloadRequestForVerification(key, req, false) -} - -func takePreparedDownloadRequest(key string, fresh DownloadRequest) (DownloadRequest, bool, bool) { - itemID := strings.TrimSpace(fresh.ItemID) - if itemID == "" || strings.TrimSpace(key) == "" { - return fresh, false, false - } - - preparedDownloadRequestsMu.Lock() - defer preparedDownloadRequestsMu.Unlock() - now := time.Now() - prunePreparedDownloadRequestsLocked(now) - entry, ok := preparedDownloadRequests[itemID] - if !ok { - return fresh, false, false - } - delete(preparedDownloadRequests, itemID) - if entry.key != key { - return fresh, false, false - } - - prepared := entry.request - fresh.ISRC = prepared.ISRC - fresh.SpotifyID = prepared.SpotifyID - fresh.TrackName = prepared.TrackName - fresh.ArtistName = prepared.ArtistName - fresh.AlbumName = prepared.AlbumName - fresh.AlbumArtist = prepared.AlbumArtist - fresh.CoverURL = prepared.CoverURL - fresh.TrackNumber = prepared.TrackNumber - fresh.DiscNumber = prepared.DiscNumber - fresh.TotalTracks = prepared.TotalTracks - fresh.TotalDiscs = prepared.TotalDiscs - fresh.ReleaseDate = prepared.ReleaseDate - fresh.DurationMS = prepared.DurationMS - fresh.Genre = prepared.Genre - fresh.Label = prepared.Label - fresh.Copyright = prepared.Copyright - fresh.Composer = prepared.Composer - fresh.TidalID = prepared.TidalID - fresh.QobuzID = prepared.QobuzID - fresh.DeezerID = prepared.DeezerID - return fresh, entry.metadataPrepared, true -} diff --git a/go_backend/download_preparation_cache_test.go b/go_backend/download_preparation_cache_test.go deleted file mode 100644 index 4ea0eac6..00000000 --- a/go_backend/download_preparation_cache_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package gobackend - -import ( - "testing" - "time" -) - -func resetPreparedDownloadRequestCacheForTest() { - preparedDownloadRequestsMu.Lock() - preparedDownloadRequests = make(map[string]preparedDownloadRequestEntry) - preparedDownloadRequestsMu.Unlock() -} - -func TestPreparedDownloadRequestCache(t *testing.T) { - t.Cleanup(resetPreparedDownloadRequestCacheForTest) - resetPreparedDownloadRequestCacheForTest() - - fresh := DownloadRequest{ - ItemID: "item-1", - Service: "provider-a", - Source: "source-a", - SpotifyID: "source-track-1", - TrackName: "Track", - ArtistName: "Artist", - OutputDir: "/new/output", - OutputPath: "/new/output/current.flac", - OutputFD: 42, - Quality: "lossless", - EmbedMetadata: true, - } - key := downloadPreparationKey(fresh) - prepared := fresh - prepared.ISRC = "USRC17607839" - prepared.AlbumName = "Resolved Album" - prepared.AlbumArtist = "Resolved Album Artist" - prepared.DeezerID = "alternate-track-1" - prepared.Genre = "Pop" - prepared.OutputDir = "/stale/output" - prepared.OutputPath = "/stale/output/old.flac" - prepared.OutputFD = 7 - prepared.Quality = "stale-quality" - prepared.EmbedMetadata = false - cachePreparedDownloadRequest(key, prepared) - - got, metadataPrepared, ok := takePreparedDownloadRequest(key, fresh) - if !ok { - t.Fatal("expected prepared request cache hit") - } - if !metadataPrepared { - t.Fatal("prepared request should be marked as metadata-prepared") - } - if got.ISRC != prepared.ISRC || got.AlbumName != prepared.AlbumName || got.DeezerID != prepared.DeezerID || got.Genre != prepared.Genre { - t.Fatalf("prepared metadata was not restored: %#v", got) - } - if got.OutputDir != fresh.OutputDir || got.OutputPath != fresh.OutputPath || got.OutputFD != fresh.OutputFD || got.Quality != fresh.Quality || got.EmbedMetadata != fresh.EmbedMetadata { - t.Fatalf("fresh output/settings fields were overwritten: %#v", got) - } - if _, _, ok := takePreparedDownloadRequest(key, fresh); ok { - t.Fatal("prepared request should be consumed after one retry") - } - - cacheUnpreparedDownloadRequest(key, fresh) - _, metadataPrepared, ok = takePreparedDownloadRequest(key, fresh) - if !ok || metadataPrepared { - t.Fatalf("unprepared verification request = hit:%v metadataPrepared:%v", ok, metadataPrepared) - } -} - -func TestPreparedDownloadRequestCacheRejectsChangedTrackAndExpiry(t *testing.T) { - t.Cleanup(resetPreparedDownloadRequestCacheForTest) - resetPreparedDownloadRequestCacheForTest() - - req := DownloadRequest{ - ItemID: "item-2", - Service: "provider-a", - SpotifyID: "source-track-2", - TrackName: "Track", - ArtistName: "Artist", - } - key := downloadPreparationKey(req) - cachePreparedDownloadRequest(key, req) - - changed := req - changed.SpotifyID = "source-track-other" - if _, _, ok := takePreparedDownloadRequest(downloadPreparationKey(changed), changed); ok { - t.Fatal("changed track must not reuse another track's prepared metadata") - } - preparedDownloadRequestsMu.Lock() - _, staleEntryExists := preparedDownloadRequests[req.ItemID] - preparedDownloadRequestsMu.Unlock() - if staleEntryExists { - t.Fatal("mismatched prepared request should be discarded") - } - - cachePreparedDownloadRequest(key, req) - preparedDownloadRequestsMu.Lock() - entry := preparedDownloadRequests[req.ItemID] - entry.createdAt = time.Now().Add(-downloadPreparationCacheTTL) - preparedDownloadRequests[req.ItemID] = entry - preparedDownloadRequestsMu.Unlock() - if _, _, ok := takePreparedDownloadRequest(key, req); ok { - t.Fatal("expired prepared request must not be reused") - } -} diff --git a/go_backend/duplicate.go b/go_backend/duplicate.go deleted file mode 100644 index 6cd8abcf..00000000 --- a/go_backend/duplicate.go +++ /dev/null @@ -1,446 +0,0 @@ -package gobackend - -import ( - "encoding/binary" - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "time" -) - -// isrcFileEntry caches the parse result for one file so index rebuilds only -// re-read files whose size or mtime changed. -type isrcFileEntry struct { - size int64 - modTime int64 // UnixNano - isrc string // uppercase; empty when the file carries no ISRC tag -} - -type ISRCIndex struct { - index map[string]string // ISRC (uppercase) -> file path - files map[string]isrcFileEntry // file path -> cached parse result - outputDir string - buildTime atomic.Int64 // UnixNano of the last build or write - mu sync.RWMutex -} - -var ( - isrcIndexCache = make(map[string]*ISRCIndex) - isrcIndexCacheMu sync.RWMutex - isrcBuildingMu sync.Map // Per-directory build lock to prevent concurrent builds - isrcIndexTTL = 5 * time.Minute - - isrcIndexBuildWorkers = 4 -) - -func (idx *ISRCIndex) isFresh() bool { - return time.Since(time.Unix(0, idx.buildTime.Load())) < isrcIndexTTL -} - -func GetISRCIndex(outputDir string) *ISRCIndex { - isrcIndexCacheMu.RLock() - idx, exists := isrcIndexCache[outputDir] - isrcIndexCacheMu.RUnlock() - - if exists && idx.isFresh() { - return idx - } - - buildLock, _ := isrcBuildingMu.LoadOrStore(outputDir, &sync.Mutex{}) - mu := buildLock.(*sync.Mutex) - mu.Lock() - defer mu.Unlock() - - isrcIndexCacheMu.RLock() - idx, exists = isrcIndexCache[outputDir] - isrcIndexCacheMu.RUnlock() - - if exists && idx.isFresh() { - return idx - } - - return buildISRCIndex(outputDir) -} - -func buildISRCIndex(outputDir string) *ISRCIndex { - idx := &ISRCIndex{ - index: make(map[string]string), - files: make(map[string]isrcFileEntry), - outputDir: outputDir, - } - idx.buildTime.Store(time.Now().UnixNano()) - - if outputDir == "" { - return idx - } - - // Reuse the previous build's per-file cache: files whose size and mtime - // are unchanged are adopted without touching their content, so a rebuild - // is normally just a stat walk. - prevFiles := map[string]isrcFileEntry{} - isrcIndexCacheMu.RLock() - if prev, ok := isrcIndexCache[outputDir]; ok { - prev.mu.RLock() - for path, entry := range prev.files { - prevFiles[path] = entry - } - prev.mu.RUnlock() - } - isrcIndexCacheMu.RUnlock() - - startTime := time.Now() - type parseTask struct { - path string - size int64 - modTime int64 - } - var toParse []parseTask - reused := 0 - - filepath.Walk(outputDir, func(path string, info os.FileInfo, err error) error { - if err != nil || info.IsDir() { - return nil - } - - ext := strings.ToLower(filepath.Ext(path)) - if !isrcIndexExts[ext] { - return nil - } - - size := info.Size() - modTime := info.ModTime().UnixNano() - if entry, ok := prevFiles[path]; ok && entry.size == size && entry.modTime == modTime { - idx.files[path] = entry - if entry.isrc != "" { - idx.index[entry.isrc] = path - } - reused++ - return nil - } - toParse = append(toParse, parseTask{path: path, size: size, modTime: modTime}) - return nil - }) - - if len(toParse) > 0 { - // New/changed files: read only their Vorbis comment block, in - // parallel. Embedded cover art (megabytes per file) is never loaded. - isrcs := make([]string, len(toParse)) - workerCount := isrcIndexBuildWorkers - if len(toParse) < workerCount { - workerCount = len(toParse) - } - tasks := make(chan int) - var wg sync.WaitGroup - for w := 0; w < workerCount; w++ { - wg.Add(1) - go func() { - defer wg.Done() - for i := range tasks { - isrcs[i] = strings.ToUpper(readFileISRC(toParse[i].path)) - } - }() - } - for i := range toParse { - tasks <- i - } - close(tasks) - wg.Wait() - - for i, task := range toParse { - entry := isrcFileEntry{size: task.size, modTime: task.modTime, isrc: isrcs[i]} - idx.files[task.path] = entry - if entry.isrc != "" { - idx.index[entry.isrc] = task.path - } - } - } - - fmt.Printf("[ISRCIndex] Built index for %s: %d files (%d parsed, %d cached) in %v\n", - outputDir, len(idx.files), len(toParse), reused, time.Since(startTime).Round(time.Millisecond)) - - isrcIndexCacheMu.Lock() - isrcIndexCache[outputDir] = idx - isrcIndexCacheMu.Unlock() - - return idx -} - -// isrcIndexExts are the formats the download pipeline can produce; each has -// a native tag reader that stops at the metadata blocks. -var isrcIndexExts = map[string]bool{ - ".flac": true, - ".mp3": true, - ".m4a": true, - ".ogg": true, - ".opus": true, -} - -// readFileISRC reads the ISRC tag using the native reader for the format. -// Returns "" for unsupported formats, unreadable files, or missing tags. -func readFileISRC(path string) string { - switch strings.ToLower(filepath.Ext(path)) { - case ".flac": - return readFlacISRC(path) - case ".mp3": - if meta, err := ReadID3Tags(path); err == nil && meta != nil { - return strings.TrimSpace(meta.ISRC) - } - case ".m4a": - if meta, err := ReadM4ATags(path); err == nil && meta != nil { - return strings.TrimSpace(meta.ISRC) - } - case ".ogg", ".opus": - if meta, err := ReadOggVorbisComments(path); err == nil && meta != nil { - return strings.TrimSpace(meta.ISRC) - } - } - return "" -} - -// readFlacISRC extracts the ISRC Vorbis comment from a FLAC file by walking -// the metadata block headers and reading only the VORBIS_COMMENT payload; -// picture and padding blocks are seeked past, never loaded. Returns "" when -// the file is not FLAC or carries no ISRC tag. -func readFlacISRC(path string) string { - f, err := os.Open(path) - if err != nil { - return "" - } - defer f.Close() - - magic := make([]byte, 4) - if _, err := io.ReadFull(f, magic); err != nil || string(magic) != "fLaC" { - return "" - } - - header := make([]byte, 4) - for { - if _, err := io.ReadFull(f, header); err != nil { - return "" - } - last := header[0]&0x80 != 0 - blockType := header[0] & 0x7F - length := int64(header[1])<<16 | int64(header[2])<<8 | int64(header[3]) - if blockType == 4 { // VORBIS_COMMENT - if length > 16<<20 { - return "" - } - payload := make([]byte, length) - if _, err := io.ReadFull(f, payload); err != nil { - return "" - } - return vorbisCommentISRC(payload) - } - if last { - return "" - } - if _, err := f.Seek(length, io.SeekCurrent); err != nil { - return "" - } - } -} - -func vorbisCommentISRC(payload []byte) string { - if len(payload) < 8 { - return "" - } - offset := int(binary.LittleEndian.Uint32(payload[0:4])) + 4 - if offset < 4 || offset+4 > len(payload) { - return "" - } - count := int(binary.LittleEndian.Uint32(payload[offset : offset+4])) - offset += 4 - for i := 0; i < count; i++ { - if offset+4 > len(payload) { - return "" - } - commentLen := int(binary.LittleEndian.Uint32(payload[offset : offset+4])) - offset += 4 - if commentLen < 0 || offset+commentLen > len(payload) { - return "" - } - comment := payload[offset : offset+commentLen] - offset += commentLen - eq := strings.IndexByte(string(comment), '=') - if eq > 0 && strings.EqualFold(string(comment[:eq]), "ISRC") { - return strings.TrimSpace(string(comment[eq+1:])) - } - } - return "" -} - -func (idx *ISRCIndex) lookup(isrc string) (string, bool) { - if isrc == "" { - return "", false - } - - idx.mu.RLock() - defer idx.mu.RUnlock() - - path, exists := idx.index[strings.ToUpper(isrc)] - return path, exists -} - -func (idx *ISRCIndex) remove(isrc string) { - if isrc == "" { - return - } - - idx.mu.Lock() - defer idx.mu.Unlock() - - delete(idx.index, strings.ToUpper(isrc)) -} - -func (idx *ISRCIndex) Lookup(isrc string) (string, error) { - path, _ := idx.lookup(isrc) - return path, nil -} - -func (idx *ISRCIndex) Add(isrc, filePath string) { - if isrc == "" || filePath == "" { - return - } - - upper := strings.ToUpper(isrc) - var entry *isrcFileEntry - if info, err := os.Stat(filePath); err == nil { - entry = &isrcFileEntry{ - size: info.Size(), - modTime: info.ModTime().UnixNano(), - isrc: upper, - } - } - - idx.mu.Lock() - idx.index[upper] = filePath - if entry != nil { - if idx.files == nil { - idx.files = make(map[string]isrcFileEntry) - } - idx.files[filePath] = *entry - } - idx.mu.Unlock() - - // The index is write-maintained after every successful download; - // refreshing the timestamp keeps the TTL from forcing a full rebuild in - // the middle of the exact workload the index exists to serve. - idx.buildTime.Store(time.Now().UnixNano()) -} - -func InvalidateISRCCache(outputDir string) { - isrcIndexCacheMu.Lock() - delete(isrcIndexCache, outputDir) - isrcIndexCacheMu.Unlock() -} - -func checkISRCExistsInternal(outputDir, isrc string) (string, bool) { - if isrc == "" || outputDir == "" { - return "", false - } - - idx := GetISRCIndex(outputDir) - filePath, exists := idx.lookup(isrc) - if !exists { - return "", false - } - - if !CheckFileExists(filePath) { - // Stale index entry; remove it and return not found. - idx.remove(isrc) - return "", false - } - - return filePath, true -} - -func CheckISRCExists(outputDir, isrc string) (string, error) { - filepath, _ := checkISRCExistsInternal(outputDir, isrc) - return filepath, nil -} - -func CheckFileExists(filePath string) bool { - info, err := os.Stat(filePath) - if err != nil { - return false - } - return !info.IsDir() && info.Size() > 0 -} - -type FileExistenceResult struct { - ISRC string `json:"isrc"` - Exists bool `json:"exists"` - FilePath string `json:"file_path,omitempty"` - TrackName string `json:"track_name,omitempty"` - ArtistName string `json:"artist_name,omitempty"` -} - -func CheckFilesExistParallel(outputDir string, tracksJSON string) (string, error) { - var tracks []struct { - ISRC string `json:"isrc"` - TrackName string `json:"track_name"` - ArtistName string `json:"artist_name"` - } - if err := json.Unmarshal([]byte(tracksJSON), &tracks); err != nil { - return "", fmt.Errorf("failed to parse tracks JSON: %w", err) - } - - results := make([]FileExistenceResult, len(tracks)) - - isrcIdx := GetISRCIndex(outputDir) - - // A lookup is a single map read. Holding one read lock for the batch avoids - // one goroutine and one lock/unlock pair per track, which was slower and - // could create thousands of goroutines for large playlists. - isrcIdx.mu.RLock() - for i, track := range tracks { - result := FileExistenceResult{ - ISRC: track.ISRC, - TrackName: track.TrackName, - ArtistName: track.ArtistName, - } - if track.ISRC != "" { - if filePath, exists := isrcIdx.index[strings.ToUpper(track.ISRC)]; exists { - result.Exists = true - result.FilePath = filePath - } - } - results[i] = result - } - isrcIdx.mu.RUnlock() - - resultJSON, err := json.Marshal(results) - if err != nil { - return "", fmt.Errorf("failed to marshal results: %w", err) - } - - return string(resultJSON), nil -} - -func PreBuildISRCIndex(outputDir string) error { - if outputDir == "" { - return fmt.Errorf("output directory is required") - } - - buildISRCIndex(outputDir) - return nil -} - -func AddToISRCIndex(outputDir, isrc, filePath string) { - if outputDir == "" || isrc == "" || filePath == "" { - return - } - - isrcIndexCacheMu.RLock() - idx, exists := isrcIndexCache[outputDir] - isrcIndexCacheMu.RUnlock() - - if exists { - idx.Add(isrc, filePath) - } -} diff --git a/go_backend/exports.go b/go_backend/exports.go deleted file mode 100644 index 192fed6e..00000000 --- a/go_backend/exports.go +++ /dev/null @@ -1,111 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "runtime/debug" - "strings" - "sync" -) - -var ( - metadataLanguageMu sync.RWMutex - metadataLanguageTag string -) - -// SetMetadataLanguage sets the app's display language (BCP 47 tag, e.g. -// "en-US" or "id"), used as Accept-Language on metadata API requests so -// providers localize names by the app language instead of IP geolocation. -func SetMetadataLanguage(tag string) { - metadataLanguageMu.Lock() - metadataLanguageTag = strings.TrimSpace(tag) - metadataLanguageMu.Unlock() -} - -func metadataAcceptLanguage() string { - metadataLanguageMu.RLock() - tag := metadataLanguageTag - metadataLanguageMu.RUnlock() - if tag == "" || strings.HasPrefix(strings.ToLower(tag), "en") { - return "en-US,en;q=0.9" - } - return tag + ",en;q=0.8" -} - -// ReleaseMemory drops idle pooled extension runtimes, forces a GC, and -// returns freed heap to the OS. Called from the app on OS memory pressure and -// when backgrounded, so the Go side's RSS doesn't sit at its high-water mark -// after large downloads/tag writes. -func ReleaseMemory() { - releaseMemory(false) -} - -// ReleaseMemoryUnderPressure additionally drops disposable live caches. It is -// reserved for an OS memory-pressure signal; ordinary backgrounding keeps -// network-backed caches warm. -func ReleaseMemoryUnderPressure() { - releaseMemory(true) -} - -func releaseMemory(underPressure bool) { - drainAllIsolatedRuntimePools() - CloseIdleConnections() - if underPressure { - clearCoverMemoryCache() - globalLyricsCache.DropMemory() - clearPrivateIPCache() - clearExtensionHealthCache() - } - debug.FreeOSMemory() -} - -// SetSongLinkNetworkOptions is kept for backward compatibility. -func SetSongLinkNetworkOptions(allowHTTP, insecureTLS bool) { - SetNetworkCompatibilityOptions(allowHTTP, insecureTLS) -} - -// GetTrackPlatformLinksJSON returns {"platforms": {platformID: url}} for a -// track, resolved via song.link (memory-cached; either ID may be empty). -func GetTrackPlatformLinksJSON(spotifyTrackID string, isrc string) (string, error) { - links, err := NewSongLinkClient().GetTrackPlatformLinks(spotifyTrackID, isrc) - if err != nil { - return "", err - } - return marshalJSONString(map[string]any{"platforms": links}) -} - -func SetDownloadDirectory(path string) error { - return setDownloadDir(path) -} - -func AllowDownloadDir(path string) { - if strings.TrimSpace(path) == "" { - return - } - AddAllowedDownloadDir(path) -} - -func CheckDuplicatesBatch(outputDir, tracksJSON string) (string, error) { - return CheckFilesExistParallel(outputDir, tracksJSON) -} - -func PreBuildDuplicateIndex(outputDir string) error { - return PreBuildISRCIndex(outputDir) -} - -func InvalidateDuplicateIndex(outputDir string) { - InvalidateISRCCache(outputDir) -} - -func BuildFilename(template string, metadataJSON string) (string, error) { - var metadata map[string]any - if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil { - return "", err - } - - filename := buildFilenameFromTemplate(template, metadata) - return filename, nil -} - -func SanitizeFilename(filename string) string { - return sanitizeFilename(filename) -} diff --git a/go_backend/exports_deezer.go b/go_backend/exports_deezer.go deleted file mode 100644 index c335149d..00000000 --- a/go_backend/exports_deezer.go +++ /dev/null @@ -1,198 +0,0 @@ -package gobackend - -import ( - "context" - "fmt" - "strings" - "time" -) - -// GetTrackCacheSize and ClearTrackIDCache back the Settings cache screen. The -// track-ID cache is currently a no-op, so these report an empty cache and clear -// nothing, but the gomobile export contract is kept for the Dart/Kotlin callers. -func GetTrackCacheSize() int { - return 0 -} - -func ClearTrackIDCache() { -} - -func GetDeezerMetadata(resourceType, resourceID string) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - client := GetDeezerClient() - var data any - var err error - - switch resourceType { - case "track": - data, err = client.GetTrack(ctx, resourceID) - case "album": - data, err = client.GetAlbum(ctx, resourceID) - case "artist": - data, err = client.GetArtist(ctx, resourceID) - case "playlist": - data, err = client.GetPlaylist(ctx, resourceID) - default: - return "", fmt.Errorf("unsupported Deezer resource type: %s", resourceType) - } - - if err != nil { - return "", err - } - - return marshalJSONString(data) -} - -func GetDeezerExtendedMetadata(trackID string) (string, error) { - if trackID == "" { - return "", fmt.Errorf("empty track ID") - } - - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - - client := GetDeezerClient() - metadata, err := client.GetExtendedMetadataByTrackID(ctx, trackID) - if err != nil { - GoLog("[Deezer] Failed to get extended metadata: %v\n", err) - return "", err - } - - result := buildDeezerExtendedMetadataResult(metadata) - - return marshalJSONString(result) -} - -func SearchDeezerByISRC(isrc string) (string, error) { - return SearchDeezerByISRCForItemID(isrc, "") -} - -func SearchDeezerByISRCForItemID(isrc string, itemID string) (string, error) { - parentCtx := context.Background() - if itemID != "" { - parentCtx = initDownloadCancel(itemID) - defer clearDownloadCancel(itemID) - if isDownloadCancelled(itemID) { - return "", ErrDownloadCancelled - } - } - - ctx, cancel := context.WithTimeout(parentCtx, 10*time.Second) - defer cancel() - - client := GetDeezerClient() - track, err := client.SearchByISRC(ctx, isrc) - if err != nil { - if isDownloadCancelled(itemID) { - return "", ErrDownloadCancelled - } - return "", err - } - if isDownloadCancelled(itemID) { - return "", ErrDownloadCancelled - } - - result := buildDeezerISRCSearchResult(track) - return marshalJSONString(result) -} - -func buildDeezerExtendedMetadataResult(metadata *AlbumExtendedMetadata) map[string]string { - if metadata == nil { - return map[string]string{ - "genre": "", - "label": "", - "copyright": "", - } - } - - return map[string]string{ - "genre": metadata.Genre, - "label": metadata.Label, - "copyright": metadata.Copyright, - } -} - -func buildDeezerISRCSearchResult(track *TrackMetadata) map[string]any { - if track == nil { - return map[string]any{} - } - - result := map[string]any{ - "spotify_id": track.SpotifyID, - "artists": track.Artists, - "name": track.Name, - "album_name": track.AlbumName, - "album_artist": track.AlbumArtist, - "duration_ms": track.DurationMS, - "images": track.Images, - "release_date": track.ReleaseDate, - "track_number": track.TrackNumber, - "total_tracks": track.TotalTracks, - "disc_number": track.DiscNumber, - "total_discs": track.TotalDiscs, - "external_urls": track.ExternalURL, - "isrc": track.ISRC, - "album_id": track.AlbumID, - "artist_id": track.ArtistID, - "album_type": track.AlbumType, - "composer": track.Composer, - } - - if deezerID := strings.TrimSpace(strings.TrimPrefix(track.SpotifyID, "deezer:")); deezerID != "" { - result["id"] = deezerID - result["track_id"] = deezerID - result["success"] = true - } - - return result -} - -func ConvertSpotifyToDeezer(resourceType, spotifyID string) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - songlink := NewSongLinkClient() - deezerClient := GetDeezerClient() - - if resourceType == "track" { - deezerID, err := songlink.GetDeezerIDFromSpotify(spotifyID) - if err != nil { - return "", fmt.Errorf("could not find Deezer equivalent: %w", err) - } - - trackResp, err := deezerClient.GetTrack(ctx, deezerID) - if err != nil { - return "", fmt.Errorf("failed to fetch Deezer metadata: %w", err) - } - - return marshalJSONString(trackResp) - } - - if resourceType == "album" { - deezerID, err := songlink.GetDeezerAlbumIDFromSpotify(spotifyID) - if err != nil { - return "", fmt.Errorf("could not find Deezer album: %w", err) - } - - albumResp, err := deezerClient.GetAlbum(ctx, deezerID) - if err != nil { - return "", fmt.Errorf("failed to fetch Deezer album metadata: %w", err) - } - - return marshalJSONString(albumResp) - } - - return "", fmt.Errorf("spotify to Deezer conversion only supported for tracks and albums: please search by name for %s", resourceType) -} - -func GetSpotifyIDFromDeezerTrack(deezerTrackID string) (string, error) { - client := NewSongLinkClient() - return client.GetSpotifyIDFromDeezer(deezerTrackID) -} - -func GetTidalURLFromDeezerTrack(deezerTrackID string) (string, error) { - client := NewSongLinkClient() - return client.GetTidalURLFromDeezer(deezerTrackID) -} diff --git a/go_backend/exports_deezer_metadata_test.go b/go_backend/exports_deezer_metadata_test.go deleted file mode 100644 index 763965fc..00000000 --- a/go_backend/exports_deezer_metadata_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package gobackend - -import "testing" - -func TestBuildDeezerExtendedMetadataResultHandlesNil(t *testing.T) { - result := buildDeezerExtendedMetadataResult(nil) - - if result["genre"] != "" { - t.Fatalf("expected empty genre, got %q", result["genre"]) - } - if result["label"] != "" { - t.Fatalf("expected empty label, got %q", result["label"]) - } - if result["copyright"] != "" { - t.Fatalf("expected empty copyright, got %q", result["copyright"]) - } -} - -func TestBuildDeezerExtendedMetadataResultIncludesCopyright(t *testing.T) { - result := buildDeezerExtendedMetadataResult(&AlbumExtendedMetadata{ - Genre: "Rock", - Label: "EMI", - Copyright: "(C) Queen", - }) - - if result["genre"] != "Rock" { - t.Fatalf("unexpected genre: %q", result["genre"]) - } - if result["label"] != "EMI" { - t.Fatalf("unexpected label: %q", result["label"]) - } - if result["copyright"] != "(C) Queen" { - t.Fatalf("unexpected copyright: %q", result["copyright"]) - } -} - -func TestBuildDeezerISRCSearchResultAddsCompatibilityIDs(t *testing.T) { - result := buildDeezerISRCSearchResult(&TrackMetadata{ - SpotifyID: "deezer:3135556", - Name: "Love Of My Life", - Artists: "Queen", - AlbumName: "A Night at the Opera", - ISRC: "GBUM71029604", - ReleaseDate: "1975-11-21", - }) - - if result["spotify_id"] != "deezer:3135556" { - t.Fatalf("unexpected spotify_id: %v", result["spotify_id"]) - } - if result["id"] != "3135556" { - t.Fatalf("unexpected id: %v", result["id"]) - } - if result["track_id"] != "3135556" { - t.Fatalf("unexpected track_id: %v", result["track_id"]) - } - if result["success"] != true { - t.Fatalf("expected success=true, got %v", result["success"]) - } -} diff --git a/go_backend/exports_download.go b/go_backend/exports_download.go deleted file mode 100644 index cffd6fc4..00000000 --- a/go_backend/exports_download.go +++ /dev/null @@ -1,564 +0,0 @@ -package gobackend - -import ( - "context" - "encoding/json" - "strings" - "time" -) - -type DownloadRequest struct { - ContractVersion int `json:"contract_version,omitempty"` - ISRC string `json:"isrc"` - Service string `json:"service"` - DownloadProvider string `json:"download_provider,omitempty"` - ProviderTrackID string `json:"provider_track_id,omitempty"` - SpotifyID string `json:"spotify_id"` - TrackName string `json:"track_name"` - ArtistName string `json:"artist_name"` - AlbumName string `json:"album_name"` - AlbumArtist string `json:"album_artist"` - CoverURL string `json:"cover_url"` - CoverMaxDimension int `json:"cover_max_dimension,omitempty"` - OutputDir string `json:"output_dir"` - AlbumFolderTemplate string `json:"album_folder_template,omitempty"` - OutputPath string `json:"output_path,omitempty"` - OutputFD int `json:"output_fd,omitempty"` - OutputExt string `json:"output_ext,omitempty"` - FilenameFormat string `json:"filename_format"` - Quality string `json:"quality"` - EmbedMetadata bool `json:"embed_metadata"` - ArtistTagMode string `json:"artist_tag_mode,omitempty"` - EmbedLyrics bool `json:"embed_lyrics"` - EmbedReplayGain bool `json:"embed_replaygain,omitempty"` - PostProcessingEnabled bool `json:"post_processing_enabled,omitempty"` - TrackNumber int `json:"track_number"` - PlaylistPosition int `json:"playlist_position,omitempty"` - DiscNumber int `json:"disc_number"` - TotalTracks int `json:"total_tracks"` - TotalDiscs int `json:"total_discs,omitempty"` - ReleaseDate string `json:"release_date"` - ItemID string `json:"item_id"` - DurationMS int `json:"duration_ms"` - Source string `json:"source"` - Genre string `json:"genre,omitempty"` - Label string `json:"label,omitempty"` - Copyright string `json:"copyright,omitempty"` - Composer string `json:"composer,omitempty"` - Comment string `json:"comment,omitempty"` - Explicit bool `json:"explicit,omitempty"` - AlbumType string `json:"album_type,omitempty"` - UPC string `json:"upc,omitempty"` - TidalID string `json:"tidal_id,omitempty"` - QobuzID string `json:"qobuz_id,omitempty"` - DeezerID string `json:"deezer_id,omitempty"` - LyricsMode string `json:"lyrics_mode,omitempty"` - UseExtensions bool `json:"use_extensions,omitempty"` - UseFallback bool `json:"use_fallback,omitempty"` - RequiresContainerConversion bool `json:"requires_container_conversion,omitempty"` - AllowQualityVariant bool `json:"allow_quality_variant,omitempty"` - QualityVariant string `json:"quality_variant,omitempty"` - SongLinkRegion string `json:"songlink_region,omitempty"` - NetworkConcurrencyLimit int `json:"network_concurrency_limit,omitempty"` -} - -type DownloadResponse struct { - Success bool `json:"success"` - Message string `json:"message"` - FilePath string `json:"file_path,omitempty"` - ResolvedAlbumFolder string `json:"resolved_album_folder,omitempty"` - ResolvedFileName string `json:"resolved_file_name,omitempty"` - ProviderTrackID string `json:"provider_track_id,omitempty"` - Error string `json:"error,omitempty"` - ErrorType string `json:"error_type,omitempty"` - RetryAfterSeconds int `json:"retry_after_seconds,omitempty"` - AlreadyExists bool `json:"already_exists,omitempty"` - ActualBitDepth int `json:"actual_bit_depth,omitempty"` - ActualSampleRate int `json:"actual_sample_rate,omitempty"` - AudioCodec string `json:"audio_codec,omitempty"` - ActualExtension string `json:"actual_extension,omitempty"` - ActualContainer string `json:"actual_container,omitempty"` - RequiresContainerConversion bool `json:"requires_container_conversion,omitempty"` - Service string `json:"service,omitempty"` - Title string `json:"title,omitempty"` - Artist string `json:"artist,omitempty"` - Album string `json:"album,omitempty"` - AlbumArtist string `json:"album_artist,omitempty"` - ReleaseDate string `json:"release_date,omitempty"` - TrackNumber int `json:"track_number,omitempty"` - DiscNumber int `json:"disc_number,omitempty"` - TotalTracks int `json:"total_tracks,omitempty"` - TotalDiscs int `json:"total_discs,omitempty"` - ISRC string `json:"isrc,omitempty"` - CoverURL string `json:"cover_url,omitempty"` - Genre string `json:"genre,omitempty"` - Label string `json:"label,omitempty"` - Copyright string `json:"copyright,omitempty"` - Composer string `json:"composer,omitempty"` - Comment string `json:"comment,omitempty"` - Explicit bool `json:"explicit,omitempty"` - AlbumType string `json:"album_type,omitempty"` - UPC string `json:"upc,omitempty"` - SkipMetadataEnrichment bool `json:"skip_metadata_enrichment,omitempty"` - LyricsLRC string `json:"lyrics_lrc,omitempty"` - DecryptionKey string `json:"decryption_key,omitempty"` - Decryption *DownloadDecryptionInfo `json:"decryption,omitempty"` -} - -type DownloadResult struct { - FilePath string - BitDepth int - SampleRate int - AudioCodec string - Title string - Artist string - Album string - ReleaseDate string - TrackNumber int - TotalTracks int - DiscNumber int - TotalDiscs int - ISRC string - CoverURL string - Genre string - Label string - Copyright string - Composer string - Comment string - Explicit bool - AlbumType string - UPC string - LyricsLRC string - DecryptionKey string - Decryption *DownloadDecryptionInfo - ActualExtension string - ActualContainer string - RequiresContainerConversion bool -} - -func buildDownloadedFileComment(sourceComment, providerComment string) string { - comment := strings.TrimSpace(sourceComment) - if comment != "" { - return comment - } - return strings.TrimSpace(providerComment) -} - -func buildDownloadSuccessResponse( - req DownloadRequest, - result DownloadResult, - service string, - message string, - filePath string, - alreadyExists bool, -) DownloadResponse { - title := strings.TrimSpace(req.TrackName) - if title == "" { - title = strings.TrimSpace(result.Title) - } - - artist := result.Artist - if artist == "" { - artist = req.ArtistName - } - - // Preserve requested release metadata when available so mixed-provider - // fallback downloads from the same source album do not get split into - // different albums just because Tidal/Qobuz report variant titles/dates. - album, releaseDate, trackNumber, discNumber := preferredReleaseMetadata( - req, - result.Album, - result.ReleaseDate, - result.TrackNumber, - result.DiscNumber, - ) - - isrc := result.ISRC - if isrc == "" { - isrc = req.ISRC - } - - genre := result.Genre - if genre == "" { - genre = req.Genre - } - - label := result.Label - if label == "" { - label = req.Label - } - - copyright := result.Copyright - if copyright == "" { - copyright = req.Copyright - } - - composer := result.Composer - if composer == "" { - composer = req.Composer - } - comment := buildDownloadedFileComment(req.Comment, result.Comment) - - albumType := result.AlbumType - if albumType == "" { - albumType = req.AlbumType - } - - upc := result.UPC - if upc == "" { - upc = req.UPC - } - - coverURL := strings.TrimSpace(req.CoverURL) - if coverURL == "" { - coverURL = strings.TrimSpace(result.CoverURL) - } - - return DownloadResponse{ - Success: true, - Message: message, - FilePath: filePath, - ResolvedFileName: resolvedDownloadFilename(req, result, filePath), - ProviderTrackID: req.ProviderTrackID, - AlreadyExists: alreadyExists, - ActualBitDepth: result.BitDepth, - ActualSampleRate: result.SampleRate, - AudioCodec: result.AudioCodec, - ActualExtension: result.ActualExtension, - ActualContainer: result.ActualContainer, - RequiresContainerConversion: result.RequiresContainerConversion, - Service: service, - Title: title, - Artist: artist, - Album: album, - AlbumArtist: req.AlbumArtist, - ReleaseDate: releaseDate, - TrackNumber: trackNumber, - TotalTracks: req.TotalTracks, - DiscNumber: discNumber, - TotalDiscs: req.TotalDiscs, - ISRC: isrc, - CoverURL: coverURL, - Genre: genre, - Label: label, - Copyright: copyright, - Composer: composer, - Comment: comment, - Explicit: result.Explicit || req.Explicit, - AlbumType: albumType, - UPC: upc, - LyricsLRC: result.LyricsLRC, - DecryptionKey: result.DecryptionKey, - Decryption: normalizeDownloadDecryptionInfo(result.Decryption, result.DecryptionKey), - } -} - -func shouldSkipQualityProbe(filePath string) bool { - path := strings.TrimSpace(filePath) - if path == "" { - return true - } - if strings.HasPrefix(path, "/proc/self/fd/") { - return true - } - // Content URI and other non-filesystem schemes cannot be read directly by os.Open. - if strings.Contains(path, "://") { - return true - } - return false -} - -func enrichResultQualityFromFile(result *DownloadResult) { - if result == nil { - return - } - - path := strings.TrimSpace(result.FilePath) - if shouldSkipQualityProbe(path) { - if strings.HasPrefix(path, "/proc/self/fd/") { - LogDebug("Download", "Skipping quality probe for ephemeral SAF FD output: %s", path) - } - return - } - - quality, qErr := GetAudioQuality(path) - if qErr == nil { - result.BitDepth = quality.BitDepth - result.SampleRate = quality.SampleRate - result.AudioCodec = quality.Codec - if quality.Codec != "" { - GoLog("[Download] Actual quality from file: %s %d-bit/%dHz\n", quality.Codec, quality.BitDepth, quality.SampleRate) - } else { - GoLog("[Download] Actual quality from file: %d-bit/%dHz\n", quality.BitDepth, quality.SampleRate) - } - return - } - - LogDebug("Download", "Post-download quality probe unavailable for %s: %v", path, qErr) -} - -func applyExtendedMetadataFields( - genre *string, - label *string, - copyright *string, - extMeta *AlbumExtendedMetadata, -) { - if extMeta == nil { - return - } - - if genre != nil && *genre == "" && extMeta.Genre != "" { - *genre = extMeta.Genre - } - if label != nil && *label == "" && extMeta.Label != "" { - *label = extMeta.Label - } - if copyright != nil && *copyright == "" && extMeta.Copyright != "" { - *copyright = extMeta.Copyright - } -} - -func enrichExtraMetadataByISRC( - logPrefix string, - isrc string, - genre *string, - label *string, - copyright *string, -) { - normalizedISRC := strings.TrimSpace(isrc) - if normalizedISRC == "" { - return - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - extMeta, err := fetchDeezerExtendedMetadataByISRC(ctx, normalizedISRC) - if err != nil { - GoLog("[%s] Failed to get extended metadata from Deezer: %v\n", logPrefix, err) - } - applyExtendedMetadataFields(genre, label, copyright, extMeta) - - if genre != nil && *genre == "" { - musicBrainzGenre, err := fetchMusicBrainzGenreByISRC(normalizedISRC) - if err != nil { - GoLog("[%s] Failed to get genre from MusicBrainz: %v\n", logPrefix, err) - } else if musicBrainzGenre != "" { - *genre = musicBrainzGenre - GoLog("[%s] Genre fallback from MusicBrainz: %s\n", logPrefix, *genre) - } - } - - currentGenre := "" - currentLabel := "" - currentCopyright := "" - if genre != nil { - currentGenre = *genre - } - if label != nil { - currentLabel = *label - } - if copyright != nil { - currentCopyright = *copyright - } - if currentGenre != "" || currentLabel != "" || currentCopyright != "" { - GoLog("[%s] Extended metadata ready: genre=%s, label=%s, copyright=%s\n", logPrefix, currentGenre, currentLabel, currentCopyright) - } -} - -func enrichRequestExtendedMetadata(req *DownloadRequest) { - if req == nil { - return - } - - if req.ISRC == "" { - return - } - - if strings.TrimSpace(req.AlbumArtist) == "" { - albumArtist, err := fetchMusicBrainzAlbumArtistByISRC(req.ISRC, req.AlbumName) - if err != nil { - GoLog("[DownloadWithFallback] Failed to get album artist from MusicBrainz: %v\n", err) - } else if strings.TrimSpace(albumArtist) != "" { - req.AlbumArtist = strings.TrimSpace(albumArtist) - GoLog("[DownloadWithFallback] Album artist fallback from MusicBrainz: %s\n", req.AlbumArtist) - } - } - - if req.Genre == "" || req.Label == "" || req.Copyright == "" { - enrichExtraMetadataByISRC( - "DownloadWithFallback", - req.ISRC, - &req.Genre, - &req.Label, - &req.Copyright, - ) - } -} - -func applySongLinkRegionFromRequest(req *DownloadRequest) { - if req == nil { - return - } - SetSongLinkRegion(req.SongLinkRegion) -} - -// DownloadByStrategy routes all download requests through extension providers. -func DownloadByStrategy(requestJSON string) (string, error) { - var req DownloadRequest - if err := json.Unmarshal([]byte(requestJSON), &req); err != nil { - return errorResponse("Invalid request: " + err.Error()) - } - normalizedBytes, err := json.Marshal(req) - if err != nil { - return errorResponse("Invalid request: " + err.Error()) - } - normalizedJSON := string(normalizedBytes) - - if req.UseExtensions { - resp, err := DownloadWithExtensionsJSON(normalizedJSON) - if err != nil { - return errorResponse(err.Error()) - } - return resp, nil - } - - return errorResponse("Extension providers are disabled; built-in download providers have been retired") -} - -func GetAllDownloadProgress() string { - return GetMultiProgress() -} - -func GetAllDownloadProgressDelta(sinceSeq int64) string { - return GetMultiProgressDelta(sinceSeq) -} - -func WaitForAllDownloadProgressDelta(sinceSeq, timeoutMs int64) string { - return WaitForMultiProgressDelta(sinceSeq, timeoutMs) -} - -func InitItemProgress(itemID string) { - StartItemProgress(itemID) -} - -func ClearItemProgress(itemID string) { - RemoveItemProgress(itemID) -} - -func CancelDownload(itemID string) { - cancelDownload(itemID) -} - -// CancelAllActiveDownloads is a lifecycle safety valve for platforms that are -// about to suspend the process. It only cancels entries with live work and -// does not create cancellation flags for queued/future items. -func CancelAllActiveDownloads() string { - itemIDs := cancelAllActiveDownloads() - payload, err := json.Marshal(itemIDs) - if err != nil { - return "[]" - } - return string(payload) -} - -// ResetDownloadCancel drops a pre-registered cancellation flag for an item -// with no active download, so a user-initiated retry does not consume a stale -// cancel and abort instantly. Entries with live references are left alone. -func ResetDownloadCancel(itemID string) { - resetDownloadCancel(itemID) -} - -func CleanupConnections() { - CloseIdleConnections() -} - -func errorResponse(msg string) (string, error) { - errorType := classifyDownloadErrorType(msg) - - resp := DownloadResponse{ - Success: false, - Error: msg, - ErrorType: errorType, - } - s, _ := marshalJSONString(resp) - return s, nil -} - -func classifyDownloadErrorType(msg string) string { - lowerMsg := strings.ToLower(msg) - - if strings.Contains(lowerMsg, "isp blocking") || - strings.Contains(lowerMsg, "try using vpn") || - strings.Contains(lowerMsg, "change dns") { - return "isp_blocked" - } else if strings.Contains(lowerMsg, "cancel") { - return "cancelled" - } else if strings.Contains(lowerMsg, "verification_required") || - strings.Contains(lowerMsg, "session is not authenticated") || - strings.Contains(lowerMsg, "signed session is not authenticated") || - strings.Contains(lowerMsg, "signed session expired") { - return "verification_required" - } else if strings.Contains(lowerMsg, "byoa_provider_reauth_required") || - strings.Contains(lowerMsg, "reauth_provider") { - return "provider_reauth_required" - } else if strings.Contains(lowerMsg, "request_auth_invalid") { - return "request_auth_invalid" - } else if strings.Contains(lowerMsg, "provider_auth_failed") { - return "provider_auth_failed" - } else if strings.Contains(lowerMsg, "provider_unavailable") { - return "provider_unavailable" - } else if strings.Contains(lowerMsg, "rate limit") || - messageHasHTTPStatusCode(lowerMsg, "429") || - strings.Contains(lowerMsg, "too many requests") { - return "rate_limit" - } else if strings.Contains(lowerMsg, "permission") || - strings.Contains(lowerMsg, "operation not permitted") || - strings.Contains(lowerMsg, "access denied") || - strings.Contains(lowerMsg, "failed to create file") || - strings.Contains(lowerMsg, "failed to create directory") { - return "permission" - } else if strings.Contains(lowerMsg, "not found") || - strings.Contains(lowerMsg, "not available") || - strings.Contains(lowerMsg, "no results") || - strings.Contains(lowerMsg, "track not found") || - strings.Contains(lowerMsg, "all services failed") { - return "not_found" - } else if strings.Contains(lowerMsg, "network") || - strings.Contains(lowerMsg, "connection") || - strings.Contains(lowerMsg, "timeout") || - strings.Contains(lowerMsg, "dial") { - return "network" - } - - return "unknown" -} - -// isOutputStorageWriteFailure distinguishes an unwritable destination from a -// provider-specific failure. Provider fallback cannot repair the former: all -// providers receive the same output path, so continuing only delays the -// storage fallback and can replace the useful permission error with an -// unrelated error from the last provider. -func isOutputStorageWriteFailure(errorType, message string) bool { - if strings.EqualFold(strings.TrimSpace(errorType), "permission") { - return true - } - lowerMsg := strings.ToLower(strings.TrimSpace(message)) - if lowerMsg == "" { - return false - } - return strings.Contains(lowerMsg, "operation not permitted") || - strings.Contains(lowerMsg, "permission denied") || - strings.Contains(lowerMsg, "read-only file system") || - strings.Contains(lowerMsg, "failed to create file") || - strings.Contains(lowerMsg, "failed to create directory") -} - -func messageHasHTTPStatusCode(lowerMsg, code string) bool { - return strings.Contains(lowerMsg, "http "+code) || - strings.Contains(lowerMsg, "http status "+code) || - strings.Contains(lowerMsg, "status "+code) || - strings.Contains(lowerMsg, code+" for ") || - strings.Contains(lowerMsg, code+":") || - strings.Contains(lowerMsg, code+";") -} diff --git a/go_backend/exports_extension_wrappers_supplement_test.go b/go_backend/exports_extension_wrappers_supplement_test.go deleted file mode 100644 index 82270704..00000000 --- a/go_backend/exports_extension_wrappers_supplement_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package gobackend - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "testing" -) - -func TestExtensionPackageExportWrappers(t *testing.T) { - dir := t.TempDir() - extensionsDir := filepath.Join(dir, "extensions") - dataDir := filepath.Join(dir, "data") - if err := InitExtensionSystem(extensionsDir, dataDir); err != nil { - t.Fatalf("InitExtensionSystem: %v", err) - } - CleanupExtensions() - defer CleanupExtensions() - - js := ` -registerExtension({ - initialize: function(settings) { this.settings = settings || {}; }, - cleanup: function() {}, - doAction: function() { return { message: "wrapped", setting_updates: { quality: "lossless" } }; }, - searchTracks: function() { return { tracks: [], total: 0 }; }, - fetchLyrics: function() { return { syncType: "UNSYNCED", lines: [{ words: "hello" }] }; }, - getDownloadUrl: function() { return { url: "https://example.test/a.flac" }; } -}); -` - pkgV1 := filepath.Join(dir, "wrapper-ext-v1.spotiflac-ext") - pkgV2 := filepath.Join(dir, "wrapper-ext-v2.spotiflac-ext") - createTestExtensionPackage(t, pkgV1, "wrapper-ext", "1.0.0", js, nil) - createTestExtensionPackage(t, pkgV2, "wrapper-ext", "1.1.0", js, nil) - - loadedJSON, err := LoadExtensionFromPath(pkgV1) - if err != nil || !strings.Contains(loadedJSON, "wrapper-ext") { - t.Fatalf("LoadExtensionFromPath = %q/%v", loadedJSON, err) - } - if installedJSON, err := GetInstalledExtensions(); err != nil || !strings.Contains(installedJSON, "wrapper-ext") { - t.Fatalf("GetInstalledExtensions = %q/%v", installedJSON, err) - } - if err := SetExtensionEnabledByID("wrapper-ext", true); err != nil { - t.Fatalf("SetExtensionEnabledByID true: %v", err) - } - if actionJSON, err := InvokeExtensionActionJSON("wrapper-ext", "doAction"); err != nil || !strings.Contains(actionJSON, "wrapped") { - t.Fatalf("InvokeExtensionActionJSON = %q/%v", actionJSON, err) - } - if upgradeJSON, err := CheckExtensionUpgradeFromPath(pkgV2); err != nil || !strings.Contains(upgradeJSON, `"can_upgrade":true`) { - t.Fatalf("CheckExtensionUpgradeFromPath = %q/%v", upgradeJSON, err) - } - if upgradedJSON, err := UpgradeExtensionFromPath(pkgV2); err != nil || !strings.Contains(upgradedJSON, "1.1.0") { - t.Fatalf("UpgradeExtensionFromPath = %q/%v", upgradedJSON, err) - } - if err := SetExtensionEnabledByID("wrapper-ext", false); err != nil { - t.Fatalf("SetExtensionEnabledByID false: %v", err) - } - if err := UnloadExtensionByID("wrapper-ext"); err != nil { - t.Fatalf("UnloadExtensionByID: %v", err) - } - - dirExt := filepath.Join(extensionsDir, "wrapper-dir-ext") - if err := createDirectoryExtension(dirExt, "wrapper-dir-ext", "1.0.0"); err != nil { - t.Fatalf("create directory extension: %v", err) - } - if loadedDirJSON, err := LoadExtensionsFromDir(extensionsDir); err != nil || !strings.Contains(loadedDirJSON, "wrapper-dir-ext") { - t.Fatalf("LoadExtensionsFromDir = %q/%v", loadedDirJSON, err) - } - if err := RemoveExtensionByID("wrapper-dir-ext"); err != nil { - t.Fatalf("RemoveExtensionByID: %v", err) - } -} - -func createDirectoryExtension(dir, name, version string) error { - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - manifest := fmt.Sprintf(`{"name":%q,"displayName":%q,"version":%q,"description":"Directory wrapper extension","type":["metadata_provider"],"permissions":{}}`, name, name, version) - if err := os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(manifest), 0600); err != nil { - return err - } - return os.WriteFile(filepath.Join(dir, "index.js"), []byte(`registerExtension({searchTracks:function(){return {tracks:[], total:0};}});`), 0600) -} diff --git a/go_backend/exports_extensions.go b/go_backend/exports_extensions.go deleted file mode 100644 index 29a5c8cd..00000000 --- a/go_backend/exports_extensions.go +++ /dev/null @@ -1,1082 +0,0 @@ -package gobackend - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "path/filepath" - "strings" - "time" - - "github.com/dop251/goja" -) - -func normalizeExtensionTrackMetadataMap( - track ExtTrackMetadata, - fallbackCover string, - fallbackTrackNumber int, -) map[string]any { - coverURL := track.ResolvedCoverURL() - if coverURL == "" { - coverURL = fallbackCover - } - - trackNum := track.TrackNumber - if trackNum == 0 && fallbackTrackNumber > 0 { - trackNum = fallbackTrackNumber - } - - return map[string]any{ - "id": track.ID, - "name": track.Name, - "artists": track.Artists, - "album_name": track.AlbumName, - "album_artist": track.AlbumArtist, - "album_id": track.AlbumID, - "album_url": track.AlbumURL, - "artist_id": track.ArtistID, - "artist_url": track.ArtistURL, - "external_urls": track.ExternalURL, - "duration_ms": track.DurationMS, - "images": coverURL, - "cover_url": coverURL, - "preview_url": track.PreviewURL, - "release_date": track.ReleaseDate, - "track_number": trackNum, - "total_tracks": track.TotalTracks, - "disc_number": track.DiscNumber, - "total_discs": track.TotalDiscs, - "isrc": track.ISRC, - "provider_id": track.ProviderID, - "item_type": track.ItemType, - "album_type": track.AlbumType, - "spotify_id": track.SpotifyID, - "external_links": track.ExternalLinks, - "genre": track.Genre, - "label": track.Label, - "copyright": track.Copyright, - "composer": track.Composer, - "comment": track.Comment, - "audio_quality": track.AudioQuality, - "audio_modes": track.AudioModes, - "explicit": track.Explicit, - "upc": track.UPC, - } -} - -func normalizeExtensionAlbumInfoMap(album *ExtAlbumMetadata) map[string]any { - if album == nil { - return map[string]any{} - } - - return map[string]any{ - "id": album.ID, - "name": album.Name, - "artists": album.Artists, - "artist_id": album.ArtistID, - "images": album.CoverURL, - "cover_url": album.CoverURL, - "header_image": album.HeaderImage, - "header_video": album.HeaderVideo, - "release_date": album.ReleaseDate, - "total_tracks": album.TotalTracks, - "album_type": album.AlbumType, - "audio_traits": album.AudioTraits, - "provider_id": album.ProviderID, - } -} - -func normalizeExtensionArtistAlbumMap(album ExtAlbumMetadata) map[string]any { - return map[string]any{ - "id": album.ID, - "name": album.Name, - "artists": album.Artists, - "images": album.CoverURL, - "cover_url": album.CoverURL, - "release_date": album.ReleaseDate, - "total_tracks": album.TotalTracks, - "album_type": album.AlbumType, - "provider_id": album.ProviderID, - } -} - -func getExtensionProviderMetadataResponse( - providerID, - resourceType, - resourceID string, -) (map[string]any, error) { - manager := getExtensionManager() - ext, err := manager.GetExtension(providerID) - if err != nil { - return nil, err - } - - if !ext.Manifest.IsMetadataProvider() { - return nil, fmt.Errorf("extension '%s' is not a metadata provider", providerID) - } - if !ext.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", providerID) - } - - provider := newExtensionProviderWrapper(ext) - - switch resourceType { - case "track": - track, err := provider.GetTrack(resourceID) - if err != nil { - return nil, err - } - if track == nil { - return nil, fmt.Errorf("track not found") - } - return map[string]any{ - "track": normalizeExtensionTrackMetadataMap(*track, "", 0), - }, nil - case "album": - album, err := provider.GetAlbum(resourceID) - if err != nil { - return nil, err - } - if album == nil { - return nil, fmt.Errorf("album not found") - } - - tracks := make([]map[string]any, len(album.Tracks)) - for i, track := range album.Tracks { - tracks[i] = normalizeExtensionTrackMetadataMap(track, album.CoverURL, i+1) - } - - return map[string]any{ - "album_info": normalizeExtensionAlbumInfoMap(album), - "track_list": tracks, - }, nil - case "playlist": - playlist, err := provider.GetPlaylist(resourceID) - if err != nil { - return nil, err - } - if playlist == nil { - return nil, fmt.Errorf("playlist not found") - } - - tracks := make([]map[string]any, len(playlist.Tracks)) - for i, track := range playlist.Tracks { - tracks[i] = normalizeExtensionTrackMetadataMap(track, playlist.CoverURL, i+1) - } - - return map[string]any{ - "playlist_info": map[string]any{ - "id": playlist.ID, - "name": playlist.Name, - "images": playlist.CoverURL, - "cover_url": playlist.CoverURL, - "header_image": playlist.HeaderImage, - "header_video": playlist.HeaderVideo, - "provider_id": playlist.ProviderID, - "owner": map[string]any{ - "name": playlist.Artists, - "images": playlist.CoverURL, - }, - }, - "track_list": tracks, - }, nil - case "artist": - artist, err := provider.GetArtist(resourceID) - if err != nil { - return nil, err - } - if artist == nil { - return nil, fmt.Errorf("artist not found") - } - - albums := make([]map[string]any, len(artist.Albums)) - for i, album := range artist.Albums { - albums[i] = normalizeExtensionArtistAlbumMap(album) - } - - response := map[string]any{ - "artist_info": map[string]any{ - "id": artist.ID, - "name": artist.Name, - "images": firstNonEmptyTrimmed(artist.HeaderImage, artist.ImageURL), - "cover_url": artist.ImageURL, - "header_image": artist.HeaderImage, - "header_video": artist.HeaderVideo, - "provider_id": artist.ProviderID, - }, - "albums": albums, - } - - if len(artist.Releases) > 0 { - releases := make([]map[string]any, len(artist.Releases)) - for i, release := range artist.Releases { - releases[i] = normalizeExtensionArtistAlbumMap(release) - } - response["releases"] = releases - } - - if artist.Listeners > 0 { - artistInfo := response["artist_info"].(map[string]any) - artistInfo["listeners"] = artist.Listeners - } - - if len(artist.TopTracks) > 0 { - topTracks := make([]map[string]any, len(artist.TopTracks)) - for i, track := range artist.TopTracks { - topTracks[i] = normalizeExtensionTrackMetadataMap(track, artist.ImageURL, i+1) - } - response["top_tracks"] = topTracks - } - - return response, nil - default: - return nil, fmt.Errorf("unsupported provider resource type: %s", resourceType) - } -} - -func firstNonEmptyTrimmed(values ...string) string { - for _, value := range values { - if trimmed := strings.TrimSpace(value); trimmed != "" { - return trimmed - } - } - return "" -} - -func GetProviderMetadataJSON(providerID, resourceType, resourceID string) (string, error) { - trimmedProviderID := strings.TrimSpace(providerID) - if trimmedProviderID == "" { - return "", fmt.Errorf("empty provider ID") - } - - switch strings.ToLower(trimmedProviderID) { - case "deezer": - if response, ok, err := getEnabledExtensionProviderMetadataResponse(trimmedProviderID, resourceType, resourceID); ok || err != nil { - if err != nil { - return "", err - } - return marshalJSONString(response) - } - return GetDeezerMetadata(resourceType, resourceID) - default: - response, err := getExtensionProviderMetadataResponse(trimmedProviderID, resourceType, resourceID) - if err != nil { - return "", err - } - - return marshalJSONString(response) - } -} - -func getEnabledExtensionProviderMetadataResponse(providerID, resourceType, resourceID string) (map[string]any, bool, error) { - manager := getExtensionManager() - ext, err := manager.GetExtension(providerID) - if err != nil || ext == nil || !ext.Enabled || !ext.Manifest.IsMetadataProvider() { - return nil, false, nil - } - response, err := getExtensionProviderMetadataResponse(providerID, resourceType, resourceID) - if err != nil { - return nil, true, err - } - return response, true, nil -} - -func InitExtensionSystem(extensionsDir, dataDir string) error { - if !extensionStorageKeyConfigured() { - return fmt.Errorf("extension storage master key is not configured") - } - manager := getExtensionManager() - if err := manager.SetDirectories(extensionsDir, dataDir); err != nil { - return err - } - - settingsStore := GetExtensionSettingsStore() - if err := settingsStore.SetDataDir(dataDir); err != nil { - return err - } - globalLyricsCache.SetPersistencePath(filepath.Join(dataDir, ".lyrics_cache.json")) - - return nil -} - -func LoadExtensionsFromDir(dirPath string) (string, error) { - manager := getExtensionManager() - loaded, errors := manager.LoadExtensionsFromDirectory(dirPath) - - result := map[string]any{ - "loaded": loaded, - "errors": make([]string, len(errors)), - } - - for i, err := range errors { - result["errors"].([]string)[i] = err.Error() - } - - return marshalJSONString(result) -} - -func LoadExtensionFromPath(filePath string) (string, error) { - manager := getExtensionManager() - ext, err := manager.LoadExtensionFromFile(filePath) - if err != nil { - return "", err - } - - result := map[string]any{ - "id": ext.ID, - "name": ext.Manifest.Name, - "display_name": ext.Manifest.DisplayName, - "version": ext.Manifest.Version, - "enabled": ext.Enabled, - } - - return marshalJSONString(result) -} - -func UnloadExtensionByID(extensionID string) error { - manager := getExtensionManager() - return manager.UnloadExtension(extensionID) -} - -func RemoveExtensionByID(extensionID string) error { - manager := getExtensionManager() - return manager.RemoveExtension(extensionID) -} - -func UpgradeExtensionFromPath(filePath string) (string, error) { - manager := getExtensionManager() - ext, err := manager.UpgradeExtension(filePath) - if err != nil { - return "", err - } - - result := map[string]any{ - "id": ext.ID, - "display_name": ext.Manifest.DisplayName, - "version": ext.Manifest.Version, - "enabled": ext.Enabled, - } - - return marshalJSONString(result) -} - -func CheckExtensionUpgradeFromPath(filePath string) (string, error) { - manager := getExtensionManager() - return manager.CheckExtensionUpgradeJSON(filePath) -} - -func GetInstalledExtensions() (string, error) { - manager := getExtensionManager() - return manager.GetInstalledExtensionsJSON() -} - -func SetExtensionEnabledByID(extensionID string, enabled bool) error { - manager := getExtensionManager() - return manager.SetExtensionEnabled(extensionID, enabled) -} - -func SetProviderPriorityJSON(priorityJSON string) error { - var priority []string - if err := json.Unmarshal([]byte(priorityJSON), &priority); err != nil { - return err - } - - SetProviderPriority(priority) - return nil -} - -func GetProviderPriorityJSON() (string, error) { - priority := GetProviderPriority() - return marshalJSONString(priority) -} - -func SetExtensionFallbackProviderIDsJSON(providerIDsJSON string) error { - if strings.TrimSpace(providerIDsJSON) == "" { - SetExtensionFallbackProviderIDs(nil) - return nil - } - - var providerIDs []string - if err := json.Unmarshal([]byte(providerIDsJSON), &providerIDs); err != nil { - return err - } - - SetExtensionFallbackProviderIDs(providerIDs) - return nil -} - -func SetMetadataProviderPriorityJSON(priorityJSON string) error { - var priority []string - if err := json.Unmarshal([]byte(priorityJSON), &priority); err != nil { - return err - } - - SetMetadataProviderPriority(priority) - return nil -} - -func GetMetadataProviderPriorityJSON() (string, error) { - priority := GetMetadataProviderPriority() - return marshalJSONString(priority) -} - -func GetExtensionSettingsJSON(extensionID string) (string, error) { - store := GetExtensionSettingsStore() - settings := store.GetAll(extensionID) - - return marshalJSONString(settings) -} - -func SetExtensionSettingsJSON(extensionID, settingsJSON string) error { - var settings map[string]any - if err := json.Unmarshal([]byte(settingsJSON), &settings); err != nil { - return err - } - - store := GetExtensionSettingsStore() - if err := store.SetAll(extensionID, settings); err != nil { - return err - } - - manager := getExtensionManager() - return manager.InitializeExtension(extensionID, settings) -} - -func SearchTracksWithMetadataProvidersJSON(query string, limit int, includeExtensions bool) (string, error) { - manager := getExtensionManager() - tracks, err := manager.SearchTracksWithMetadataProviders(query, limit, includeExtensions) - if err != nil { - return "", err - } - - return marshalJSONString(tracks) -} - -func SearchTracksWithMetadataProviderJSON(providerID, query string, limit int) (string, error) { - manager := getExtensionManager() - tracks, err := manager.SearchTracksWithMetadataProvider(providerID, query, limit) - if err != nil { - return "", err - } - - return marshalJSONString(tracks) -} - -func preflightExtensionDownloadSession(extensionID string) (bool, error) { - extensionID = strings.TrimSpace(extensionID) - if extensionID == "" { - return false, nil - } - - ext, err := getExtensionManager().GetExtension(extensionID) - if err != nil || ext == nil || !ext.Enabled || ext.Manifest == nil || - !ext.Manifest.IsDownloadProvider() || ext.Manifest.SignedSession == nil { - return false, nil - } - - if _, err := ext.lockReadyVM(); err != nil { - return false, err - } - runtime := ext.runtime - ext.VMMu.Unlock() - if runtime == nil { - return false, fmt.Errorf("extension '%s' runtime is unavailable", extensionID) - } - - // Preflight touches only the runtime's thread-safe HTTP/session state. Do - // not hold the Goja VM lock across bootstrap network I/O: metadata/status - // calls on the same extension must remain responsive while auth is slow. - return runtime.preflightSignedSession() -} - -func DownloadWithExtensionsJSON(requestJSON string) (string, error) { - var req DownloadRequest - if err := json.Unmarshal([]byte(requestJSON), &req); err != nil { - return "", fmt.Errorf("invalid request: %w", err) - } - applySongLinkRegionFromRequest(&req) - defer closeOwnedOutputFD(req.OutputFD) - if req.ItemID != "" { - initDownloadCancel(req.ItemID) - defer clearDownloadCancel(req.ItemID) - if isDownloadCancelled(req.ItemID) { - return "", ErrDownloadCancelled - } - } - - req.TrackName = strings.TrimSpace(req.TrackName) - req.ArtistName = strings.TrimSpace(req.ArtistName) - req.AlbumName = strings.TrimSpace(req.AlbumName) - req.AlbumArtist = strings.TrimSpace(req.AlbumArtist) - req.OutputDir = strings.TrimSpace(req.OutputDir) - req.OutputPath = strings.TrimSpace(req.OutputPath) - req.OutputExt = strings.TrimSpace(req.OutputExt) - if req.OutputPath == "" && req.OutputFD <= 0 && req.OutputDir != "" { - AddAllowedDownloadDir(req.OutputDir) - } - - sessionProvider := strings.TrimSpace(req.Service) - if sessionProvider == "" { - sessionProvider = strings.TrimSpace(req.Source) - } - if req.ItemID != "" { - StartItemProgress(req.ItemID) - SetItemPreparingStage(req.ItemID, "checking_session") - } - preflightStartedAt := time.Now() - verificationRequired, preflightErr := preflightExtensionDownloadSession(sessionProvider) - if preflightErr != nil { - message := fmt.Sprintf("Could not start verification for %s: %v", sessionProvider, preflightErr) - GoLog("[DownloadWithExtensions] Signed-session preflight for %s failed after %s: %v\n", sessionProvider, time.Since(preflightStartedAt).Round(time.Millisecond), preflightErr) - if req.ItemID != "" { - RemoveItemProgress(req.ItemID) - } - return marshalJSONString(&DownloadResponse{ - Success: false, - Error: message, - ErrorType: classifyDownloadErrorType(message), - Service: sessionProvider, - }) - } else if verificationRequired { - GoLog("[DownloadWithExtensions] Signed-session verification required for %s after %s; skipping metadata preparation\n", sessionProvider, time.Since(preflightStartedAt).Round(time.Millisecond)) - cacheUnpreparedDownloadRequest(downloadPreparationKey(req), req) - if req.ItemID != "" { - RemoveItemProgress(req.ItemID) - } - return marshalJSONString(&DownloadResponse{ - Success: false, - Error: "Verification required before download", - ErrorType: "verification_required", - Service: sessionProvider, - }) - } else if sessionProvider != "" { - LogDebug("DownloadWithExtensions", "Signed-session preflight ready for %s in %s", sessionProvider, time.Since(preflightStartedAt).Round(time.Millisecond)) - } - - if isDownloadCancelled(req.ItemID) { - if req.ItemID != "" { - RemoveItemProgress(req.ItemID) - } - return "", ErrDownloadCancelled - } - - result, err := DownloadWithExtensionFallback(req) - if err != nil { - if req.ItemID != "" { - RemoveItemProgress(req.ItemID) - } - return "", err - } - if req.ItemID != "" && (result == nil || !result.Success) { - RemoveItemProgress(req.ItemID) - } - - return marshalJSONString(result) -} - -func CleanupExtensions() { - manager := getExtensionManager() - manager.UnloadAllExtensions() -} - -func InvokeExtensionActionJSON(extensionID, actionName string) (string, error) { - manager := getExtensionManager() - result, err := manager.InvokeAction(extensionID, actionName) - if err != nil { - return "", err - } - - return marshalJSONString(result) -} - -func GetExtensionPendingAuthJSON(extensionID string) (string, error) { - req, err := ensureExtensionPendingAuthRequest(extensionID) - if err != nil { - return "", err - } - if req == nil { - return "", nil - } - - result := map[string]any{ - "extension_id": req.ExtensionID, - "auth_url": req.AuthURL, - "callback_url": req.CallbackURL, - } - - return marshalJSONString(result) -} - -func ensureExtensionPendingAuthRequest(extensionID string) (*PendingAuthRequest, error) { - extensionID = strings.TrimSpace(extensionID) - if extensionID == "" { - return nil, nil - } - - if req := GetPendingAuthRequest(extensionID); req != nil { - if time.Since(req.CreatedAt) < pendingAuthRequestTTL { - return req, nil - } - // The cached challenge is stale (e.g. verification was requested - // while the app was backgrounded and never completed); serving it - // would send the user to an expired page. Start a fresh one. - ClearPendingAuthRequest(extensionID) - } - - manager := getExtensionManager() - ext, err := manager.GetExtension(extensionID) - if err != nil || ext == nil || !ext.Enabled || ext.Manifest == nil || ext.Manifest.SignedSession == nil { - return nil, nil - } - - if err := ext.ensureRuntimeReady(); err != nil { - return nil, err - } - if ext.runtime == nil { - return nil, fmt.Errorf("extension '%s' runtime is unavailable", extensionID) - } - - verificationRequired, err := ext.runtime.preflightSignedSession() - if err != nil { - return nil, err - } - if !verificationRequired { - return nil, nil - } - return GetPendingAuthRequest(extensionID), nil -} - -func SetExtensionAuthCodeByID(extensionID, authCode string) { - SetExtensionAuthCode(extensionID, authCode) -} - -func SetExtensionSessionGrantByID(extensionID, grant string) { - setPendingSignedSessionGrant(extensionID, grant) -} - -func SetExtensionTokensByID(extensionID, accessToken, refreshToken string, expiresIn int) { - var expiresAt time.Time - if expiresIn > 0 { - expiresAt = time.Now().Add(time.Duration(expiresIn) * time.Second) - } - SetExtensionTokens(extensionID, accessToken, refreshToken, expiresAt) -} - -func ClearExtensionPendingAuthByID(extensionID string) { - ClearPendingAuthRequest(extensionID) -} - -func IsExtensionAuthenticatedByID(extensionID string) bool { - extensionAuthStateMu.RLock() - defer extensionAuthStateMu.RUnlock() - - state, exists := extensionAuthState[extensionID] - if !exists { - return false - } - - if state.IsAuthenticated && !state.ExpiresAt.IsZero() && time.Now().After(state.ExpiresAt) { - return false - } - - return state.IsAuthenticated -} - -func GetAllPendingAuthRequestsJSON() (string, error) { - pendingAuthRequestsMu.RLock() - defer pendingAuthRequestsMu.RUnlock() - - requests := make([]map[string]any, 0, len(pendingAuthRequests)) - for _, req := range pendingAuthRequests { - requests = append(requests, map[string]any{ - "extension_id": req.ExtensionID, - "auth_url": req.AuthURL, - "callback_url": req.CallbackURL, - }) - } - - return marshalJSONString(requests) -} - -func GetPendingFFmpegCommandJSON(commandID string) (string, error) { - cmd := GetPendingFFmpegCommand(commandID) - if cmd == nil { - return "", nil - } - - result := map[string]any{ - "command_id": commandID, - "extension_id": cmd.ExtensionID, - "arguments": cmd.Arguments, - "input_path": cmd.InputPath, - "output_path": cmd.OutputPath, - } - - return marshalJSONString(result) -} - -func SetFFmpegCommandResultByID(commandID string, success bool, output, errorMsg string) { - SetFFmpegCommandResult(commandID, success, output, errorMsg) -} - -func GetAllPendingFFmpegCommandsJSON() (string, error) { - ffmpegCommandsMu.RLock() - defer ffmpegCommandsMu.RUnlock() - - commands := make([]map[string]any, 0) - for cmdID, cmd := range ffmpegCommands { - if !cmd.Completed && !cmd.Claimed { - commands = append(commands, map[string]any{ - "command_id": cmdID, - "extension_id": cmd.ExtensionID, - "arguments": cmd.Arguments, - }) - } - } - - return marshalJSONString(commands) -} - -// WaitForPendingFFmpegCommandsJSON blocks until work is available or the -// timeout elapses. Native command pumps use this instead of polling the bridge -// every 100 ms while still retaining GetAllPendingFFmpegCommandsJSON for older -// clients. -func WaitForPendingFFmpegCommandsJSON(timeoutMillis int64) (string, error) { - if timeoutMillis < 0 { - timeoutMillis = 0 - } - deadline := time.NewTimer(time.Duration(timeoutMillis) * time.Millisecond) - defer deadline.Stop() - - for { - ffmpegCommandsMu.Lock() - commands := make([]map[string]any, 0) - for cmdID, cmd := range ffmpegCommands { - if cmd.Completed || cmd.Claimed { - continue - } - cmd.Claimed = true - commands = append(commands, map[string]any{ - "command_id": cmdID, - "extension_id": cmd.ExtensionID, - "arguments": cmd.Arguments, - }) - } - ffmpegCommandsMu.Unlock() - if len(commands) > 0 { - return marshalJSONString(commands) - } - - select { - case <-ffmpegCommandQueued: - // A buffered notification can be stale if another command pump - // already consumed the work, so re-check until the deadline. - continue - case <-deadline.C: - return "[]", nil - } - } -} - -func EnrichTrackWithExtensionJSON(extensionID, trackJSON string) (string, error) { - manager := getExtensionManager() - ext, err := manager.GetExtension(extensionID) - if err != nil { - return trackJSON, nil - } - - if !ext.Manifest.IsMetadataProvider() { - return trackJSON, nil - } - - var track ExtTrackMetadata - if err := json.Unmarshal([]byte(trackJSON), &track); err != nil { - return trackJSON, fmt.Errorf("failed to parse track: %w", err) - } - - provider := newExtensionProviderWrapper(ext) - enrichedTrack, err := provider.EnrichTrack(&track) - if err != nil { - return trackJSON, nil - } - - jsonBytes, err := json.Marshal(enrichedTrack) - if err != nil { - return trackJSON, nil - } - - return string(jsonBytes), nil -} - -func CustomSearchWithExtensionJSON(extensionID, query string, optionsJSON string) (string, error) { - return CustomSearchWithExtensionJSONWithRequestID(extensionID, query, optionsJSON, "") -} - -func CustomSearchWithExtensionJSONWithRequestID(extensionID, query string, optionsJSON string, requestID string) (string, error) { - manager := getExtensionManager() - ext, err := manager.GetExtension(extensionID) - if err != nil { - return "", err - } - - if !ext.Manifest.HasCustomSearch() { - return "", fmt.Errorf("extension '%s' does not support custom search", extensionID) - } - - var options map[string]any - if optionsJSON != "" { - if err := json.Unmarshal([]byte(optionsJSON), &options); err != nil { - options = make(map[string]any) - } - } - - provider := newExtensionProviderWrapper(ext) - tracks, err := provider.CustomSearchForRequestID(query, options, requestID) - if err != nil { - return "", err - } - - result := make([]map[string]any, len(tracks)) - for i, track := range tracks { - result[i] = normalizeExtensionTrackMetadataMap(track, "", 0) - } - - return marshalJSONString(result) -} - -func HandleURLWithExtensionJSON(url string) (string, error) { - manager := getExtensionManager() - resultWithID, err := manager.HandleURLWithExtension(url) - if err != nil { - return "", err - } - - result := resultWithID.Result - extensionID := resultWithID.ExtensionID - - if result == nil { - return "", fmt.Errorf("extension %s failed to handle URL", extensionID) - } - - response := map[string]any{ - "type": result.Type, - "id": result.ID, - "extension_id": extensionID, - "name": result.Name, - "cover_url": result.CoverURL, - "header_image": result.HeaderImage, - "header_video": result.HeaderVideo, - } - - if result.Track != nil { - response["track"] = normalizeExtensionTrackMetadataMap(*result.Track, "", 0) - } - - if len(result.Tracks) > 0 { - tracks := make([]map[string]any, len(result.Tracks)) - for i, track := range result.Tracks { - tracks[i] = normalizeExtensionTrackMetadataMap(track, "", 0) - } - response["tracks"] = tracks - } - - if result.Album != nil { - response["album"] = map[string]any{ - "id": result.Album.ID, - "name": result.Album.Name, - "artists": result.Album.Artists, - "cover_url": result.Album.CoverURL, - "header_image": result.Album.HeaderImage, - "header_video": result.Album.HeaderVideo, - "audio_traits": result.Album.AudioTraits, - "release_date": result.Album.ReleaseDate, - "total_tracks": result.Album.TotalTracks, - "album_type": result.Album.AlbumType, - "provider_id": result.Album.ProviderID, - } - } - - if result.Artist != nil { - artistResponse := map[string]any{ - "id": result.Artist.ID, - "name": result.Artist.Name, - "image_url": result.Artist.ImageURL, - "header_image": result.Artist.HeaderImage, - "header_video": result.Artist.HeaderVideo, - "listeners": result.Artist.Listeners, - "provider_id": result.Artist.ProviderID, - } - - if len(result.Artist.Albums) > 0 { - albums := make([]map[string]any, len(result.Artist.Albums)) - for i, album := range result.Artist.Albums { - albumType := album.AlbumType - if albumType == "" { - albumType = "album" - } - albums[i] = map[string]any{ - "id": album.ID, - "name": album.Name, - "artists": album.Artists, - "images": album.CoverURL, - "cover_url": album.CoverURL, - "release_date": album.ReleaseDate, - "total_tracks": album.TotalTracks, - "album_type": albumType, - "provider_id": album.ProviderID, - } - } - artistResponse["albums"] = albums - } - - if len(result.Artist.Releases) > 0 { - releases := make([]map[string]any, len(result.Artist.Releases)) - for i, release := range result.Artist.Releases { - releaseType := release.AlbumType - if releaseType == "" { - releaseType = "album" - } - releases[i] = map[string]any{ - "id": release.ID, - "name": release.Name, - "artists": release.Artists, - "images": release.CoverURL, - "cover_url": release.CoverURL, - "release_date": release.ReleaseDate, - "total_tracks": release.TotalTracks, - "album_type": releaseType, - "provider_id": release.ProviderID, - } - } - artistResponse["releases"] = releases - } - - if len(result.Artist.TopTracks) > 0 { - topTracks := make([]map[string]any, len(result.Artist.TopTracks)) - for i, track := range result.Artist.TopTracks { - topTracks[i] = normalizeExtensionTrackMetadataMap(track, "", 0) - } - artistResponse["top_tracks"] = topTracks - } - - response["artist"] = artistResponse - } - - return marshalJSONString(response) -} - -func FindURLHandlerJSON(url string) string { - manager := getExtensionManager() - handler := manager.FindURLHandler(url) - if handler == nil { - return "" - } - return handler.extension.ID -} - -func RunPostProcessingV2JSON(inputJSON, metadataJSON string) (string, error) { - var metadata map[string]any - if metadataJSON != "" { - if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil { - metadata = make(map[string]any) - } - } - - var input PostProcessInput - if inputJSON != "" { - if err := json.Unmarshal([]byte(inputJSON), &input); err != nil { - input = PostProcessInput{} - } - } - - manager := getExtensionManager() - result, err := manager.RunPostProcessingV2(input, metadata) - if err != nil { - return "", err - } - - return marshalJSONString(result) -} - -func callExtensionFunctionJSON(extensionID, functionName string, timeout time.Duration) (string, error) { - return callExtensionFunctionJSONWithRequestID(extensionID, functionName, timeout, "") -} - -func callExtensionFunctionJSONWithRequestID(extensionID, functionName string, timeout time.Duration, requestID string) (string, error) { - manager := getExtensionManager() - ext, err := manager.GetExtension(extensionID) - if err != nil { - return "", err - } - - if !ext.Enabled { - return "", fmt.Errorf("extension '%s' is disabled", extensionID) - } - perf := newExtensionCallPerf(extensionID, functionName) - defer perf.finish() - initStartedAt := time.Now() - vm, err := ext.lockReadyVM() - if err != nil { - return "", err - } - perf.recordInit(time.Since(initStartedAt)) - defer ext.VMMu.Unlock() - requestCtx := context.Background() - if requestID != "" { - if ext.runtime != nil { - ext.runtime.setActiveRequestID(requestID) - defer ext.runtime.clearActiveRequestID() - } - requestCtx = initExtensionRequestCancel(requestID) - defer clearExtensionRequestCancel(requestID) - if isExtensionRequestCancelled(requestID) { - return "", ErrExtensionRequestCancelled - } - } - - jsStartedAt := time.Now() - result, err := runGojaCallWithTimeoutContextAndRecover(requestCtx, vm, func() (goja.Value, error) { - return invokeExtensionOrGlobal(vm, functionName) - }, timeout) - perf.recordJS(time.Since(jsStartedAt)) - if err != nil { - if IsRuntimeUnsafeError(err) { - quarantineRuntimeLocked(ext, vm, err) - } - if isExtensionRequestCancelled(requestID) || errors.Is(err, ErrExtensionRequestCancelled) { - return "", ErrExtensionRequestCancelled - } - return "", fmt.Errorf("%s failed: %w", functionName, err) - } - if isExtensionRequestCancelled(requestID) { - return "", ErrExtensionRequestCancelled - } - - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return "", fmt.Errorf("%s returned null", functionName) - } - - parseStartedAt := time.Now() - jsonBytes, err := json.Marshal(result) - perf.recordParse(time.Since(parseStartedAt)) - if err != nil { - return "", fmt.Errorf("failed to marshal result: %w", err) - } - perf.setPayloadBytes(len(jsonBytes)) - perf.setItems(countExtensionTopLevelItems(vm, result)) - - return string(jsonBytes), nil -} - -func GetExtensionHomeFeedJSON(extensionID string) (string, error) { - return callExtensionFunctionJSON(extensionID, "getHomeFeed", 60*time.Second) -} - -func GetExtensionHomeFeedJSONWithRequestID(extensionID, requestID string) (string, error) { - return callExtensionFunctionJSONWithRequestID(extensionID, "getHomeFeed", 60*time.Second, requestID) -} - -func CancelExtensionRequestJSON(requestID string) { - cancelExtensionRequest(requestID) -} diff --git a/go_backend/exports_library.go b/go_backend/exports_library.go deleted file mode 100644 index edc93128..00000000 --- a/go_backend/exports_library.go +++ /dev/null @@ -1,37 +0,0 @@ -package gobackend - -func SetLibraryCoverCacheDirJSON(cacheDir string) { - SetLibraryCoverCacheDir(cacheDir) -} - -func ScanLibraryFolderJSON(folderPath string) (string, error) { - return ScanLibraryFolder(folderPath) -} - -func ScanLibraryFolderToNDJSONFileJSON(folderPath, outputPath string) (int, error) { - return ScanLibraryFolderToNDJSONFile(folderPath, outputPath) -} - -func ScanLibraryFolderIncrementalJSON(folderPath, existingFilesJSON string) (string, error) { - return ScanLibraryFolderIncremental(folderPath, existingFilesJSON) -} - -func ScanLibraryFolderIncrementalFromSnapshotJSON(folderPath, snapshotPath string) (string, error) { - return ScanLibraryFolderIncrementalFromSnapshot(folderPath, snapshotPath) -} - -func GetLibraryScanProgressJSON() string { - return GetLibraryScanProgress() -} - -func CancelLibraryScanJSON() { - CancelLibraryScan() -} - -func ReadAudioMetadataJSON(filePath string) (string, error) { - return ReadAudioMetadata(filePath) -} - -func ReadAudioMetadataWithHintAndCoverCacheKeyJSON(filePath, displayName, coverCacheKey string) (string, error) { - return ReadAudioMetadataWithDisplayNameAndCoverCacheKey(filePath, displayName, coverCacheKey) -} diff --git a/go_backend/exports_lyrics.go b/go_backend/exports_lyrics.go deleted file mode 100644 index 079c60f8..00000000 --- a/go_backend/exports_lyrics.go +++ /dev/null @@ -1,172 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "os" - "strings" -) - -func GetLyricsLRC(spotifyID, trackName, artistName string, filePath string, durationMs int64) (string, error) { - if filePath != "" { - lyrics, err := ExtractLyrics(filePath) - if err == nil && rawLyricsHasUsableContent(lyrics) { - return lyrics, nil - } - return "", nil - } - - client := NewLyricsClient() - durationSec := float64(durationMs) / 1000.0 - lyricsData, err := client.FetchLyricsAllSources(spotifyID, trackName, artistName, durationSec) - if err != nil { - return "", err - } - - if lyricsData.Instrumental { - return "[instrumental:true]", nil - } - - lrcContent := convertToLRCWithMetadata(lyricsData, trackName, artistName) - return lrcContent, nil -} - -func GetLyricsLRCWithSource(spotifyID, trackName, artistName string, filePath string, durationMs int64) (string, error) { - if filePath != "" { - lyrics, err := ExtractLyrics(filePath) - if err == nil && rawLyricsHasUsableContent(lyrics) { - source := extractLyricsSourceFromLRC(lyrics) - if source == "" { - source = "Embedded" - } - result := map[string]any{ - "lyrics": lyrics, - "source": source, - "sync_type": "EMBEDDED", - "instrumental": isInstrumentalLyricsMarker(lyrics), - } - return marshalJSONString(result) - } - - result := map[string]any{ - "lyrics": "", - "source": "", - "sync_type": "", - "instrumental": false, - } - return marshalJSONString(result) - } - - client := NewLyricsClient() - durationSec := float64(durationMs) / 1000.0 - lyricsData, err := client.FetchLyricsAllSources(spotifyID, trackName, artistName, durationSec) - if err != nil { - return "", err - } - - lrcContent := "" - if lyricsData.Instrumental { - lrcContent = "[instrumental:true]" - } else { - lrcContent = convertToLRCWithMetadata(lyricsData, trackName, artistName) - } - - result := map[string]any{ - "lyrics": lrcContent, - "source": lyricsData.Source, - "sync_type": lyricsData.SyncType, - "instrumental": lyricsData.Instrumental, - } - return marshalJSONString(result) -} - -func EmbedLyricsToFile(filePath, lyrics string) (string, error) { - err := EmbedLyrics(filePath, lyrics) - if err != nil { - return errorResponse("Failed to embed lyrics: " + err.Error()) - } - - resp := map[string]any{ - "success": true, - "message": "Lyrics embedded successfully", - } - - s, _ := marshalJSONString(resp) - return s, nil -} - -func FetchAndSaveLyrics(trackName, artistName, spotifyID string, durationMs int64, outputPath string, audioFilePath string) error { - // If the audio file already has embedded lyrics or a sidecar .lrc, - // use those directly instead of making redundant network requests. - if audioFilePath != "" { - existing, err := ExtractLyrics(audioFilePath) - if err == nil && rawLyricsHasUsableContent(existing) { - if err := os.WriteFile(outputPath, []byte(existing), 0644); err != nil { - return fmt.Errorf("failed to write LRC file: %w", err) - } - GoLog("[Lyrics] Saved LRC from embedded/sidecar to: %s\n", outputPath) - return nil - } - } - - client := NewLyricsClient() - durationSec := float64(durationMs) / 1000.0 - - lyrics, err := client.FetchLyricsAllSources(spotifyID, trackName, artistName, durationSec) - if err != nil { - return fmt.Errorf("lyrics not found: %w", err) - } - - if lyrics.Instrumental { - return fmt.Errorf("track is instrumental, no lyrics available") - } - - lrcContent := convertToLRCWithMetadata(lyrics, trackName, artistName) - if lrcContent == "" { - return fmt.Errorf("failed to generate LRC content") - } - - if err := os.WriteFile(outputPath, []byte(lrcContent), 0644); err != nil { - return fmt.Errorf("failed to write LRC file: %w", err) - } - - GoLog("[Lyrics] Saved LRC to: %s (%d lines)\n", outputPath, len(lyrics.Lines)) - return nil -} - -func SetLyricsProvidersJSON(providersJSON string) error { - var providers []string - if err := json.Unmarshal([]byte(providersJSON), &providers); err != nil { - return err - } - - SetLyricsProviderOrder(providers) - return nil -} - -func GetLyricsProvidersJSON() (string, error) { - providers := GetLyricsProviderOrder() - return marshalJSONString(providers) -} - -func GetAvailableLyricsProvidersJSON() (string, error) { - providers := GetAvailableLyricsProviders() - return marshalJSONString(providers) -} - -func SetLyricsFetchOptionsJSON(optionsJSON string) error { - opts := GetLyricsFetchOptions() - if strings.TrimSpace(optionsJSON) != "" { - if err := json.Unmarshal([]byte(optionsJSON), &opts); err != nil { - return err - } - } - - SetLyricsFetchOptions(opts) - return nil -} - -func GetLyricsFetchOptionsJSON() (string, error) { - opts := GetLyricsFetchOptions() - return marshalJSONString(opts) -} diff --git a/go_backend/exports_metadata.go b/go_backend/exports_metadata.go deleted file mode 100644 index f8c98e00..00000000 --- a/go_backend/exports_metadata.go +++ /dev/null @@ -1,607 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "strings" - "time" -) - -func applyAudioMetadataToResult(result map[string]any, meta *AudioMetadata) { - result["title"] = meta.Title - result["artist"] = meta.Artist - result["album"] = meta.Album - result["album_artist"] = meta.AlbumArtist - result["date"] = meta.Date - if meta.Date == "" { - result["date"] = meta.Year - } - result["track_number"] = meta.TrackNumber - result["total_tracks"] = meta.TotalTracks - result["disc_number"] = meta.DiscNumber - result["total_discs"] = meta.TotalDiscs - result["isrc"] = meta.ISRC - result["lyrics"] = meta.Lyrics - result["genre"] = meta.Genre - result["label"] = meta.Label - result["copyright"] = meta.Copyright - result["composer"] = meta.Composer - result["comment"] = meta.Comment - result["album_type"] = meta.AlbumType - result["explicit"] = meta.Explicit - result["upc"] = meta.UPC - result["replaygain_track_gain"] = meta.ReplayGainTrackGain - result["replaygain_track_peak"] = meta.ReplayGainTrackPeak - result["replaygain_album_gain"] = meta.ReplayGainAlbumGain - result["replaygain_album_peak"] = meta.ReplayGainAlbumPeak -} - -func successMethodJSON(method string) (string, error) { - return marshalJSONString(map[string]any{"success": true, "method": method}) -} - -func ReadFileMetadata(filePath string) (string, error) { - return ReadFileMetadataWithHint(filePath, "") -} - -// ReadFileMetadataWithHint reads complete tags from extensionless descriptor -// paths without changing their identity or requiring an audio-file copy. -func ReadFileMetadataWithHint(filePath, displayNameHint string) (string, error) { - lower := resolveLibraryAudioExt(filePath, displayNameHint) - isFlac := strings.HasSuffix(lower, ".flac") - isM4A := strings.HasSuffix(lower, ".m4a") || strings.HasSuffix(lower, ".mp4") || strings.HasSuffix(lower, ".aac") - isMp3 := strings.HasSuffix(lower, ".mp3") - isOgg := strings.HasSuffix(lower, ".opus") || strings.HasSuffix(lower, ".ogg") - isApe := strings.HasSuffix(lower, ".ape") - isWv := strings.HasSuffix(lower, ".wv") - isMpc := strings.HasSuffix(lower, ".mpc") - isWav := strings.HasSuffix(lower, ".wav") - isAiff := strings.HasSuffix(lower, ".aiff") || strings.HasSuffix(lower, ".aif") || strings.HasSuffix(lower, ".aifc") - - result := map[string]any{ - "title": "", - "artist": "", - "album": "", - "album_artist": "", - "date": "", - "track_number": 0, - "total_tracks": 0, - "disc_number": 0, - "total_discs": 0, - "isrc": "", - "lyrics": "", - "genre": "", - "label": "", - "copyright": "", - "composer": "", - "comment": "", - "album_type": "", - "explicit": false, - "upc": "", - "duration": 0, - "format": "", - "audio_codec": "", - } - - var metadataErr error - if isFlac { - result["format"] = "flac" - result["audio_codec"] = "flac" - metadata, err := ReadMetadata(filePath) - if err != nil { - // File may have wrong extension (e.g. opus saved as .flac). - // Try Ogg/Opus parser as fallback before giving up. - GoLog("[ReadFileMetadata] FLAC parse failed for %s, trying Ogg fallback: %v\n", filePath, err) - oggMeta, oggErr := ReadOggVorbisComments(filePath) - if oggErr == nil && oggMeta != nil { - applyAudioMetadataToResult(result, oggMeta) - quality, qualityErr := GetOggQuality(filePath) - if qualityErr == nil { - result["sample_rate"] = quality.SampleRate - result["duration"] = quality.Duration - if quality.Bitrate > 0 { - result["bitrate"] = quality.Bitrate / 1000 - } - } - result["format"] = "opus" - result["audio_codec"] = "opus" - } else { - return "", fmt.Errorf("failed to read metadata: %w", err) - } - } else { - result["title"] = metadata.Title - result["artist"] = metadata.Artist - result["album"] = metadata.Album - result["album_artist"] = metadata.AlbumArtist - result["date"] = metadata.Date - result["track_number"] = metadata.TrackNumber - result["total_tracks"] = metadata.TotalTracks - result["disc_number"] = metadata.DiscNumber - result["total_discs"] = metadata.TotalDiscs - result["isrc"] = metadata.ISRC - result["lyrics"] = metadata.Lyrics - result["genre"] = metadata.Genre - result["label"] = metadata.Label - result["copyright"] = metadata.Copyright - result["composer"] = metadata.Composer - result["comment"] = metadata.Comment - result["album_type"] = metadata.AlbumType - result["explicit"] = metadata.Explicit - result["upc"] = metadata.UPC - result["replaygain_track_gain"] = metadata.ReplayGainTrackGain - result["replaygain_track_peak"] = metadata.ReplayGainTrackPeak - result["replaygain_album_gain"] = metadata.ReplayGainAlbumGain - result["replaygain_album_peak"] = metadata.ReplayGainAlbumPeak - - quality, qualityErr := GetAudioQuality(filePath) - if qualityErr == nil { - result["bit_depth"] = quality.BitDepth - result["sample_rate"] = quality.SampleRate - if quality.Codec != "" { - result["audio_codec"] = quality.Codec - } - if quality.SampleRate > 0 && quality.TotalSamples > 0 { - result["duration"] = int(quality.TotalSamples / int64(quality.SampleRate)) - // Average bitrate from file size: helps spot lossy audio - // repackaged as "24-bit" FLAC (real hi-res sits well above - // ~1500 kbps, upconverted files far below). - durationSec := float64(quality.TotalSamples) / float64(quality.SampleRate) - if info, statErr := os.Stat(filePath); statErr == nil && info.Size() > 0 && durationSec > 0 { - result["bitrate"] = int(float64(info.Size()) * 8 / durationSec / 1000) - } - } - } - } - } else if isM4A { - result["format"] = "m4a" - meta, err := ReadM4ATags(filePath) - metadataErr = err - if err == nil && meta != nil { - applyAudioMetadataToResult(result, meta) - } - quality, qualityErr := GetM4AQuality(filePath) - if qualityErr == nil { - result["bit_depth"] = quality.BitDepth - result["sample_rate"] = quality.SampleRate - result["duration"] = quality.Duration - result["audio_codec"] = quality.Codec - if format := libraryFormatForM4ACodec(quality.Codec); format != "" { - result["format"] = format - } - if quality.Bitrate > 0 && !isLosslessLibraryFormat(fmt.Sprint(result["format"])) { - result["bitrate"] = quality.Bitrate - } else if quality.Duration > 0 { - if info, statErr := os.Stat(filePath); statErr == nil && info.Size() > 0 { - result["bitrate"] = int(float64(info.Size()) * 8 / float64(quality.Duration) / 1000) - } - } - } - } else if isMp3 { - result["format"] = "mp3" - result["audio_codec"] = "mp3" - meta, err := ReadID3Tags(filePath) - metadataErr = err - if err == nil && meta != nil { - applyAudioMetadataToResult(result, meta) - } - quality, qualityErr := GetMP3Quality(filePath) - if qualityErr == nil { - result["bit_depth"] = quality.BitDepth - result["sample_rate"] = quality.SampleRate - result["duration"] = quality.Duration - if quality.Bitrate > 0 { - result["bitrate"] = quality.Bitrate / 1000 - } - } - } else if isOgg { - result["format"] = "opus" - result["audio_codec"] = "opus" - meta, err := ReadOggVorbisComments(filePath) - metadataErr = err - if err == nil && meta != nil { - applyAudioMetadataToResult(result, meta) - } - quality, qualityErr := GetOggQuality(filePath) - if qualityErr == nil { - result["sample_rate"] = quality.SampleRate - result["duration"] = quality.Duration - if quality.Bitrate > 0 { - result["bitrate"] = quality.Bitrate / 1000 - } - } - } else if isApe || isWv || isMpc { - result["format"] = strings.TrimPrefix(lower, ".") - result["audio_codec"] = result["format"] - apeTag, apeErr := ReadAPETags(filePath) - metadataErr = apeErr - if apeErr == nil && apeTag != nil { - meta := APETagToAudioMetadata(apeTag) - if meta != nil { - applyAudioMetadataToResult(result, meta) - } - } - } else if isWav || isAiff { - var meta *AudioMetadata - var quality *WAVQuality - var qualityErr error - if isAiff { - result["format"] = "aiff" - result["audio_codec"] = "pcm" - meta, metadataErr = ReadAIFFTags(filePath) - quality, qualityErr = GetAIFFQuality(filePath) - } else { - result["format"] = "wav" - result["audio_codec"] = "pcm" - meta, metadataErr = ReadWAVTags(filePath) - quality, qualityErr = GetWAVQuality(filePath) - } - if meta != nil { - applyAudioMetadataToResult(result, meta) - } - if qualityErr == nil && quality != nil { - result["bit_depth"] = quality.BitDepth - result["sample_rate"] = quality.SampleRate - result["duration"] = quality.Duration - } - } else { - return "", fmt.Errorf("unsupported file format: %s", filePath) - } - - // A readable audio file can legitimately have no tags. Filesystem errors, - // however, must reach the native bridge so an unreadable SAF descriptor - // triggers its temporary-file fallback instead of returning empty tags. - var pathErr *os.PathError - if errors.As(metadataErr, &pathErr) { - return "", fmt.Errorf("failed to read metadata: %w", metadataErr) - } - return marshalJSONString(result) -} - -// ParseCueSheet is called from Dart to get track listing and timing data for CUE splitting. -// audioDir, if non-empty, overrides the directory used for resolving the -// referenced audio file (useful for SAF temp file scenarios). -func ParseCueSheet(cuePath string, audioDir string) (string, error) { - return ParseCueFileJSON(cuePath, audioDir) -} - -// ScanCueSheetForLibrary parses a .cue file and returns a JSON array of -// LibraryScanResult entries (one per track). This is the SAF-friendly variant: -// - audioDir overrides where the referenced audio file is resolved -// - virtualPathPrefix replaces cuePath in filePath / id fields (e.g. a content:// URI) -// - fileModTime is stamped on every result (pass 0 to stat cuePath instead) -func ScanCueSheetForLibrary(cuePath, audioDir, virtualPathPrefix string, fileModTime int64) (string, error) { - scanTime := time.Now().UTC().Format(time.RFC3339) - results, err := ScanCueFileForLibraryExt(cuePath, audioDir, virtualPathPrefix, fileModTime, scanTime) - if err != nil { - return "[]", err - } - jsonBytes, err := json.Marshal(results) - if err != nil { - return "[]", fmt.Errorf("failed to marshal cue scan results: %w", err) - } - return string(jsonBytes), nil -} - -func ScanCueSheetForLibraryWithCoverCacheKey(cuePath, audioDir, virtualPathPrefix string, fileModTime int64, coverCacheKey string) (string, error) { - scanTime := time.Now().UTC().Format(time.RFC3339) - results, err := ScanCueFileForLibraryExtWithCoverCacheKey( - cuePath, - audioDir, - virtualPathPrefix, - fileModTime, - coverCacheKey, - scanTime, - ) - if err != nil { - return "[]", err - } - jsonBytes, err := json.Marshal(results) - if err != nil { - return "[]", fmt.Errorf("failed to marshal cue scan results: %w", err) - } - return string(jsonBytes), nil -} - -// WriteM4AFreeformTags writes ISRC and label into an M4A/MP4 file as iTunes -// freeform atoms. FFmpeg's MP4 muxer ignores these keys, so they must be -// written natively after the FFmpeg metadata pass for the values to persist. -// Only keys present in the JSON are touched; an empty value clears the tag. -func WriteM4AFreeformTags(filePath, metadataJSON string) (string, error) { - var fields map[string]string - if err := json.Unmarshal([]byte(metadataJSON), &fields); err != nil { - return "", fmt.Errorf("invalid metadata JSON: %w", err) - } - - if err := EditM4AFreeformText(filePath, fields); err != nil { - return "", fmt.Errorf("failed to write M4A freeform tags: %w", err) - } - - return successMethodJSON("native_m4a_freeform") -} - -// EnsureAC4Config normalizes a decrypted AC-4 file to a standards-compliant ISO -// MP4 and injects the dac4 configuration box copied from sourcePath. No-op when -// the file is not AC-4. -func EnsureAC4Config(filePath, sourcePath string) (string, error) { - if err := EnsureAC4ConfigBox(filePath, sourcePath); err != nil { - return "", fmt.Errorf("failed to finalize AC-4 container: %w", err) - } - return `{"success":true}`, nil -} - -// WriteAC4Metadata writes iTunes-style metadata into an AC-4 MP4. The JSON -// "handled" field reports whether the file was AC-4 (true) so the caller can -// skip the FFmpeg metadata pass that would re-wrap it as QuickTime. -func WriteAC4Metadata(filePath, metadataJSON, coverPath string) (string, error) { - handled, err := WriteAC4MetadataIfApplicable(filePath, metadataJSON, coverPath) - if err != nil { - return "", fmt.Errorf("failed to write AC-4 metadata: %w", err) - } - resp := map[string]any{"success": true, "handled": handled} - s, _ := marshalJSONString(resp) - return s, nil -} - -// EditFileMetadata writes audio file tags: FLAC via native Go library, MP3/Opus returns map for Dart/FFmpeg. -func EditFileMetadata(filePath, metadataJSON string) (string, error) { - var fields map[string]string - if err := json.Unmarshal([]byte(metadataJSON), &fields); err != nil { - return "", fmt.Errorf("invalid metadata JSON: %w", err) - } - - lower := strings.ToLower(filePath) - isFlac := strings.HasSuffix(lower, ".flac") - isApeFile := strings.HasSuffix(lower, ".ape") || strings.HasSuffix(lower, ".wv") || strings.HasSuffix(lower, ".mpc") - isM4AFile := strings.HasSuffix(lower, ".m4a") || strings.HasSuffix(lower, ".mp4") || strings.HasSuffix(lower, ".m4b") - isWavFile := strings.HasSuffix(lower, ".wav") - isAiffFile := strings.HasSuffix(lower, ".aiff") || strings.HasSuffix(lower, ".aif") || strings.HasSuffix(lower, ".aifc") - coverPath := strings.TrimSpace(fields["cover_path"]) - - if hasOnlyM4AReplayGainFields(fields) && (isM4AFile || isMP4ContainerFile(filePath)) { - if err := EditM4AReplayGain(filePath, fields); err != nil { - return "", fmt.Errorf("failed to write M4A metadata: %w", err) - } - - return successMethodJSON("native_m4a_replaygain") - } - - if isFlac { - // A .flac name does not guarantee FLAC content: providers sometimes - // deliver an MP4/M4A stream that ends up under the requested name. - // The FLAC writer would fail "fLaC head incorrect" on every attempt, - // so detect the mismatch up front and say what is actually wrong. - if isMP4ContainerFile(filePath) { - return "", fmt.Errorf( - "failed to write FLAC metadata: file is an MP4/M4A stream under a .flac name; rename it to .m4a", - ) - } - if err := EditFlacFields(filePath, fields); err != nil { - return "", fmt.Errorf("failed to write FLAC metadata: %w", err) - } - - return successMethodJSON("native") - } - - // WAV / AIFF: write tags into an embedded ID3v2.4 chunk natively. - if isWavFile { - if err := WriteWAVTags(filePath, fields); err != nil { - return "", fmt.Errorf("failed to write WAV metadata: %w", err) - } - return successMethodJSON("native_wav") - } - if isAiffFile { - if err := WriteAIFFTags(filePath, fields); err != nil { - return "", fmt.Errorf("failed to write AIFF metadata: %w", err) - } - return successMethodJSON("native_aiff") - } - - if isApeFile { - meta := audioMetadataFromEditFields(fields) - - newItems := AudioMetadataToAPEItems(meta) - - // If a cover image was provided, embed it as a binary APE item. - // APEv2 cover format: "cover.jpg\0", flagged binary. - if coverPath != "" { - coverData, coverErr := os.ReadFile(coverPath) - if coverErr == nil && len(coverData) > 0 { - // The value is "filename\0" + raw bytes. We store the - // description as the Value field, but since the item is - // flagged binary, the writer serializes it verbatim. - desc := "cover.jpg\x00" - binaryValue := desc + string(coverData) - newItems = append(newItems, APETagItem{ - Key: "Cover Art (Front)", - Value: binaryValue, - Flags: apeItemFlagBinary, - }) - } - } - - // Build the set of APE keys that the edit explicitly controls. - // Even if the value is empty (user cleared the field), the old - // value must be removed during merge. - overrideKeys := apeKeysFromFields(fields) - if coverPath != "" { - overrideKeys["COVER ART (FRONT)"] = struct{}{} - } - - // Read existing tags so we can merge rather than replace. - // This preserves cover art and custom items not in the edit set. - existingTag, _ := ReadAPETags(filePath) - var finalItems []APETagItem - if existingTag != nil && len(existingTag.Items) > 0 { - finalItems = MergeAPEItems(existingTag.Items, newItems, overrideKeys) - } else { - finalItems = newItems - } - - tag := &APETag{ - Version: apeTagVersion2, - Items: finalItems, - } - - if err := WriteAPETags(filePath, tag); err != nil { - return "", fmt.Errorf("failed to write APE tags: %w", err) - } - - return successMethodJSON("native_ape") - } - - // MP3, Ogg/Opus, and M4A have native editors that preserve foreign - // tags and skip the ffmpeg remux. Any failure falls back to the ffmpeg - // response so callers keep the old behavior for exotic files. - isMp3 := strings.HasSuffix(lower, ".mp3") - isOggFile := strings.HasSuffix(lower, ".opus") || strings.HasSuffix(lower, ".ogg") - - if isMp3 { - if err := EditMP3Fields(filePath, fields); err != nil { - GoLog("[Metadata] Native MP3 edit failed, falling back to ffmpeg: %v\n", err) - } else { - return successMethodJSON("native_mp3") - } - } - if isOggFile { - if err := EditOggFields(filePath, fields); err != nil { - GoLog("[Metadata] Native Ogg edit failed, falling back to ffmpeg: %v\n", err) - } else { - return successMethodJSON("native_ogg") - } - } - if isM4AFile || isMP4ContainerFile(filePath) { - if err := EditM4AFields(filePath, fields); err != nil { - GoLog("[Metadata] Native M4A edit failed, falling back to ffmpeg: %v\n", err) - } else { - return successMethodJSON("native_m4a") - } - } - - resp := map[string]any{ - "success": true, - "method": "ffmpeg", - "fields": fields, - } - s, _ := marshalJSONString(resp) - return s, nil -} - -func isMP4ContainerFile(filePath string) bool { - f, err := os.Open(filePath) - if err != nil { - return false - } - defer f.Close() - - header := make([]byte, 12) - n, err := f.Read(header) - if err != nil || n < 8 { - return false - } - return string(header[4:8]) == "ftyp" -} - -func hasOnlyM4AReplayGainFields(fields map[string]string) bool { - allowed := map[string]struct{}{ - "replaygain_track_gain": {}, - "replaygain_track_peak": {}, - "replaygain_album_gain": {}, - "replaygain_album_peak": {}, - } - - hasReplayGain := false - for key, value := range fields { - if strings.TrimSpace(value) == "" { - continue - } - if _, ok := allowed[strings.ToLower(strings.TrimSpace(key))]; ok { - hasReplayGain = true - continue - } - return false - } - - return hasReplayGain -} - -// RewriteSplitArtistTagsExport rewrites ARTIST and ALBUMARTIST Vorbis -// comments in a FLAC file as multiple separate entries (one per artist). -// Call this after FFmpeg metadata embedding to fix split artist tags, -// since FFmpeg deduplicates -metadata keys and only keeps the last value. -func RewriteSplitArtistTagsExport(filePath, artist, albumArtist string) (string, error) { - err := RewriteSplitArtistTags(filePath, artist, albumArtist) - if err != nil { - return errorResponse("Failed to rewrite artist tags: " + err.Error()) - } - - resp := map[string]any{ - "success": true, - "message": "Split artist tags written successfully", - } - - s, _ := marshalJSONString(resp) - return s, nil -} - -// The final bool is retained for gomobile ABI compatibility with existing -// native shells. Resolution selection is extension-owned and the value is -// intentionally ignored. -func DownloadCoverToFile(coverURL string, outputPath string, _ bool) error { - return DownloadCoverToFileSized(coverURL, outputPath, 0) -} - -// DownloadCoverToFileSized downloads provider artwork and optionally caps its -// longest side before writing it. It is a separate export so the legacy -// gomobile ABI remains available to older native shells. -func DownloadCoverToFileSized(coverURL string, outputPath string, maxDimension int) error { - if coverURL == "" { - return fmt.Errorf("no cover URL provided") - } - - data, err := downloadCoverToMemorySized(coverURL, maxDimension) - if err != nil { - return fmt.Errorf("failed to download cover: %w", err) - } - - if err := os.WriteFile(outputPath, data, 0644); err != nil { - return fmt.Errorf("failed to write cover file: %w", err) - } - - return nil -} - -func ExtractCoverToFile(audioPath string, outputPath string) error { - lower := strings.ToLower(audioPath) - - var coverData []byte - var err error - - if strings.HasSuffix(lower, ".flac") { - coverData, err = ExtractCoverArt(audioPath) - } else if strings.HasSuffix(lower, ".m4a") || strings.HasSuffix(lower, ".aac") { - coverData, err = extractCoverFromM4A(audioPath) - } else if strings.HasSuffix(lower, ".mp3") { - coverData, _, err = extractMP3CoverArt(audioPath) - } else if strings.HasSuffix(lower, ".opus") || strings.HasSuffix(lower, ".ogg") { - coverData, _, err = extractOggCoverArt(audioPath) - } else if strings.HasSuffix(lower, ".wav") || - strings.HasSuffix(lower, ".aiff") || - strings.HasSuffix(lower, ".aif") || - strings.HasSuffix(lower, ".aifc") { - coverData, _, err = extractWAVAIFFCover(audioPath) - } else { - return fmt.Errorf("unsupported audio format for cover extraction") - } - - if err != nil { - return fmt.Errorf("failed to extract cover: %w", err) - } - - if err := os.WriteFile(outputPath, coverData, 0644); err != nil { - return fmt.Errorf("failed to write cover file: %w", err) - } - - return nil -} diff --git a/go_backend/exports_musicbrainz.go b/go_backend/exports_musicbrainz.go deleted file mode 100644 index 5a616152..00000000 --- a/go_backend/exports_musicbrainz.go +++ /dev/null @@ -1,304 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "net/http" - "net/url" - "strings" - "sync" - "time" - - "golang.org/x/sync/singleflight" - "golang.org/x/text/cases" - "golang.org/x/text/language" -) - -const musicBrainzAPIBase = "https://musicbrainz.org/ws/2" - -// MusicBrainz lookups run in the per-track finalize stage and album batches -// repeat the same ISRC-adjacent questions; each miss costs up to 3 attempts -// with 2s sleeps. Positive results are stable (6h TTL); misses/errors expire -// quickly (10min) so a manual re-enrich can retry soon. -const ( - musicBrainzCachePositiveTTL = 6 * time.Hour - musicBrainzCacheNegativeTTL = 10 * time.Minute - musicBrainzCacheMaxEntries = 256 -) - -type musicBrainzCacheEntry struct { - value string - err error - expiresAt time.Time -} - -var ( - musicBrainzCacheMu sync.Mutex - musicBrainzCache = make(map[string]musicBrainzCacheEntry) - musicBrainzFlight singleflight.Group -) - -func musicBrainzCached(key string, fetch func() (string, error)) (string, error) { - musicBrainzCacheMu.Lock() - if entry, ok := musicBrainzCache[key]; ok && time.Now().Before(entry.expiresAt) { - musicBrainzCacheMu.Unlock() - return entry.value, entry.err - } - musicBrainzCacheMu.Unlock() - - result, err, _ := musicBrainzFlight.Do(key, func() (any, error) { - // A waiter can reach the flight after the leader populated the cache. - musicBrainzCacheMu.Lock() - if entry, ok := musicBrainzCache[key]; ok && time.Now().Before(entry.expiresAt) { - musicBrainzCacheMu.Unlock() - return entry.value, entry.err - } - musicBrainzCacheMu.Unlock() - - value, fetchErr := fetch() - ttl := musicBrainzCachePositiveTTL - if fetchErr != nil || value == "" { - ttl = musicBrainzCacheNegativeTTL - } - musicBrainzCacheMu.Lock() - if len(musicBrainzCache) >= musicBrainzCacheMaxEntries { - now := time.Now() - for k, e := range musicBrainzCache { - if now.After(e.expiresAt) { - delete(musicBrainzCache, k) - } - } - if len(musicBrainzCache) >= musicBrainzCacheMaxEntries { - musicBrainzCache = make(map[string]musicBrainzCacheEntry) - } - } - musicBrainzCache[key] = musicBrainzCacheEntry{ - value: value, - err: fetchErr, - expiresAt: time.Now().Add(ttl), - } - musicBrainzCacheMu.Unlock() - return value, fetchErr - }) - if err != nil { - return "", err - } - value, _ := result.(string) - return value, nil -} - -type musicBrainzTag struct { - Count int `json:"count"` - Name string `json:"name"` -} - -type musicBrainzArtistCredit struct { - Name string `json:"name"` - JoinPhrase string `json:"joinphrase"` -} - -type musicBrainzRelease struct { - Title string `json:"title"` - ArtistCredit []musicBrainzArtistCredit `json:"artist-credit"` -} - -type musicBrainzCombinedResponse struct { - Recordings []struct { - Tags []musicBrainzTag `json:"tags"` - Releases []musicBrainzRelease `json:"releases"` - } `json:"recordings"` -} - -// fetchMusicBrainzCombinedByISRC is the shared recording snapshot used by -// both genre and album-artist lookups. MusicBrainz accepts all includes in a -// single request, so two native calls for the same ISRC still cost one HTTP -// request and concurrent callers share that request through singleflight. -func fetchMusicBrainzCombinedByISRC(isrc string) (*musicBrainzCombinedResponse, string, error) { - normalizedISRC := strings.ToUpper(strings.TrimSpace(isrc)) - key := "recording\x00" + normalizedISRC - encoded, err := musicBrainzCached(key, func() (string, error) { - var payload musicBrainzCombinedResponse - normalized, fetchErr := fetchMusicBrainzRecordingByISRC( - isrc, - "tags+releases+artist-credits", - &payload, - ) - if fetchErr != nil { - return "", fetchErr - } - data, marshalErr := json.Marshal(payload) - if marshalErr != nil { - return "", marshalErr - } - if normalizedISRC == "" { - normalizedISRC = normalized - } - return string(data), nil - }) - if err != nil { - return nil, normalizedISRC, err - } - var payload musicBrainzCombinedResponse - if err := json.Unmarshal([]byte(encoded), &payload); err != nil { - return nil, normalizedISRC, err - } - return &payload, normalizedISRC, nil -} - -func formatMusicBrainzGenre(tags []musicBrainzTag) string { - if len(tags) == 0 { - return "" - } - - caser := cases.Title(language.English) - seen := make(map[string]struct{}, len(tags)) - maxCount := -1 - bestTag := "" - - for _, tag := range tags { - name := strings.TrimSpace(tag.Name) - if name == "" { - continue - } - - key := strings.ToLower(name) - if _, exists := seen[key]; exists { - continue - } - seen[key] = struct{}{} - - formatted := caser.String(name) - if tag.Count > maxCount { - maxCount = tag.Count - bestTag = formatted - } - } - - return bestTag -} - -func formatMusicBrainzArtistCredit(credits []musicBrainzArtistCredit) string { - var builder strings.Builder - for _, credit := range credits { - name := strings.TrimSpace(credit.Name) - if name == "" { - continue - } - builder.WriteString(name) - builder.WriteString(credit.JoinPhrase) - } - return strings.TrimSpace(builder.String()) -} - -func selectMusicBrainzAlbumArtist(releases []musicBrainzRelease, albumName string) string { - if len(releases) == 0 { - return "" - } - - normalizedAlbum := strings.ToLower(strings.TrimSpace(albumName)) - if normalizedAlbum != "" { - for _, release := range releases { - if strings.ToLower(strings.TrimSpace(release.Title)) != normalizedAlbum { - continue - } - if albumArtist := formatMusicBrainzArtistCredit(release.ArtistCredit); albumArtist != "" { - return albumArtist - } - } - } - - for _, release := range releases { - if albumArtist := formatMusicBrainzArtistCredit(release.ArtistCredit); albumArtist != "" { - return albumArtist - } - } - - return "" -} - -// fetchMusicBrainzRecordingByISRC queries the MusicBrainz recording endpoint -// for the given ISRC with the supplied inc= parameter, retrying up to 3 times, -// and decodes the JSON response into payload. It returns the normalized ISRC. -func fetchMusicBrainzRecordingByISRC(isrc string, inc string, payload any) (string, error) { - normalizedISRC := strings.ToUpper(strings.TrimSpace(isrc)) - if normalizedISRC == "" { - return "", fmt.Errorf("no ISRC provided") - } - - client := NewMetadataHTTPClient(10 * time.Second) - query := fmt.Sprintf("isrc:%s", normalizedISRC) - reqURL := fmt.Sprintf( - "%s/recording?query=%s&fmt=json&inc=%s", - musicBrainzAPIBase, - url.QueryEscape(query), - inc, - ) - - req, err := http.NewRequest(http.MethodGet, reqURL, nil) - if err != nil { - return "", err - } - req.Header.Set("User-Agent", getRandomUserAgent()) - - var resp *http.Response - var lastErr error - for attempt := 0; attempt < 3; attempt++ { - resp, lastErr = client.Do(req) - if lastErr == nil && resp.StatusCode == http.StatusOK { - break - } - if resp != nil { - resp.Body.Close() - } - if attempt < 2 { - time.Sleep(2 * time.Second) - } - } - - if lastErr != nil { - return "", lastErr - } - if resp == nil { - return "", fmt.Errorf("MusicBrainz request failed without response") - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return "", fmt.Errorf("MusicBrainz API returned status: %d", resp.StatusCode) - } - defer resp.Body.Close() - - if err := json.NewDecoder(resp.Body).Decode(payload); err != nil { - return "", err - } - return normalizedISRC, nil -} - -func FetchMusicBrainzAlbumArtistByISRC(isrc string, albumName string) (string, error) { - payload, normalizedISRC, err := fetchMusicBrainzCombinedByISRC(isrc) - if err != nil { - return "", err - } - for _, recording := range payload.Recordings { - if albumArtist := selectMusicBrainzAlbumArtist(recording.Releases, albumName); albumArtist != "" { - return albumArtist, nil - } - } - - return "", fmt.Errorf("no MusicBrainz album artist found for ISRC: %s", normalizedISRC) -} - -func FetchMusicBrainzGenreByISRC(isrc string) (string, error) { - payload, normalizedISRC, err := fetchMusicBrainzCombinedByISRC(isrc) - if err != nil { - return "", err - } - if len(payload.Recordings) == 0 { - return "", fmt.Errorf("no recordings found for ISRC: %s", normalizedISRC) - } - - genre := formatMusicBrainzGenre(payload.Recordings[0].Tags) - if genre == "" { - return "", fmt.Errorf("no MusicBrainz genre tags found for ISRC: %s", normalizedISRC) - } - return genre, nil -} diff --git a/go_backend/exports_reenrich.go b/go_backend/exports_reenrich.go deleted file mode 100644 index 2e622f7b..00000000 --- a/go_backend/exports_reenrich.go +++ /dev/null @@ -1,902 +0,0 @@ -package gobackend - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - "time" -) - -var fetchDeezerExtendedMetadataByISRC = func(ctx context.Context, isrc string) (*AlbumExtendedMetadata, error) { - return GetDeezerClient().GetExtendedMetadataByISRC(ctx, isrc) -} - -var fetchMusicBrainzGenreByISRC = FetchMusicBrainzGenreByISRC - -var fetchMusicBrainzAlbumArtistByISRC = FetchMusicBrainzAlbumArtistByISRC - -type reEnrichRequest struct { - FilePath string `json:"file_path"` - CoverURL string `json:"cover_url"` - CoverMaxDimension int `json:"cover_max_dimension,omitempty"` - EmbedLyrics bool `json:"embed_lyrics"` - LyricsMode string `json:"lyrics_mode,omitempty"` - ArtistTagMode string `json:"artist_tag_mode,omitempty"` - SpotifyID string `json:"spotify_id"` - TrackName string `json:"track_name"` - ArtistName string `json:"artist_name"` - AlbumName string `json:"album_name"` - AlbumArtist string `json:"album_artist"` - TrackNumber int `json:"track_number"` - DiscNumber int `json:"disc_number"` - TotalTracks int `json:"total_tracks,omitempty"` - TotalDiscs int `json:"total_discs,omitempty"` - ReleaseDate string `json:"release_date"` - ISRC string `json:"isrc"` - Genre string `json:"genre"` - Label string `json:"label"` - Copyright string `json:"copyright"` - Composer string `json:"composer"` - DurationMs int64 `json:"duration_ms"` - SearchOnline bool `json:"search_online"` - UpdateFields []string `json:"update_fields,omitempty"` - // PreviewOnly resolves the metadata candidate and returns the proposed - // values without downloading artwork, fetching lyrics, or touching the - // audio file. Batch callers use this to review changes before embedding. - PreviewOnly bool `json:"preview_only,omitempty"` - // ReplaceReleaseMetadata lets a deliberate single-file re-enrich action - // repair a stale album identity (for example, a playlist name stored as - // ALBUM). Batch and older callers keep the conservative mismatch guard. - ReplaceReleaseMetadata bool `json:"replace_release_metadata,omitempty"` -} - -// shouldUpdateField returns true if the given field group should be updated. -// When UpdateFields is empty/nil, all fields are updated (backward compatible). -func (r *reEnrichRequest) shouldUpdateField(field string) bool { - if len(r.UpdateFields) == 0 { - return true - } - for _, f := range r.UpdateFields { - if f == field { - return true - } - } - return false -} - -// shouldUpdateTag accepts both the original field-group keys and granular tag -// keys. This keeps existing callers compatible while allowing batch actions -// such as "ISRC only" and "fill missing tags" to avoid changing neighboring -// values from the same group. -func (r *reEnrichRequest) shouldUpdateTag(group, tag string) bool { - if len(r.UpdateFields) == 0 { - return true - } - for _, field := range r.UpdateFields { - if field == group || field == tag { - return true - } - } - return false -} - -func (r *reEnrichRequest) shouldUpdateAnyTag(group string, tags ...string) bool { - if r.shouldUpdateField(group) { - return true - } - for _, tag := range tags { - if r.shouldUpdateTag(group, tag) { - return true - } - } - return false -} - -// lyricsEmbedEnabled reports whether lyrics should be written into the audio -// file's tags. It mirrors the download path semantics: 'embed' and 'both' embed, -// 'external' does not. An empty mode keeps the legacy behavior (embed) so older -// callers that do not send lyrics_mode are unaffected. -func (r *reEnrichRequest) lyricsEmbedEnabled() bool { - return strings.ToLower(strings.TrimSpace(r.LyricsMode)) != "external" -} - -// lyricsSidecarEnabled reports whether a .lrc sidecar file should be written -// next to the audio file. Only 'external' and 'both' request a sidecar. -func (r *reEnrichRequest) lyricsSidecarEnabled() bool { - mode := strings.ToLower(strings.TrimSpace(r.LyricsMode)) - return mode == "external" || mode == "both" -} - -// reEnrichSameRelease reports whether the candidate track appears to come -// from the same release as the file's existing album. An ISRC identifies a -// recording, not a release: the same song often also resolves to a -// compilation, whose album name, cover, and track positions must not -// replace the original release's. -func reEnrichSameRelease(currentAlbum, candidateAlbum string) bool { - if isPlaceholderReEnrichValue(currentAlbum) || - strings.TrimSpace(candidateAlbum) == "" { - return true - } - return titlesMatch(currentAlbum, candidateAlbum) -} - -func applyReEnrichTrackMetadata(req *reEnrichRequest, track ExtTrackMetadata) { - if req == nil { - return - } - - albumMatches := reEnrichSameRelease(req.AlbumName, track.AlbumName) - sameRelease := req.ReplaceReleaseMetadata || albumMatches - if !sameRelease { - GoLog("[ReEnrich] Candidate album %q differs from file album %q; keeping release identity (album, cover, positions, date)\n", - track.AlbumName, req.AlbumName) - } else if req.ReplaceReleaseMetadata && !albumMatches { - GoLog("[ReEnrich] Candidate album %q differs from file album %q; replacing release identity as requested\n", - track.AlbumName, req.AlbumName) - } - - if track.SpotifyID != "" { - req.SpotifyID = track.SpotifyID - } else if track.DeezerID != "" { - req.SpotifyID = "deezer:" + track.DeezerID - } else if track.QobuzID != "" { - req.SpotifyID = "qobuz:" + track.QobuzID - } else if track.TidalID != "" { - req.SpotifyID = "tidal:" + track.TidalID - } else if track.ID != "" { - req.SpotifyID = track.ID - } - - if req.shouldUpdateTag("basic_tags", "track_name") { - if track.Name != "" { - req.TrackName = track.Name - } - } - if req.shouldUpdateTag("basic_tags", "artist_name") { - if track.Artists != "" { - req.ArtistName = track.Artists - } - } - if sameRelease && req.shouldUpdateTag("basic_tags", "album_name") { - if track.AlbumName != "" { - req.AlbumName = track.AlbumName - } - } - if sameRelease && req.shouldUpdateTag("basic_tags", "album_artist") { - if track.AlbumArtist != "" { - req.AlbumArtist = track.AlbumArtist - } - } - if sameRelease && req.shouldUpdateTag("track_info", "track_number") { - if track.TrackNumber > 0 { - req.TrackNumber = track.TrackNumber - } - } - if sameRelease && req.shouldUpdateTag("track_info", "total_tracks") { - if track.TotalTracks > 0 { - req.TotalTracks = track.TotalTracks - } - } - if sameRelease && req.shouldUpdateTag("track_info", "disc_number") { - if track.DiscNumber > 0 { - req.DiscNumber = track.DiscNumber - } - } - if sameRelease && req.shouldUpdateTag("track_info", "total_discs") { - if track.TotalDiscs > 0 { - req.TotalDiscs = track.TotalDiscs - } - } - if sameRelease && req.shouldUpdateTag("release_info", "release_date") { - if track.ReleaseDate != "" { - req.ReleaseDate = track.ReleaseDate - } - } - if req.shouldUpdateTag("release_info", "isrc") { - if track.ISRC != "" { - req.ISRC = track.ISRC - } - } - if sameRelease && req.shouldUpdateTag("cover", "cover") { - if coverURL := track.ResolvedCoverURL(); coverURL != "" { - req.CoverURL = coverURL - } - } - if track.DurationMS > 0 { - req.DurationMs = int64(track.DurationMS) - } - if req.shouldUpdateTag("extra", "genre") { - if track.Genre != "" { - req.Genre = track.Genre - } - } - if req.shouldUpdateTag("extra", "label") { - if track.Label != "" { - req.Label = track.Label - } - } - if req.shouldUpdateTag("extra", "copyright") { - if track.Copyright != "" { - req.Copyright = track.Copyright - } - } - if req.shouldUpdateTag("extra", "composer") { - if track.Composer != "" { - req.Composer = track.Composer - } - } -} - -func isPlaceholderReEnrichValue(value string) bool { - switch strings.ToLower(strings.TrimSpace(value)) { - case "", "unknown", "unknown artist", "unknown title", "unknown album": - return true - default: - return false - } -} - -func buildReEnrichSearchQuery(req reEnrichRequest) string { - parts := make([]string, 0, 2) - if !isPlaceholderReEnrichValue(req.TrackName) { - parts = append(parts, strings.TrimSpace(req.TrackName)) - } - if !isPlaceholderReEnrichValue(req.ArtistName) { - parts = append(parts, strings.TrimSpace(req.ArtistName)) - } - if len(parts) == 0 && !isPlaceholderReEnrichValue(req.AlbumName) { - parts = append(parts, strings.TrimSpace(req.AlbumName)) - } - return strings.TrimSpace(strings.Join(parts, " ")) -} - -func reEnrichDownloadRequest(req reEnrichRequest) DownloadRequest { - return DownloadRequest{ - TrackName: req.TrackName, - ArtistName: req.ArtistName, - AlbumName: req.AlbumName, - ReleaseDate: req.ReleaseDate, - ISRC: req.ISRC, - DurationMS: int(req.DurationMs), - ArtistTagMode: req.ArtistTagMode, - TrackNumber: req.TrackNumber, - TotalTracks: req.TotalTracks, - DiscNumber: req.DiscNumber, - TotalDiscs: req.TotalDiscs, - Composer: req.Composer, - } -} - -func buildReEnrichFFmpegMetadata(req *reEnrichRequest, lyricsLRC string) map[string]string { - metadata := map[string]string{} - if req.shouldUpdateTag("basic_tags", "track_name") { - if req.TrackName != "" { - metadata["TITLE"] = req.TrackName - } - } - if req.shouldUpdateTag("basic_tags", "artist_name") { - if req.ArtistName != "" { - metadata["ARTIST"] = req.ArtistName - } - } - if req.shouldUpdateTag("basic_tags", "album_name") { - if req.AlbumName != "" { - metadata["ALBUM"] = req.AlbumName - } - } - if req.shouldUpdateTag("basic_tags", "album_artist") { - if req.AlbumArtist != "" { - metadata["ALBUMARTIST"] = req.AlbumArtist - } - } - if req.shouldUpdateTag("release_info", "release_date") { - if req.ReleaseDate != "" { - metadata["DATE"] = req.ReleaseDate - } - } - if req.shouldUpdateTag("release_info", "isrc") { - if req.ISRC != "" { - metadata["ISRC"] = req.ISRC - } - } - if req.shouldUpdateTag("extra", "genre") { - if req.Genre != "" { - metadata["GENRE"] = req.Genre - } - } - if req.shouldUpdateTag("extra", "label") { - if req.Label != "" { - metadata["ORGANIZATION"] = req.Label - } - } - if req.shouldUpdateTag("extra", "copyright") { - if req.Copyright != "" { - metadata["COPYRIGHT"] = req.Copyright - } - } - if req.shouldUpdateTag("extra", "composer") { - if req.Composer != "" { - metadata["COMPOSER"] = req.Composer - } - } - if req.shouldUpdateTag("track_info", "track_number") || req.shouldUpdateTag("track_info", "total_tracks") { - if req.TrackNumber > 0 { - metadata["TRACKNUMBER"] = formatIndexValue(req.TrackNumber, req.TotalTracks) - } - } - if req.shouldUpdateTag("track_info", "disc_number") || req.shouldUpdateTag("track_info", "total_discs") { - if req.DiscNumber > 0 { - metadata["DISCNUMBER"] = formatIndexValue(req.DiscNumber, req.TotalDiscs) - } - } - if req.shouldUpdateTag("lyrics", "lyrics") { - if lyricsLRC != "" && req.lyricsEmbedEnabled() { - metadata["LYRICS"] = lyricsLRC - metadata["UNSYNCEDLYRICS"] = lyricsLRC - } - } - return metadata -} - -func buildReEnrichResultMetadata(req *reEnrichRequest) map[string]any { - enrichedMeta := map[string]any{ - "spotify_id": req.SpotifyID, - "duration_ms": req.DurationMs, - } - if req.shouldUpdateTag("basic_tags", "track_name") { - enrichedMeta["track_name"] = req.TrackName - } - if req.shouldUpdateTag("basic_tags", "artist_name") { - enrichedMeta["artist_name"] = req.ArtistName - } - if req.shouldUpdateTag("basic_tags", "album_name") { - enrichedMeta["album_name"] = req.AlbumName - } - if req.shouldUpdateTag("basic_tags", "album_artist") { - enrichedMeta["album_artist"] = req.AlbumArtist - } - if req.shouldUpdateTag("track_info", "track_number") { - enrichedMeta["track_number"] = req.TrackNumber - } - if req.shouldUpdateTag("track_info", "total_tracks") { - enrichedMeta["total_tracks"] = req.TotalTracks - } - if req.shouldUpdateTag("track_info", "disc_number") { - enrichedMeta["disc_number"] = req.DiscNumber - } - if req.shouldUpdateTag("track_info", "total_discs") { - enrichedMeta["total_discs"] = req.TotalDiscs - } - if req.shouldUpdateTag("release_info", "release_date") { - enrichedMeta["release_date"] = req.ReleaseDate - } - if req.shouldUpdateTag("release_info", "isrc") { - enrichedMeta["isrc"] = req.ISRC - } - if req.shouldUpdateTag("cover", "cover") { - enrichedMeta["cover_url"] = req.CoverURL - } - if req.shouldUpdateTag("extra", "genre") { - enrichedMeta["genre"] = req.Genre - } - if req.shouldUpdateTag("extra", "label") { - enrichedMeta["label"] = req.Label - } - if req.shouldUpdateTag("extra", "copyright") { - enrichedMeta["copyright"] = req.Copyright - } - if req.shouldUpdateTag("extra", "composer") { - enrichedMeta["composer"] = req.Composer - } - return enrichedMeta -} - -func selectBestReEnrichTrack(req reEnrichRequest, tracks []ExtTrackMetadata) *ExtTrackMetadata { - if len(tracks) == 0 { - return nil - } - - downloadReq := reEnrichDownloadRequest(req) - currentISRC := strings.TrimSpace(req.ISRC) - currentAlbum := strings.TrimSpace(req.AlbumName) - effectiveTrackName := req.TrackName - if isPlaceholderReEnrichValue(effectiveTrackName) { - effectiveTrackName = "" - } - effectiveArtistName := req.ArtistName - if isPlaceholderReEnrichValue(effectiveArtistName) { - effectiveArtistName = "" - } - var best *ExtTrackMetadata - bestScore := -1 << 30 - - for i := range tracks { - track := &tracks[i] - score := 0 - exactISRCMatch := currentISRC != "" && strings.EqualFold(currentISRC, strings.TrimSpace(track.ISRC)) - titleMatches := effectiveTrackName != "" && track.Name != "" && titlesMatch(effectiveTrackName, track.Name) - artistMatches := effectiveArtistName != "" && track.Artists != "" && artistsMatch(effectiveArtistName, track.Artists) - albumMatches := currentAlbum != "" && track.AlbumName != "" && titlesMatch(currentAlbum, track.AlbumName) - - resolved := resolvedTrackInfo{ - Title: track.Name, - ArtistName: track.Artists, - ISRC: track.ISRC, - Duration: track.DurationMS / 1000, - } - verified := trackMatchesRequest(downloadReq, resolved, "ReEnrich") - - if !exactISRCMatch { - if effectiveTrackName != "" && !titleMatches { - continue - } - if effectiveArtistName != "" && !artistMatches { - continue - } - if effectiveTrackName == "" && effectiveArtistName == "" && currentAlbum != "" && !albumMatches { - continue - } - if effectiveTrackName == "" && effectiveArtistName == "" && currentAlbum == "" && !verified { - continue - } - } - - if verified { - score += 2000 - } - - if exactISRCMatch { - score += 10000 - } - if titleMatches { - score += 400 - } - if artistMatches { - score += 320 - } - if currentAlbum != "" && track.AlbumName != "" { - switch { - case albumMatches: - score += 120 - case strings.Contains(strings.ToLower(track.AlbumName), strings.ToLower(currentAlbum)), - strings.Contains(strings.ToLower(currentAlbum), strings.ToLower(track.AlbumName)): - score += 50 - } - } - - if req.DurationMs > 0 && track.DurationMS > 0 { - diff := int(req.DurationMs/1000) - (track.DurationMS / 1000) - if diff < 0 { - diff = -diff - } - if diff <= 10 { - score += 80 - } - } - - if track.ReleaseDate != "" { - score += 70 - } - if track.TrackNumber > 0 { - score += 20 - } - if track.DiscNumber > 0 { - score += 10 - } - if track.ISRC != "" { - score += 40 - } - - if best == nil || score > bestScore { - best = track - bestScore = score - } - } - - return best -} - -func extTrackFromTrackMetadata(track *TrackMetadata, providerID string) *ExtTrackMetadata { - if track == nil { - return nil - } - - deezerID := strings.TrimSpace(strings.TrimPrefix(track.SpotifyID, "deezer:")) - return &ExtTrackMetadata{ - ID: track.SpotifyID, - Name: track.Name, - Artists: track.Artists, - AlbumName: track.AlbumName, - AlbumArtist: track.AlbumArtist, - DurationMS: track.DurationMS, - CoverURL: track.Images, - Images: track.Images, - ReleaseDate: track.ReleaseDate, - TrackNumber: track.TrackNumber, - TotalTracks: track.TotalTracks, - DiscNumber: track.DiscNumber, - TotalDiscs: track.TotalDiscs, - ISRC: track.ISRC, - ProviderID: providerID, - DeezerID: deezerID, - SpotifyID: track.SpotifyID, - Composer: track.Composer, - Explicit: track.Explicit, - } -} - -func normalizeReEnrichSpotifyTrackID(raw string) string { - trimmed := strings.TrimSpace(raw) - if trimmed == "" { - return "" - } - if extracted := extractSpotifyIDFromURL(trimmed); extracted != "" { - return extracted - } - if len(trimmed) == 22 && !strings.Contains(trimmed, ":") && !strings.Contains(trimmed, "/") { - return trimmed - } - return "" -} - -func resolveReEnrichTrackFromIdentifiers(req reEnrichRequest) (*ExtTrackMetadata, error) { - deezerClient := GetDeezerClient() - downloadReq := reEnrichDownloadRequest(req) - - if isrc := strings.TrimSpace(req.ISRC); isrc != "" { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - track, err := deezerClient.SearchByISRC(ctx, isrc) - cancel() - if err == nil && track != nil { - resolved := resolvedTrackInfo{ - Title: track.Name, - ArtistName: track.Artists, - ISRC: track.ISRC, - Duration: track.DurationMS / 1000, - } - if trackMatchesRequest(downloadReq, resolved, "ReEnrich") { - return extTrackFromTrackMetadata(track, "deezer"), nil - } - } - } - - sourceTrackID := strings.TrimSpace(req.SpotifyID) - if sourceTrackID == "" { - return nil, nil - } - - deezerID := strings.TrimSpace(strings.TrimPrefix(sourceTrackID, "deezer:")) - if deezerID == sourceTrackID { - deezerID = extractDeezerIDFromURL(sourceTrackID) - } - if deezerID == "" { - spotifyID := normalizeReEnrichSpotifyTrackID(sourceTrackID) - if spotifyID != "" { - resolvedDeezerID, err := NewSongLinkClient().GetDeezerIDFromSpotify(spotifyID) - if err == nil { - deezerID = strings.TrimSpace(resolvedDeezerID) - } - } - } - if deezerID == "" { - return nil, nil - } - - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - - trackResp, err := deezerClient.GetTrack(ctx, deezerID) - if err != nil || trackResp == nil { - return nil, err - } - - track := &trackResp.Track - resolved := resolvedTrackInfo{ - Title: track.Name, - ArtistName: track.Artists, - ISRC: track.ISRC, - Duration: track.DurationMS / 1000, - } - if !trackMatchesRequest(downloadReq, resolved, "ReEnrich") { - return nil, nil - } - - return extTrackFromTrackMetadata(track, "deezer"), nil -} - -func preferredReleaseMetadata( - req DownloadRequest, - album string, - releaseDate string, - trackNumber int, - discNumber int, -) (string, string, int, int) { - preferredAlbum := strings.TrimSpace(req.AlbumName) - if preferredAlbum == "" { - preferredAlbum = album - } - - preferredReleaseDate := strings.TrimSpace(req.ReleaseDate) - if preferredReleaseDate == "" { - preferredReleaseDate = releaseDate - } - - preferredTrackNumber := req.TrackNumber - if preferredTrackNumber == 0 { - preferredTrackNumber = trackNumber - } - - preferredDiscNumber := req.DiscNumber - if preferredDiscNumber == 0 { - preferredDiscNumber = discNumber - } - - return preferredAlbum, preferredReleaseDate, preferredTrackNumber, preferredDiscNumber -} - -// ReEnrichFile re-embeds metadata, cover art, and lyrics into an existing audio file. -// When search_online is true, searches Spotify/Deezer by track name + artist to fetch -// complete metadata from the internet before embedding. -func ReEnrichFile(requestJSON string) (string, error) { - var req reEnrichRequest - - if err := json.Unmarshal([]byte(requestJSON), &req); err != nil { - return "", fmt.Errorf("failed to parse request: %w", err) - } - - if req.FilePath == "" { - return "", fmt.Errorf("file_path is required") - } - - GoLog("[ReEnrich] Starting re-enrichment\n") - - if req.SearchOnline { - found := false - - GoLog("[ReEnrich] Trying metadata providers in configured priority...\n") - manager := getExtensionManager() - if identifierTrack, err := resolveReEnrichTrackFromIdentifiers(req); err == nil && identifierTrack != nil { - GoLog("[ReEnrich] Identifier-first metadata match via %s\n", identifierTrack.ProviderID) - applyReEnrichTrackMetadata(&req, *identifierTrack) - found = true - } - - searchQuery := buildReEnrichSearchQuery(req) - if searchQuery != "" { - GoLog("[ReEnrich] Searching online metadata\n") - tracks, searchErr := manager.SearchTracksWithMetadataProviders(searchQuery, 5, true) - if searchErr == nil && len(tracks) > 0 { - track := selectBestReEnrichTrack(req, tracks) - if track != nil { - GoLog("[ReEnrich] Metadata match via %s\n", track.ProviderID) - applyReEnrichTrackMetadata(&req, *track) - found = true - } - } else if searchErr != nil { - GoLog("[ReEnrich] Metadata provider search failed: %v\n", searchErr) - } - } else { - GoLog("[ReEnrich] Skipping provider search: no usable title/artist/album query\n") - } - - if req.shouldUpdateTag("basic_tags", "album_artist") && req.AlbumArtist == "" && req.ISRC != "" { - albumArtist, err := fetchMusicBrainzAlbumArtistByISRC(req.ISRC, req.AlbumName) - if err != nil { - GoLog("[ReEnrich] Failed to get album artist from MusicBrainz: %v\n", err) - } else if strings.TrimSpace(albumArtist) != "" { - req.AlbumArtist = strings.TrimSpace(albumArtist) - GoLog("[ReEnrich] Applied album artist fallback from MusicBrainz\n") - found = true - } - } - - // Try to enrich extra metadata from ISRC if not already set. - if found && req.ISRC != "" && req.shouldUpdateAnyTag("extra", "genre", "label", "copyright") && (req.Genre == "" || req.Label == "" || req.Copyright == "") { - enrichExtraMetadataByISRC("ReEnrich", req.ISRC, &req.Genre, &req.Label, &req.Copyright) - } - - if !found { - GoLog("[ReEnrich] No online match found, using existing metadata\n") - } - } - - enrichedMeta := buildReEnrichResultMetadata(&req) - if req.PreviewOnly { - result := map[string]any{ - "method": "preview", - "success": true, - "enriched_metadata": enrichedMeta, - } - s, _ := marshalJSONString(result) - return s, nil - } - - lower := strings.ToLower(req.FilePath) - isFlac := strings.HasSuffix(lower, ".flac") - - var coverTempPath string - var coverDataBytes []byte - if req.CoverURL != "" && req.shouldUpdateTag("cover", "cover") { - coverData, err := downloadCoverToMemorySized( - req.CoverURL, - req.CoverMaxDimension, - ) - if err != nil { - GoLog("[ReEnrich] Failed to download cover: %v\n", err) - } else { - coverDataBytes = coverData - GoLog("[ReEnrich] Cover downloaded: %d KB\n", len(coverData)/1024) - // MP3/Opus requires a real image file path for Dart FFmpeg. - // FLAC uses in-memory embed and does not require temp files. - if !isFlac { - tmpFile, err := os.CreateTemp("", "reenrich_cover_*.jpg") - if err != nil { - fallbackDir := filepath.Dir(req.FilePath) - if fallbackDir == "" || fallbackDir == "." { - GoLog("[ReEnrich] Failed to create cover temp file: %v\n", err) - } else { - tmpFile, err = os.CreateTemp(fallbackDir, "reenrich_cover_*.jpg") - if err != nil { - GoLog("[ReEnrich] Failed to create cover temp file (fallback dir %s): %v\n", fallbackDir, err) - } - } - } - if err == nil && tmpFile != nil { - coverTempPath = tmpFile.Name() - if _, writeErr := tmpFile.Write(coverData); writeErr != nil { - GoLog("[ReEnrich] Failed writing cover temp file: %v\n", writeErr) - tmpFile.Close() - os.Remove(coverTempPath) - coverTempPath = "" - } else if closeErr := tmpFile.Close(); closeErr != nil { - GoLog("[ReEnrich] Failed closing cover temp file: %v\n", closeErr) - os.Remove(coverTempPath) - coverTempPath = "" - } - } - } - } - } - // Only cleanup cover temp for FLAC (native embed). - // For MP3/Opus, Dart needs the file for FFmpeg — Dart handles cleanup. - cleanupCover := true - - defer func() { - if cleanupCover && coverTempPath != "" { - os.Remove(coverTempPath) - } - }() - - // Preserve existing lyrics when online enrichment does not return a replacement. - var lyricsLRC string - if req.shouldUpdateTag("lyrics", "lyrics") { - existingLyrics, existingLyricsErr := ExtractLyrics(req.FilePath) - if existingLyricsErr == nil && strings.TrimSpace(existingLyrics) != "" { - lyricsLRC = existingLyrics - GoLog("[ReEnrich] Preserving existing embedded/sidecar lyrics\n") - } - } - - if req.EmbedLyrics && req.shouldUpdateTag("lyrics", "lyrics") { - client := NewLyricsClient() - durationSec := float64(req.DurationMs) / 1000.0 - lyrics, err := client.FetchLyricsAllSources(req.SpotifyID, req.TrackName, req.ArtistName, durationSec) - if err != nil { - GoLog("[ReEnrich] Lyrics not found: %v\n", err) - } else if !lyrics.Instrumental { - lyricsLRC = convertToLRCWithMetadata(lyrics, req.TrackName, req.ArtistName) - GoLog("[ReEnrich] Lyrics fetched: %d lines\n", len(lyrics.Lines)) - } else { - GoLog("[ReEnrich] Track is instrumental\n") - } - } - - if isFlac { - // Only populate Metadata fields for selected update groups; empty/zero - // values cause EmbedMetadata's setComment() to skip those tags, - // preserving whatever is already in the file. - metadata := Metadata{ - ArtistTagMode: req.ArtistTagMode, - } - if req.shouldUpdateTag("basic_tags", "track_name") { - metadata.Title = req.TrackName - } - if req.shouldUpdateTag("basic_tags", "artist_name") { - metadata.Artist = req.ArtistName - } - if req.shouldUpdateTag("basic_tags", "album_name") { - metadata.Album = req.AlbumName - } - if req.shouldUpdateTag("basic_tags", "album_artist") { - metadata.AlbumArtist = req.AlbumArtist - } - if req.shouldUpdateTag("track_info", "track_number") || req.shouldUpdateTag("track_info", "total_tracks") { - metadata.TrackNumber = req.TrackNumber - metadata.TotalTracks = req.TotalTracks - } - if req.shouldUpdateTag("track_info", "disc_number") || req.shouldUpdateTag("track_info", "total_discs") { - metadata.DiscNumber = req.DiscNumber - metadata.TotalDiscs = req.TotalDiscs - } - if req.shouldUpdateTag("release_info", "release_date") { - metadata.Date = req.ReleaseDate - } - if req.shouldUpdateTag("release_info", "isrc") { - metadata.ISRC = req.ISRC - } - if req.shouldUpdateTag("lyrics", "lyrics") { - if req.lyricsEmbedEnabled() { - metadata.Lyrics = lyricsLRC - } - } - if req.shouldUpdateTag("extra", "genre") { - metadata.Genre = req.Genre - } - if req.shouldUpdateTag("extra", "label") { - metadata.Label = req.Label - } - if req.shouldUpdateTag("extra", "copyright") { - metadata.Copyright = req.Copyright - } - if req.shouldUpdateTag("extra", "composer") { - metadata.Composer = req.Composer - } - - if len(coverDataBytes) > 0 { - if err := EmbedMetadataWithCoverData(req.FilePath, metadata, coverDataBytes); err != nil { - return "", fmt.Errorf("failed to embed metadata with cover: %w", err) - } - } else { - if err := EmbedMetadata(req.FilePath, metadata, ""); err != nil { - return "", fmt.Errorf("failed to embed metadata: %w", err) - } - } - if len(coverDataBytes) > 0 { - embeddedCover, err := ExtractCoverArt(req.FilePath) - if err != nil || len(embeddedCover) == 0 { - if err != nil { - return "", fmt.Errorf("metadata embedded but cover verification failed: %w", err) - } - return "", fmt.Errorf("metadata embedded but cover verification failed: empty embedded cover") - } - GoLog("[ReEnrich] Cover verified after embed (%d bytes)\n", len(embeddedCover)) - } - - GoLog("[ReEnrich] FLAC metadata embedded successfully\n") - - result := map[string]any{ - "method": "native", - "success": true, - "enriched_metadata": enrichedMeta, - "lyrics": lyricsLRC, - "write_external_lrc": req.EmbedLyrics && - req.shouldUpdateTag("lyrics", "lyrics") && - req.lyricsSidecarEnabled() && - strings.TrimSpace(lyricsLRC) != "", - } - s, _ := marshalJSONString(result) - return s, nil - } - - // Don't cleanup cover temp — Dart needs it for FFmpeg embed - cleanupCover = false - ffmpegMetadata := buildReEnrichFFmpegMetadata(&req, lyricsLRC) - - result := map[string]any{ - "method": "ffmpeg", - "cover_path": coverTempPath, - "lyrics": lyricsLRC, - "enriched_metadata": enrichedMeta, - "metadata": ffmpegMetadata, - "write_external_lrc": req.EmbedLyrics && - req.shouldUpdateTag("lyrics", "lyrics") && - req.lyricsSidecarEnabled() && - strings.TrimSpace(lyricsLRC) != "", - } - - s, _ := marshalJSONString(result) - return s, nil -} diff --git a/go_backend/exports_repo.go b/go_backend/exports_repo.go deleted file mode 100644 index 7c661b35..00000000 --- a/go_backend/exports_repo.go +++ /dev/null @@ -1,148 +0,0 @@ -package gobackend - -import ( - "fmt" - "net/url" - "path/filepath" - "strings" -) - -func InitExtensionRepoJSON(cacheDir string) error { - initExtensionRepo(cacheDir) - return nil -} - -func SetRepoRegistryURLJSON(registryURL string) error { - repo := getExtensionRepo() - if repo == nil { - return fmt.Errorf("extension repo not initialized") - } - - resolved, err := resolveRegistryURL(registryURL) - if err != nil { - return err - } - - if err := requireHTTPSURL(resolved, "registry"); err != nil { - return err - } - - repo.setRegistryURL(resolved) - return nil -} - -func ClearRepoRegistryURLJSON() error { - repo := getExtensionRepo() - if repo == nil { - return fmt.Errorf("extension repo not initialized") - } - - repo.setRegistryURL("") - repo.clearCache() - return nil -} - -func GetRepoRegistryURLJSON() (string, error) { - repo := getExtensionRepo() - if repo == nil { - return "", fmt.Errorf("extension repo not initialized") - } - - return repo.getRegistryURL(), nil -} - -func GetRepoExtensionsJSON(forceRefresh bool) (string, error) { - repo := getExtensionRepo() - if repo == nil { - return "", fmt.Errorf("extension repo not initialized") - } - - extensions, err := repo.getExtensionsWithStatus(forceRefresh) - if err != nil { - return "", err - } - - return marshalJSONString(extensions) -} - -func SearchRepoExtensionsJSON(query, category string) (string, error) { - repo := getExtensionRepo() - if repo == nil { - return "", fmt.Errorf("extension repo not initialized") - } - - extensions, err := repo.searchExtensions(query, category) - if err != nil { - return "", err - } - - return marshalJSONString(extensions) -} - -func GetRepoCategoriesJSON() (string, error) { - repo := getExtensionRepo() - if repo == nil { - return "", fmt.Errorf("extension repo not initialized") - } - - categories := repo.getCategories() - return marshalJSONString(categories) -} - -func repoExtensionPackageSuffix(downloadURL string) string { - rawPath := downloadURL - if parsed, err := url.Parse(downloadURL); err == nil { - rawPath = parsed.Path - } - - lowerPath := strings.ToLower(rawPath) - if strings.HasSuffix(lowerPath, ".sflx") { - return ".sflx" - } - if strings.HasSuffix(lowerPath, ".spotiflac-ext") { - return ".spotiflac-ext" - } - return ".spotiflac-ext" -} - -func buildRepoExtensionDestPath(destDir, extensionID, downloadURL string) (string, error) { - if strings.TrimSpace(extensionID) == "" { - return "", fmt.Errorf("invalid extension id") - } - - safeExtensionID := sanitizeFilename(extensionID) - return filepath.Join(destDir, safeExtensionID+repoExtensionPackageSuffix(downloadURL)), nil -} - -func DownloadRepoExtensionJSON(extensionID, destDir string) (string, error) { - repo := getExtensionRepo() - if repo == nil { - return "", fmt.Errorf("extension repo not initialized") - } - - ext, err := repo.findExtension(extensionID) - if err != nil { - return "", err - } - - destPath, err := buildRepoExtensionDestPath(destDir, extensionID, ext.getDownloadURL()) - if err != nil { - return "", err - } - err = repo.downloadExtension(extensionID, destPath) - if err != nil { - return "", err - } - - return destPath, nil -} - -func ClearRepoCacheJSON() error { - repo := getExtensionRepo() - if repo == nil { - return fmt.Errorf("extension repo not initialized") - } - - repo.clearCache() - return nil -} diff --git a/go_backend/exports_songlink_lyrics_supplement_test.go b/go_backend/exports_songlink_lyrics_supplement_test.go deleted file mode 100644 index 8c295e92..00000000 --- a/go_backend/exports_songlink_lyrics_supplement_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package gobackend - -import ( - "io" - "net/http" - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -func TestLyricsExportWrappersWithoutNetwork(t *testing.T) { - dir := t.TempDir() - audioPath := filepath.Join(dir, "sidecar.mp3") - if err := os.WriteFile(audioPath, []byte("audio"), 0600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "sidecar.lrc"), []byte("[00:00.00]Sidecar lyric"), 0600); err != nil { - t.Fatal(err) - } - - if lrc, err := GetLyricsLRC("spotify-1", "Song Instrumental", "Artist", "", 180000); err != nil || lrc != "[instrumental:true]" { - t.Fatalf("GetLyricsLRC instrumental = %q/%v", lrc, err) - } - if jsonText, err := GetLyricsLRCWithSource("spotify-1", "Song Instrumental", "Artist", "", 180000); err != nil || !strings.Contains(jsonText, `"instrumental":true`) { - t.Fatalf("GetLyricsLRCWithSource instrumental = %q/%v", jsonText, err) - } - if lrc, err := GetLyricsLRC("", "", "", audioPath, 0); err != nil || !strings.Contains(lrc, "Sidecar lyric") { - t.Fatalf("GetLyricsLRC sidecar = %q/%v", lrc, err) - } - if jsonText, err := GetLyricsLRCWithSource("", "", "", audioPath, 0); err != nil || !strings.Contains(jsonText, "Sidecar lyric") { - t.Fatalf("GetLyricsLRCWithSource sidecar = %q/%v", jsonText, err) - } - - outPath := filepath.Join(dir, "lyrics.lrc") - if err := FetchAndSaveLyrics("Song", "Artist", "", 0, outPath, audioPath); err != nil { - t.Fatalf("FetchAndSaveLyrics sidecar: %v", err) - } - if data := string(mustReadFile(t, outPath)); !strings.Contains(data, "Sidecar lyric") { - t.Fatalf("saved lyrics = %q", data) - } - if response, err := EmbedLyricsToFile(filepath.Join(dir, "not-flac.mp3"), "lyrics"); err != nil || !strings.Contains(response, `"success":false`) { - t.Fatalf("EmbedLyricsToFile error = %q/%v", response, err) - } - if response, err := RewriteSplitArtistTagsExport(filepath.Join(dir, "not-flac.mp3"), "A;B", "A"); err != nil || !strings.Contains(response, `"success":false`) { - t.Fatalf("RewriteSplitArtistTagsExport error = %q/%v", response, err) - } -} - -func TestLyricsExportWrappersRejectMetadataOnlySidecar(t *testing.T) { - dir := t.TempDir() - audioPath := filepath.Join(dir, "metadata-only.mp3") - if err := os.WriteFile(audioPath, []byte("audio"), 0600); err != nil { - t.Fatal(err) - } - metadataOnly := "[ti:Title]\n[ar:Artist]\n[al:Album]\n[by:SpotiFLAC Mobile]" - if err := os.WriteFile(filepath.Join(dir, "metadata-only.lrc"), []byte(metadataOnly), 0600); err != nil { - t.Fatal(err) - } - - if rawLyricsHasUsableContent(metadataOnly) { - t.Fatal("metadata-only LRC must not be considered usable") - } - if !rawLyricsHasUsableContent("[00:01.00]Actual lyric") { - t.Fatal("timed lyric must be considered usable") - } - if !rawLyricsHasUsableContent("[instrumental:true]") { - t.Fatal("instrumental marker must be considered usable") - } - - if lrc, err := GetLyricsLRC("", "", "", audioPath, 0); err != nil || lrc != "" { - t.Fatalf("GetLyricsLRC metadata-only sidecar = %q/%v", lrc, err) - } - if jsonText, err := GetLyricsLRCWithSource("", "", "", audioPath, 0); err != nil || - !strings.Contains(jsonText, `"lyrics":""`) || strings.Contains(jsonText, `"source":"Embedded"`) { - t.Fatalf("GetLyricsLRCWithSource metadata-only sidecar = %q/%v", jsonText, err) - } -} - -func TestSongLinkExportWrappersWithFakeClient(t *testing.T) { - origClient := globalSongLinkClient - defer func() { - globalSongLinkClient = origClient - SetSongLinkNetworkOptions(false, false) - }() - globalSongLinkClient = &SongLinkClient{fallbackResolver: &stubPlatformResolver{result: resolverResult{ - Links: map[string]songLinkPlatformLink{ - "spotify": {URL: "https://open.spotify.com/track/spotify-1"}, - "deezer": {URL: "https://www.deezer.com/track/101"}, - "tidal": {URL: "https://listen.tidal.com/track/202"}, - "youtubeMusic": {URL: "https://music.youtube.com/watch?v=ytm1"}, - "amazonMusic": {URL: "https://music.amazon.com/tracks/amz1"}, - "qobuz": {URL: "https://open.qobuz.com/track/303"}, - }, - }}} - songLinkClientOnce.Do(func() {}) - - SetSongLinkNetworkOptions(true, true) - if spotifyID, err := GetSpotifyIDFromDeezerTrack("101"); err != nil || spotifyID != "spotify-1" { - t.Fatalf("GetSpotifyIDFromDeezerTrack = %q/%v", spotifyID, err) - } - if tidalURL, err := GetTidalURLFromDeezerTrack("101"); err != nil || !strings.Contains(tidalURL, "tidal") { - t.Fatalf("GetTidalURLFromDeezerTrack = %q/%v", tidalURL, err) - } - if urls, err := NewSongLinkClient().GetStreamingURLs("spotify-1"); err != nil || urls["tidal"] == "" || urls["amazon"] == "" { - t.Fatalf("GetStreamingURLs = %#v/%v", urls, err) - } - if youtubeURL, err := NewSongLinkClient().GetYouTubeURLFromSpotify("spotify-1"); err != nil || !strings.Contains(youtubeURL, "youtu") { - t.Fatalf("GetYouTubeURLFromSpotify = %q/%v", youtubeURL, err) - } - if amazonURL, err := NewSongLinkClient().GetAmazonURLFromDeezer("101"); err != nil || !strings.Contains(amazonURL, "amazon") { - t.Fatalf("GetAmazonURLFromDeezer = %q/%v", amazonURL, err) - } - if youtubeURL, err := NewSongLinkClient().GetYouTubeURLFromDeezer("101"); err != nil || !strings.Contains(youtubeURL, "youtube") { - t.Fatalf("GetYouTubeURLFromDeezer = %q/%v", youtubeURL, err) - } - if deezerID, err := NewSongLinkClient().GetDeezerIDFromSpotify("spotify-1"); err != nil || deezerID != "101" { - t.Fatalf("GetDeezerIDFromSpotify = %q/%v", deezerID, err) - } - if album, err := NewSongLinkClient().CheckAlbumAvailability("album-1"); err != nil || !album.Deezer || album.DeezerID == "" { - t.Fatalf("CheckAlbumAvailability = %#v/%v", album, err) - } - if albumID, err := NewSongLinkClient().GetDeezerAlbumIDFromSpotify("album-1"); err != nil || albumID == "" { - t.Fatalf("GetDeezerAlbumIDFromSpotify = %q/%v", albumID, err) - } - if availability, err := NewSongLinkClient().CheckAvailabilityFromURL("https://www.deezer.com/track/101"); err != nil || !availability.Deezer { - t.Fatalf("CheckAvailabilityFromURL = %#v/%v", availability, err) - } - - if songLinkExtractDeezerTrackID(nil) != "" || songLinkExtractDeezerTrackID(&TrackMetadata{ExternalURL: "https://www.deezer.com/track/202"}) != "202" { - t.Fatal("songLinkExtractDeezerTrackID mismatch") - } - - if linksJSON, err := GetTrackPlatformLinksJSON("spotify-1", ""); err != nil || - !strings.Contains(linksJSON, `"tidal":"https://listen.tidal.com/track/202"`) || - !strings.Contains(linksJSON, `"spotify":`) { - t.Fatalf("GetTrackPlatformLinksJSON = %q/%v", linksJSON, err) - } - // Second call must come from the links cache, not a new request. - if cached, hit, cachedErr := trackPlatformLinksCacheLookup(GetSongLinkRegion() + "|spotify:spotify-1"); !hit || cachedErr || cached["tidal"] == "" { - t.Fatalf("trackPlatformLinksCacheLookup = %#v hit=%v err=%v", cached, hit, cachedErr) - } - if _, err := GetTrackPlatformLinksJSON("", ""); err == nil { - t.Fatal("GetTrackPlatformLinksJSON with empty IDs should error") - } - - deezerClient = &DeezerClient{ - httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - body := fakeDeezerResponse(req.URL.Path, req.URL.RawQuery) - if body == "" { - body = `{"error":"missing"}` - } - return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil - })}, - searchCache: map[string]*cacheEntry{}, - albumCache: map[string]*cacheEntry{}, - artistCache: map[string]*cacheEntry{}, - isrcCache: map[string]string{}, - cacheCleanupInterval: time.Hour, - } - deezerClientOnce.Do(func() {}) - if jsonText, err := ConvertSpotifyToDeezer("track", "spotify-1"); err != nil || !strings.Contains(jsonText, `"spotify_id":"deezer:101"`) { - t.Fatalf("ConvertSpotifyToDeezer track = %q/%v", jsonText, err) - } - if jsonText, err := ConvertSpotifyToDeezer("album", "album-1"); err != nil || jsonText == "" { - t.Fatalf("ConvertSpotifyToDeezer album = %q/%v", jsonText, err) - } -} diff --git a/go_backend/exports_supplement_test.go b/go_backend/exports_supplement_test.go deleted file mode 100644 index cc73276e..00000000 --- a/go_backend/exports_supplement_test.go +++ /dev/null @@ -1,588 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -func TestDownloadErrorClassificationPrioritizesRateLimit(t *testing.T) { - got := classifyDownloadErrorType("All providers failed. Last error: HTTP status 429: too many requests") - if got != "rate_limit" { - t.Fatalf("expected rate_limit, got %q", got) - } - - responseJSON, err := errorResponse("All services failed. Last error: rate limit exceeded") - if err != nil { - t.Fatalf("errorResponse returned error: %v", err) - } - - var response DownloadResponse - if err := json.Unmarshal([]byte(responseJSON), &response); err != nil { - t.Fatalf("invalid response JSON: %v", err) - } - if response.ErrorType != "rate_limit" { - t.Fatalf("expected rate_limit response, got %q", response.ErrorType) - } -} - -func TestDownloadErrorClassificationDetectsVerificationRequired(t *testing.T) { - cases := []string{ - "verification_required: canonical gateway challenge", - "signed session expired", - } - for _, tc := range cases { - if got := classifyDownloadErrorType(tc); got != "verification_required" { - t.Fatalf("classifyDownloadErrorType(%q) = %q, want verification_required", tc, got) - } - } -} - -func TestDownloadErrorClassificationDoesNotInferVerificationFromHTTPStatus(t *testing.T) { - cases := []string{ - "HTTP 401 for /tickets", - "HTTP 403 forbidden", - "HTTP status 428: precondition required", - "Provider returned unauthorized", - "VERIFY_REQUIRED without canonical origin and action", - "Verification required without a typed contract", - } - for _, tc := range cases { - if got := classifyDownloadErrorType(tc); got == "verification_required" { - t.Fatalf("classifyDownloadErrorType(%q) inferred verification from an ambiguous status", tc) - } - } -} - -func TestDownloadErrorClassificationPreservesProviderContracts(t *testing.T) { - tests := map[string]string{ - "PROVIDER_AUTH_FAILED": "provider_auth_failed", - "PROVIDER_UNAVAILABLE": "provider_unavailable", - "REQUEST_AUTH_INVALID": "request_auth_invalid", - "BYOA_PROVIDER_REAUTH_REQUIRED action": "provider_reauth_required", - } - for message, want := range tests { - if got := classifyDownloadErrorType(message); got != want { - t.Fatalf("classifyDownloadErrorType(%q) = %q, want %q", message, got, want) - } - } -} - -func TestOutputStorageWriteFailureDetection(t *testing.T) { - cases := []struct { - name string - errorType string - message string - want bool - }{ - { - name: "typed permission failure", - errorType: "permission", - message: "backend omitted details", - want: true, - }, - { - name: "android operation not permitted", - message: "failed to create file: open /storage/song.partial: operation not permitted", - want: true, - }, - { - name: "read only destination", - message: "open /music/song.flac: read-only file system", - want: true, - }, - { - name: "provider API error", - errorType: "api_error", - message: "HTTP 404 for /download", - want: false, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := isOutputStorageWriteFailure(tc.errorType, tc.message); got != tc.want { - t.Fatalf("isOutputStorageWriteFailure(%q, %q) = %v, want %v", tc.errorType, tc.message, got, tc.want) - } - }) - } -} - -func TestGetProviderMetadataPrefersEnabledDeezerExtension(t *testing.T) { - dir := t.TempDir() - if err := InitExtensionSystem(filepath.Join(dir, "extensions"), filepath.Join(dir, "data")); err != nil { - t.Fatalf("InitExtensionSystem: %v", err) - } - CleanupExtensions() - defer CleanupExtensions() - - ext := newTestLoadedExtension(t, ExtensionTypeMetadataProvider) - ext.ID = "deezer" - ext.Manifest.Name = "deezer" - manager := getExtensionManager() - manager.mu.Lock() - manager.extensions = map[string]*loadedExtension{ext.ID: ext} - manager.mu.Unlock() - - jsonText, err := GetProviderMetadataJSON("deezer", "album", "201") - if err != nil { - t.Fatalf("GetProviderMetadataJSON deezer album: %v", err) - } - if !strings.Contains(jsonText, "album-track") { - t.Fatalf("expected enabled deezer extension metadata, got %s", jsonText) - } -} - -func TestExtensionTrackExportsPreserveExplicitFlag(t *testing.T) { - dir := t.TempDir() - if err := InitExtensionSystem(filepath.Join(dir, "extensions"), filepath.Join(dir, "data")); err != nil { - t.Fatalf("InitExtensionSystem: %v", err) - } - - ext := newTestLoadedExtension(t, ExtensionTypeMetadataProvider) - manager := getExtensionManager() - manager.mu.Lock() - manager.extensions = map[string]*loadedExtension{ext.ID: ext} - manager.mu.Unlock() - defer CleanupExtensions() - - assertExplicit := func(name, jsonText string, err error) { - t.Helper() - if err != nil { - t.Fatalf("%s: %v", name, err) - } - if !strings.Contains(jsonText, `"explicit":true`) { - t.Fatalf("%s dropped explicit flag: %s", name, jsonText) - } - } - - jsonText, err := CustomSearchWithExtensionJSON(ext.ID, "needle", `{"filter":"tracks"}`) - assertExplicit("custom search", jsonText, err) - - for _, resourceType := range []string{"track", "album", "playlist", "artist"} { - jsonText, err = GetProviderMetadataJSON(ext.ID, resourceType, resourceType+"-1") - assertExplicit("provider metadata "+resourceType, jsonText, err) - } - - jsonText, err = HandleURLWithExtensionJSON("https://example.test/track/1") - assertExplicit("URL handler", jsonText, err) -} - -func TestSearchTracksWithMetadataProviderUsesOnlySelectedExtension(t *testing.T) { - dir := t.TempDir() - if err := InitExtensionSystem(filepath.Join(dir, "extensions"), filepath.Join(dir, "data")); err != nil { - t.Fatalf("InitExtensionSystem: %v", err) - } - - selected := newTestLoadedExtension(t, ExtensionTypeMetadataProvider) - selected.ID = "selected-metadata" - selected.Manifest.Name = selected.ID - other := newTestLoadedExtension(t, ExtensionTypeMetadataProvider) - other.ID = "other-metadata" - other.Manifest.Name = other.ID - - manager := getExtensionManager() - manager.mu.Lock() - manager.extensions = map[string]*loadedExtension{ - selected.ID: selected, - other.ID: other, - } - manager.mu.Unlock() - defer CleanupExtensions() - - jsonText, err := SearchTracksWithMetadataProviderJSON(selected.ID, "needle", 5) - if err != nil { - t.Fatalf("SearchTracksWithMetadataProviderJSON: %v", err) - } - if !strings.Contains(jsonText, `"provider_id":"selected-metadata"`) { - t.Fatalf("expected selected provider attribution, got %s", jsonText) - } - if strings.Contains(jsonText, `"provider_id":"other-metadata"`) { - t.Fatalf("unexpected fallback to another provider: %s", jsonText) - } -} - -func TestExportsJSONWrappersAndExtensionManagerSurface(t *testing.T) { - dir := t.TempDir() - dataDir := filepath.Join(dir, "data") - extensionsDir := filepath.Join(dir, "extensions") - if err := InitExtensionSystem(extensionsDir, dataDir); err != nil { - t.Fatalf("InitExtensionSystem: %v", err) - } - - ext := newTestLoadedExtension(t, ExtensionTypeMetadataProvider, ExtensionTypeDownloadProvider, ExtensionTypeLyricsProvider) - manager := getExtensionManager() - manager.mu.Lock() - if manager.extensions == nil { - manager.extensions = map[string]*loadedExtension{} - } - manager.extensions[ext.ID] = ext - manager.mu.Unlock() - defer func() { - manager.mu.Lock() - delete(manager.extensions, ext.ID) - manager.mu.Unlock() - }() - - if response, err := DownloadByStrategy(`not-json`); err != nil || !strings.Contains(response, "Invalid request") { - t.Fatalf("DownloadByStrategy invalid = %q/%v", response, err) - } - if response, err := DownloadByStrategy(`{"use_extensions":false}`); err != nil || !strings.Contains(response, "disabled") { - t.Fatalf("DownloadByStrategy disabled = %q/%v", response, err) - } - - InitItemProgress("item-1") - ClearItemProgress("item-1") - CancelDownload("item-1") - if GetAllDownloadProgress() == "" || GetAllDownloadProgressDelta(0) == "" { - t.Fatal("expected progress JSON") - } - CleanupConnections() - - cuePath, audioPath := writeExportCueFixture(t, dir) - if jsonText, err := ParseCueSheet(cuePath, ""); err != nil { - t.Fatalf("ParseCueSheet = %q/%v", jsonText, err) - } else { - var parsed CueSplitInfo - if err := json.Unmarshal([]byte(jsonText), &parsed); err != nil { - t.Fatalf("decode ParseCueSheet: %v", err) - } - if parsed.AudioPath != audioPath { - t.Fatalf("ParseCueSheet audio path = %q want %q", parsed.AudioPath, audioPath) - } - } - if jsonText, err := ScanCueSheetForLibrary(cuePath, "", "virtual.cue", 111); err != nil || !strings.Contains(jsonText, "cue+wav") { - t.Fatalf("ScanCueSheetForLibrary = %q/%v", jsonText, err) - } - if jsonText, err := ScanCueSheetForLibraryWithCoverCacheKey(cuePath, "", "virtual.cue", 111, "cover-key"); err != nil || !strings.Contains(jsonText, "cue+wav") { - t.Fatalf("ScanCueSheetForLibraryWithCoverCacheKey = %q/%v", jsonText, err) - } - - apePath := filepath.Join(dir, "edit.ape") - if err := os.WriteFile(apePath, []byte("audio"), 0600); err != nil { - t.Fatal(err) - } - editJSON := `{"title":"Edited","artist":"Artist","track_number":"1","track_total":"2","disc_number":"1","disc_total":"1"}` - if response, err := EditFileMetadata(apePath, editJSON); err != nil || !strings.Contains(response, "native_ape") { - t.Fatalf("EditFileMetadata ape = %q/%v", response, err) - } - if response, err := EditFileMetadata(filepath.Join(dir, "edit.mp3"), editJSON); err != nil || !strings.Contains(response, "ffmpeg") { - t.Fatalf("EditFileMetadata ffmpeg = %q/%v", response, err) - } - misnamedM4APath := filepath.Join(dir, "misnamed.flac") - if err := os.WriteFile(misnamedM4APath, buildM4AFileWithIlst(buildM4ATextTag("\xa9nam", "Misnamed"), true), 0600); err != nil { - t.Fatal(err) - } - replayGainJSON := `{"replaygain_track_gain":"-1 dB","replaygain_track_peak":"0.9"}` - if response, err := EditFileMetadata(misnamedM4APath, replayGainJSON); err != nil || !strings.Contains(response, "native_m4a_replaygain") { - t.Fatalf("EditFileMetadata misnamed m4a replaygain = %q/%v", response, err) - } - if _, err := EditFileMetadata(apePath, `not-json`); err == nil { - t.Fatal("expected invalid metadata JSON") - } - if !hasOnlyM4AReplayGainFields(map[string]string{"replaygain_track_gain": "-1 dB"}) { - t.Fatal("expected replaygain-only fields") - } - if hasOnlyM4AReplayGainFields(map[string]string{"title": "Song"}) { - t.Fatal("expected non-replaygain field rejection") - } - - AllowDownloadDir(dir) - if err := SetDownloadDirectory(dir); err != nil { - t.Fatalf("SetDownloadDirectory: %v", err) - } - if batchJSON, err := CheckDuplicatesBatch(dir, `[{"isrc":"","track_name":"Song","artist_name":"Artist"}]`); err != nil || !strings.Contains(batchJSON, "Song") { - t.Fatalf("CheckDuplicatesBatch = %q/%v", batchJSON, err) - } - _ = PreBuildDuplicateIndex(dir) - InvalidateDuplicateIndex(dir) - if filename, err := BuildFilename("{artist} - {title}", `{"artist":"A/B","title":"Song?"}`); err != nil || filename == "" { - t.Fatalf("BuildFilename = %q/%v", filename, err) - } - if _, err := BuildFilename("{title}", `not-json`); err == nil { - t.Fatal("expected BuildFilename JSON error") - } - if got := SanitizeFilename(`A/B:C*D?`); strings.ContainsAny(got, `/:*?`) { - t.Fatalf("SanitizeFilename = %q", got) - } - - if GetTrackCacheSize() != 0 { - t.Fatal("expected empty track cache") - } - ClearTrackIDCache() - - if err := SetLyricsProvidersJSON(`["lrclib","apple_music"]`); err != nil { - t.Fatalf("SetLyricsProvidersJSON: %v", err) - } - if providers, err := GetLyricsProvidersJSON(); err != nil || !strings.Contains(providers, "lrclib") { - t.Fatalf("GetLyricsProvidersJSON = %q/%v", providers, err) - } - if available, err := GetAvailableLyricsProvidersJSON(); err != nil || available == "" { - t.Fatalf("GetAvailableLyricsProvidersJSON = %q/%v", available, err) - } - if err := SetLyricsFetchOptionsJSON(`{"include_translation_netease":true}`); err != nil { - t.Fatalf("SetLyricsFetchOptionsJSON: %v", err) - } - if opts, err := GetLyricsFetchOptionsJSON(); err != nil || opts == "" { - t.Fatalf("GetLyricsFetchOptionsJSON = %q/%v", opts, err) - } - - if err := SetProviderPriorityJSON(`["coverage-ext"]`); err != nil { - t.Fatalf("SetProviderPriorityJSON: %v", err) - } - if jsonText, err := GetProviderPriorityJSON(); err != nil || !strings.Contains(jsonText, "coverage-ext") { - t.Fatalf("GetProviderPriorityJSON = %q/%v", jsonText, err) - } - if err := SetExtensionFallbackProviderIDsJSON(`["coverage-ext"]`); err != nil { - t.Fatalf("SetExtensionFallbackProviderIDsJSON: %v", err) - } - if err := SetExtensionFallbackProviderIDsJSON(""); err != nil { - t.Fatalf("reset extension fallback IDs: %v", err) - } - if err := SetMetadataProviderPriorityJSON(`["coverage-ext"]`); err != nil { - t.Fatalf("SetMetadataProviderPriorityJSON: %v", err) - } - if jsonText, err := GetMetadataProviderPriorityJSON(); err != nil || !strings.Contains(jsonText, "coverage-ext") { - t.Fatalf("GetMetadataProviderPriorityJSON = %q/%v", jsonText, err) - } - - if err := SetExtensionSettingsJSON(ext.ID, `{"quality":"lossless","_secret":"hidden"}`); err != nil { - t.Fatalf("SetExtensionSettingsJSON: %v", err) - } - if settingsJSON, err := GetExtensionSettingsJSON(ext.ID); err != nil || !strings.Contains(settingsJSON, "quality") { - t.Fatalf("GetExtensionSettingsJSON = %q/%v", settingsJSON, err) - } - if err := SetExtensionSettingsJSON(ext.ID, `not-json`); err == nil { - t.Fatal("expected settings JSON error") - } - - if jsonText, err := SearchTracksWithMetadataProvidersJSON("song", 5, true); err != nil || !strings.Contains(jsonText, "search-1") { - t.Fatalf("SearchTracksWithMetadataProvidersJSON = %q/%v", jsonText, err) - } - if jsonText, err := GetProviderMetadataJSON(ext.ID, "track", "track-1"); err != nil || !strings.Contains(jsonText, "Track track-1") { - t.Fatalf("GetProviderMetadataJSON track = %q/%v", jsonText, err) - } - for _, resourceType := range []string{"album", "playlist", "artist"} { - if jsonText, err := GetProviderMetadataJSON(ext.ID, resourceType, resourceType+"-1"); err != nil || jsonText == "" { - t.Fatalf("GetProviderMetadataJSON %s = %q/%v", resourceType, jsonText, err) - } - } - if _, err := GetProviderMetadataJSON("", "track", "id"); err == nil { - t.Fatal("expected empty provider ID error") - } - if _, err := GetProviderMetadataJSON(ext.ID, "unsupported", "id"); err == nil { - t.Fatal("expected unsupported provider type") - } - if firstNonEmptyTrimmed(" ", " value ") != "value" { - t.Fatal("expected first trimmed value") - } - requestJSON := `{"use_extensions":true,"use_fallback":false,"service":"coverage-ext","source":"coverage-ext","track_name":"Song","artist_name":"Artist","album_name":"Album","output_dir":"` + escapeJSONPath(dir) + `","output_ext":".flac","quality":"LOSSLESS"}` - if jsonText, err := DownloadWithExtensionsJSON(requestJSON); err != nil || !strings.Contains(jsonText, "coverage-ext") { - t.Fatalf("DownloadWithExtensionsJSON = %q/%v", jsonText, err) - } - if _, err := DownloadWithExtensionsJSON(`not-json`); err == nil { - t.Fatal("expected DownloadWithExtensionsJSON JSON error") - } - - SetExtensionAuthCodeByID(ext.ID, "code") - SetExtensionTokensByID(ext.ID, "access", "refresh", 60) - if !IsExtensionAuthenticatedByID(ext.ID) { - t.Fatal("expected authenticated extension") - } - if pending, err := GetExtensionPendingAuthJSON(ext.ID); err != nil || pending != "" { - t.Fatalf("GetExtensionPendingAuthJSON = %q/%v", pending, err) - } - ClearExtensionPendingAuthByID(ext.ID) - if all, err := GetAllPendingAuthRequestsJSON(); err != nil || all == "" { - t.Fatalf("GetAllPendingAuthRequestsJSON = %q/%v", all, err) - } - - ffmpegCommandsMu.Lock() - ffmpegCommands["cmd-1"] = &FFmpegCommand{ExtensionID: ext.ID, Arguments: []string{"-version"}, InputPath: "in", OutputPath: "out"} - ffmpegCommandsMu.Unlock() - if cmdJSON, err := GetPendingFFmpegCommandJSON("cmd-1"); err != nil || !strings.Contains(cmdJSON, "cmd-1") { - t.Fatalf("GetPendingFFmpegCommandJSON = %q/%v", cmdJSON, err) - } - if all, err := GetAllPendingFFmpegCommandsJSON(); err != nil || !strings.Contains(all, "cmd-1") { - t.Fatalf("GetAllPendingFFmpegCommandsJSON = %q/%v", all, err) - } - SetFFmpegCommandResultByID("cmd-1", true, "ok", "") - ClearFFmpegCommand("cmd-1") - if empty, err := GetPendingFFmpegCommandJSON("missing"); err != nil || empty != "" { - t.Fatalf("missing ffmpeg = %q/%v", empty, err) - } - - enrichedJSON, err := EnrichTrackWithExtensionJSON(ext.ID, `{"id":"track-1","name":"Old","artists":"Artist"}`) - if err != nil || !strings.Contains(enrichedJSON, "Enriched") { - t.Fatalf("EnrichTrackWithExtensionJSON = %q/%v", enrichedJSON, err) - } - if sameJSON, err := EnrichTrackWithExtensionJSON("missing", `{"name":"Old"}`); err != nil || !strings.Contains(sameJSON, "Old") { - t.Fatalf("missing EnrichTrackWithExtensionJSON = %q/%v", sameJSON, err) - } - - deezerClient = &DeezerClient{ - httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - body := fakeDeezerResponse(req.URL.Path, req.URL.RawQuery) - status := http.StatusOK - if body == "" { - status = http.StatusNotFound - body = `{"error":"missing"}` - } - return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil - })}, - searchCache: map[string]*cacheEntry{}, - albumCache: map[string]*cacheEntry{}, - artistCache: map[string]*cacheEntry{}, - isrcCache: map[string]string{}, - cacheCleanupInterval: time.Hour, - } - deezerClientOnce.Do(func() {}) - for _, item := range []struct { - typ string - id string - }{ - {"track", "101"}, - {"album", "201"}, - {"artist", "301"}, - {"playlist", "401"}, - } { - if jsonText, err := GetDeezerMetadata(item.typ, item.id); err != nil || jsonText == "" { - t.Fatalf("GetDeezerMetadata %s = %q/%v", item.typ, jsonText, err) - } - } - if _, err := GetDeezerMetadata("bad", "1"); err == nil { - t.Fatal("expected unsupported Deezer metadata type") - } - if jsonText, err := GetDeezerExtendedMetadata("101"); err != nil || !strings.Contains(jsonText, "Label") { - t.Fatalf("GetDeezerExtendedMetadata = %q/%v", jsonText, err) - } - if _, err := GetDeezerExtendedMetadata(""); err == nil { - t.Fatal("expected empty Deezer metadata ID error") - } - if jsonText, err := SearchDeezerByISRC("USRC17607839"); err != nil || !strings.Contains(jsonText, "deezer:101") { - t.Fatalf("SearchDeezerByISRC = %q/%v", jsonText, err) - } - if jsonText, err := SearchDeezerByISRCForItemID("USRC17607839", "item-isrc"); err != nil || !strings.Contains(jsonText, "deezer:101") { - t.Fatalf("SearchDeezerByISRCForItemID = %q/%v", jsonText, err) - } - - customJSON, err := CustomSearchWithExtensionJSON(ext.ID, "needle", `{"filter":"tracks"}`) - if err != nil || !strings.Contains(customJSON, "Custom needle") { - t.Fatalf("CustomSearchWithExtensionJSON = %q/%v", customJSON, err) - } - if customJSON, err := CustomSearchWithExtensionJSONWithRequestID(ext.ID, "needle", `not-json`, "req-custom"); err != nil || !strings.Contains(customJSON, "custom-1") { - t.Fatalf("CustomSearchWithExtensionJSONWithRequestID = %q/%v", customJSON, err) - } - if found := FindURLHandlerJSON("https://example.test/track/1"); found != ext.ID { - t.Fatalf("FindURLHandlerJSON = %q", found) - } - if handledJSON, err := HandleURLWithExtensionJSON("https://example.test/track/1"); err != nil || !strings.Contains(handledJSON, "url-track") { - t.Fatalf("HandleURLWithExtensionJSON = %q/%v", handledJSON, err) - } - v2Input := `{"path":"` + escapeJSONPath(filepath.Join(dir, "song.flac")) + `","uri":"content://song","name":"song.flac","mime_type":"audio/flac","size":10}` - if postJSON, err := RunPostProcessingV2JSON(v2Input, `not-json`); err != nil || !strings.Contains(postJSON, "success") { - t.Fatalf("RunPostProcessingV2JSON = %q/%v", postJSON, err) - } - if feedJSON, err := GetExtensionHomeFeedJSON(ext.ID); err != nil || !strings.Contains(feedJSON, "home-1") { - t.Fatalf("GetExtensionHomeFeedJSON = %q/%v", feedJSON, err) - } - if feedJSON, err := GetExtensionHomeFeedJSONWithRequestID(ext.ID, "req-home"); err != nil || !strings.Contains(feedJSON, "home-1") { - t.Fatalf("GetExtensionHomeFeedJSONWithRequestID = %q/%v", feedJSON, err) - } - CancelExtensionRequestJSON("req-home") - - storeDir := filepath.Join(dir, "store") - if err := InitExtensionRepoJSON(storeDir); err != nil { - t.Fatalf("InitExtensionRepoJSON: %v", err) - } - if err := SetRepoRegistryURLJSON("https://registry.example.com/index.json"); err != nil { - t.Fatalf("SetRepoRegistryURLJSON: %v", err) - } - store := getExtensionRepo() - store.cache = &repoRegistry{Extensions: []repoExtension{{ - ID: "coverage-ext", - Name: "coverage-ext", - Version: "1.0.0", - Description: "Coverage", - Category: CategoryMetadata, - Tags: []string{"metadata"}, - DownloadURL: "https://registry.example.com/coverage.spotiflac-ext", - }}} - store.cacheTime = time.Now() - if registryURL, err := GetRepoRegistryURLJSON(); err != nil || registryURL == "" { - t.Fatalf("GetRepoRegistryURLJSON = %q/%v", registryURL, err) - } - if storeJSON, err := GetRepoExtensionsJSON(false); err != nil || !strings.Contains(storeJSON, "coverage-ext") { - t.Fatalf("GetRepoExtensionsJSON = %q/%v", storeJSON, err) - } - if storeJSON, err := SearchRepoExtensionsJSON("coverage", CategoryMetadata); err != nil || !strings.Contains(storeJSON, "coverage-ext") { - t.Fatalf("SearchRepoExtensionsJSON = %q/%v", storeJSON, err) - } - if catsJSON, err := GetRepoCategoriesJSON(); err != nil || !strings.Contains(catsJSON, "metadata") { - t.Fatalf("GetRepoCategoriesJSON = %q/%v", catsJSON, err) - } - if dest, err := buildRepoExtensionDestPath( - dir, - "coverage/ext", - "https://registry.example.com/coverage.spotiflac-ext", - ); err != nil || !strings.HasSuffix(dest, ".spotiflac-ext") { - t.Fatalf("buildRepoExtensionDestPath = %q/%v", dest, err) - } - if dest, err := buildRepoExtensionDestPath( - dir, - "coverage/ext", - "https://registry.example.com/coverage.sflx", - ); err != nil || !strings.HasSuffix(dest, ".sflx") { - t.Fatalf("buildRepoExtensionDestPath sflx = %q/%v", dest, err) - } - if _, err := buildRepoExtensionDestPath( - dir, - " ", - "https://registry.example.com/coverage.sflx", - ); err == nil { - t.Fatal("expected invalid extension id") - } - if err := ClearRepoCacheJSON(); err != nil { - t.Fatalf("ClearRepoCacheJSON: %v", err) - } - if err := ClearRepoRegistryURLJSON(); err != nil { - t.Fatalf("ClearRepoRegistryURLJSON: %v", err) - } - - SetLibraryCoverCacheDirJSON(filepath.Join(dir, "covers")) - libraryDir := filepath.Join(dir, "library") - if err := os.MkdirAll(libraryDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(libraryDir, "Artist - Song.mp3"), []byte("not mp3"), 0600); err != nil { - t.Fatal(err) - } - if scanJSON, err := ScanLibraryFolderJSON(libraryDir); err != nil || !strings.Contains(scanJSON, "Song") { - t.Fatalf("ScanLibraryFolderJSON = %q/%v", scanJSON, err) - } - if scanJSON, err := ScanLibraryFolderIncrementalJSON(libraryDir, `[]`); err != nil || !strings.Contains(scanJSON, "Song") { - t.Fatalf("ScanLibraryFolderIncrementalJSON = %q/%v", scanJSON, err) - } - snapshotPath := filepath.Join(dir, "snapshot.json") - if err := os.WriteFile(snapshotPath, []byte(`[]`), 0600); err != nil { - t.Fatal(err) - } - if scanJSON, err := ScanLibraryFolderIncrementalFromSnapshotJSON(libraryDir, snapshotPath); err != nil || !strings.Contains(scanJSON, "Song") { - t.Fatalf("ScanLibraryFolderIncrementalFromSnapshotJSON = %q/%v", scanJSON, err) - } - if GetLibraryScanProgressJSON() == "" { - t.Fatal("expected scan progress JSON") - } - CancelLibraryScanJSON() - if metadataJSON, err := ReadAudioMetadataJSON(filepath.Join(libraryDir, "missing.mp3")); err != nil || metadataJSON == "" { - t.Fatalf("ReadAudioMetadataJSON = %q/%v", metadataJSON, err) - } - if metadataJSON, err := ReadAudioMetadataWithHintAndCoverCacheKeyJSON(filepath.Join(libraryDir, "missing.mp3"), "Missing", "key"); err != nil || metadataJSON == "" { - t.Fatalf("ReadAudioMetadataWithHintAndCoverCacheKeyJSON = %q/%v", metadataJSON, err) - } -} diff --git a/go_backend/exports_test.go b/go_backend/exports_test.go deleted file mode 100644 index 2fc80af0..00000000 --- a/go_backend/exports_test.go +++ /dev/null @@ -1,845 +0,0 @@ -package gobackend - -import ( - "context" - "encoding/json" - "fmt" - "testing" -) - -func TestSetExtensionFallbackProviderIDsJSONEmptyStringResetsDefault(t *testing.T) { - original := GetExtensionFallbackProviderIDs() - defer SetExtensionFallbackProviderIDs(original) - - SetExtensionFallbackProviderIDs([]string{"custom-ext"}) - - if err := SetExtensionFallbackProviderIDsJSON(""); err != nil { - t.Fatalf("SetExtensionFallbackProviderIDsJSON returned error: %v", err) - } - - if got := GetExtensionFallbackProviderIDs(); got != nil { - t.Fatalf("expected nil fallback provider list after reset, got %v", got) - } -} - -func TestBuildDownloadSuccessResponsePrefersRequestedAlbumMetadata(t *testing.T) { - req := DownloadRequest{ - TrackName: "Bonus Track", - ArtistName: "Artist", - AlbumName: "Album (Deluxe)", - AlbumArtist: "Artist", - ReleaseDate: "2024-01-01", - TrackNumber: 14, - DiscNumber: 1, - ISRC: "REQ123", - CoverURL: "https://example.com/cover.jpg", - Genre: "Pop", - Label: "Label", - Copyright: "Copyright", - } - - result := DownloadResult{ - Title: "Bonus Track", - Artist: "Artist", - Album: "Album", - ReleaseDate: "2023-12-01", - TrackNumber: 2, - DiscNumber: 9, - ISRC: "RES456", - } - - resp := buildDownloadSuccessResponse( - req, - result, - "download-provider", - "ok", - "/tmp/test.flac", - false, - ) - - if resp.Album != req.AlbumName { - t.Fatalf("album = %q, want %q", resp.Album, req.AlbumName) - } - if resp.ReleaseDate != req.ReleaseDate { - t.Fatalf("release date = %q, want %q", resp.ReleaseDate, req.ReleaseDate) - } - if resp.TrackNumber != req.TrackNumber { - t.Fatalf("track number = %d, want %d", resp.TrackNumber, req.TrackNumber) - } - if resp.DiscNumber != req.DiscNumber { - t.Fatalf("disc number = %d, want %d", resp.DiscNumber, req.DiscNumber) - } - if resp.Artist != result.Artist { - t.Fatalf("artist = %q, want provider artist %q", resp.Artist, result.Artist) - } - if resp.ISRC != result.ISRC { - t.Fatalf("isrc = %q, want provider isrc %q", resp.ISRC, result.ISRC) - } -} - -func TestPreferredReleaseMetadataPrefersRequestValues(t *testing.T) { - album, releaseDate, trackNumber, discNumber := preferredReleaseMetadata( - DownloadRequest{ - AlbumName: "Album (Deluxe Edition)", - ReleaseDate: "2024-01-01", - TrackNumber: 13, - DiscNumber: 2, - }, - "Album", - "2023-01-01", - 3, - 1, - ) - - if album != "Album (Deluxe Edition)" { - t.Fatalf("album = %q", album) - } - if releaseDate != "2024-01-01" { - t.Fatalf("release date = %q", releaseDate) - } - if trackNumber != 13 { - t.Fatalf("track number = %d", trackNumber) - } - if discNumber != 2 { - t.Fatalf("disc number = %d", discNumber) - } -} - -func TestBuildDownloadSuccessResponsePreservesRequestedTrackAndCover(t *testing.T) { - req := DownloadRequest{ - TrackName: "Album Track Title", - ArtistName: "Artist", - AlbumName: "Original Album", - AlbumArtist: "Artist", - CoverURL: "https://cdn.source.test/original-album.jpg", - ISRC: "USAAA2600001", - } - - result := DownloadResult{ - Title: "Single Version Title", - Artist: "Artist", - Album: "Single Version Title", - CoverURL: "https://cdn.provider.test/single.jpg", - ISRC: "USAAA2600001", - } - - resp := buildDownloadSuccessResponse( - req, - result, - "generic-provider", - "ok", - "/tmp/test.flac", - false, - ) - overlayExtensionDownloadMetadata(&resp, &ExtDownloadResult{ - Title: result.Title, - Artist: result.Artist, - Album: result.Album, - CoverURL: result.CoverURL, - ISRC: result.ISRC, - }) - applyExtensionRequestFallbacks(&resp, req) - - if resp.Title != req.TrackName { - t.Fatalf("title = %q, want requested title %q", resp.Title, req.TrackName) - } - if resp.Album != req.AlbumName { - t.Fatalf("album = %q, want requested album %q", resp.Album, req.AlbumName) - } - if resp.CoverURL != req.CoverURL { - t.Fatalf("cover url = %q, want requested album cover %q", resp.CoverURL, req.CoverURL) - } -} - -func TestBuildDownloadSuccessResponseFallsBackToProviderTrackAndCover(t *testing.T) { - req := DownloadRequest{ - ArtistName: "Artist", - AlbumName: "Album", - AlbumArtist: "Artist", - } - - result := DownloadResult{ - Title: "Track", - Artist: "Artist", - Album: "Album", - CoverURL: "https://cdn.provider.test/cover.jpg", - } - - resp := buildDownloadSuccessResponse( - req, - result, - "generic-provider", - "ok", - "/tmp/test.flac", - false, - ) - - if resp.Title != result.Title { - t.Fatalf("title = %q, want provider fallback %q", resp.Title, result.Title) - } - if resp.CoverURL != result.CoverURL { - t.Fatalf("cover url = %q, want provider fallback %q", resp.CoverURL, result.CoverURL) - } -} - -func TestBuildDownloadSuccessResponseReturnsResolvedProviderFilename(t *testing.T) { - req := DownloadRequest{ - TrackName: "Track", - ArtistName: "Artist", - DownloadProvider: "download-provider", - ProviderTrackID: "998877", - FilenameFormat: "{artist} - {title} [{isrc}] [{provider}-{provider_id}]", - OutputExt: ".flac", - } - result := DownloadResult{ - ISRC: "USABC1234567", - ActualExtension: ".m4a", - } - - resp := buildDownloadSuccessResponse( - req, - result, - "download-provider", - "ok", - "/proc/self/fd/10", - false, - ) - - want := "Artist - Track [USABC1234567] [download-provider-998877].m4a" - if resp.ResolvedFileName != want { - t.Fatalf("resolved filename = %q, want %q", resp.ResolvedFileName, want) - } - if resp.ProviderTrackID != req.ProviderTrackID { - t.Fatalf("provider track ID = %q, want %q", resp.ProviderTrackID, req.ProviderTrackID) - } -} - -func TestBuildDownloadSuccessResponseNormalizesDecryptionDescriptor(t *testing.T) { - req := DownloadRequest{ - TrackName: "Track", - ArtistName: "Artist", - } - - result := DownloadResult{ - Title: "Track", - Artist: "Artist", - DecryptionKey: "00112233", - } - - resp := buildDownloadSuccessResponse( - req, - result, - "download-provider", - "ok", - "/tmp/test.m4a", - false, - ) - - if resp.Decryption == nil { - t.Fatal("expected decryption descriptor to be present") - } - if resp.Decryption.Strategy != genericFFmpegMOVDecryptionStrategy { - t.Fatalf("strategy = %q", resp.Decryption.Strategy) - } - if resp.Decryption.Key != result.DecryptionKey { - t.Fatalf("key = %q, want %q", resp.Decryption.Key, result.DecryptionKey) - } -} - -func TestFormatMusicBrainzGenrePrefersHighestCountTag(t *testing.T) { - got := formatMusicBrainzGenre([]musicBrainzTag{ - {Name: "art pop", Count: 3}, - {Name: "pop", Count: 8}, - {Name: "dance pop", Count: 5}, - }) - - if got != "Pop" { - t.Fatalf("genre = %q, want %q", got, "Pop") - } -} - -func TestSelectMusicBrainzAlbumArtistPrefersMatchingRelease(t *testing.T) { - releases := []musicBrainzRelease{ - { - Title: "Other Album", - ArtistCredit: []musicBrainzArtistCredit{ - {Name: "Wrong Artist"}, - }, - }, - { - Title: "Target Album", - ArtistCredit: []musicBrainzArtistCredit{ - {Name: "Artist A", JoinPhrase: " & "}, - {Name: "Artist B"}, - }, - }, - } - - got := selectMusicBrainzAlbumArtist(releases, "Target Album") - if got != "Artist A & Artist B" { - t.Fatalf("album artist = %q, want matching release artist credit", got) - } -} - -func TestEnrichRequestExtendedMetadataUsesMusicBrainzAlbumArtist(t *testing.T) { - origDeezerFetcher := fetchDeezerExtendedMetadataByISRC - origMusicBrainzGenreFetcher := fetchMusicBrainzGenreByISRC - origMusicBrainzAlbumArtistFetcher := fetchMusicBrainzAlbumArtistByISRC - defer func() { - fetchDeezerExtendedMetadataByISRC = origDeezerFetcher - fetchMusicBrainzGenreByISRC = origMusicBrainzGenreFetcher - fetchMusicBrainzAlbumArtistByISRC = origMusicBrainzAlbumArtistFetcher - }() - - fetchDeezerExtendedMetadataByISRC = func(ctx context.Context, isrc string) (*AlbumExtendedMetadata, error) { - return &AlbumExtendedMetadata{}, nil - } - fetchMusicBrainzGenreByISRC = func(isrc string) (string, error) { - return "", fmt.Errorf("no genre") - } - fetchMusicBrainzAlbumArtistByISRC = func(isrc string, albumName string) (string, error) { - if isrc != "TESTISRC" || albumName != "Target Album" { - t.Fatalf("unexpected MusicBrainz args: %q / %q", isrc, albumName) - } - return "MusicBrainz Album Artist", nil - } - - req := DownloadRequest{ - ISRC: "TESTISRC", - ArtistName: "Track Artist", - AlbumName: "Target Album", - } - - enrichRequestExtendedMetadata(&req) - - if req.AlbumArtist != "MusicBrainz Album Artist" { - t.Fatalf("album artist = %q, want MusicBrainz value", req.AlbumArtist) - } -} - -func TestEnrichRequestExtendedMetadataDoesNotFallbackAlbumArtistToTrackArtist(t *testing.T) { - origDeezerFetcher := fetchDeezerExtendedMetadataByISRC - origMusicBrainzGenreFetcher := fetchMusicBrainzGenreByISRC - origMusicBrainzAlbumArtistFetcher := fetchMusicBrainzAlbumArtistByISRC - defer func() { - fetchDeezerExtendedMetadataByISRC = origDeezerFetcher - fetchMusicBrainzGenreByISRC = origMusicBrainzGenreFetcher - fetchMusicBrainzAlbumArtistByISRC = origMusicBrainzAlbumArtistFetcher - }() - - fetchDeezerExtendedMetadataByISRC = func(ctx context.Context, isrc string) (*AlbumExtendedMetadata, error) { - return &AlbumExtendedMetadata{}, nil - } - fetchMusicBrainzGenreByISRC = func(isrc string) (string, error) { - return "", fmt.Errorf("no genre") - } - fetchMusicBrainzAlbumArtistByISRC = func(isrc string, albumName string) (string, error) { - return "", fmt.Errorf("no album artist") - } - - req := DownloadRequest{ - ISRC: "TESTISRC", - ArtistName: "Track Artist", - AlbumName: "Target Album", - } - - enrichRequestExtendedMetadata(&req) - - if req.AlbumArtist != "" { - t.Fatalf("album artist = %q, want empty when MusicBrainz has no value", req.AlbumArtist) - } -} - -func TestEnrichExtraMetadataByISRCFallsBackToMusicBrainzGenre(t *testing.T) { - origDeezerFetcher := fetchDeezerExtendedMetadataByISRC - origMusicBrainzFetcher := fetchMusicBrainzGenreByISRC - defer func() { - fetchDeezerExtendedMetadataByISRC = origDeezerFetcher - fetchMusicBrainzGenreByISRC = origMusicBrainzFetcher - }() - - fetchDeezerExtendedMetadataByISRC = func(ctx context.Context, isrc string) (*AlbumExtendedMetadata, error) { - return nil, nil - } - fetchMusicBrainzGenreByISRC = func(isrc string) (string, error) { - if isrc != "TEST123" { - t.Fatalf("unexpected isrc: %q", isrc) - } - return "Alternative Rock", nil - } - - genre := "" - label := "" - copyright := "" - enrichExtraMetadataByISRC("DownloadWithFallback", "TEST123", &genre, &label, ©right) - - if genre != "Alternative Rock" { - t.Fatalf("genre = %q, want fallback genre", genre) - } - if label != "" { - t.Fatalf("label = %q, want empty", label) - } - if copyright != "" { - t.Fatalf("copyright = %q, want empty", copyright) - } -} - -func TestEnrichExtraMetadataByISRCPrefersDeezerGenre(t *testing.T) { - origDeezerFetcher := fetchDeezerExtendedMetadataByISRC - origMusicBrainzFetcher := fetchMusicBrainzGenreByISRC - defer func() { - fetchDeezerExtendedMetadataByISRC = origDeezerFetcher - fetchMusicBrainzGenreByISRC = origMusicBrainzFetcher - }() - - musicBrainzCalled := false - fetchDeezerExtendedMetadataByISRC = func(ctx context.Context, isrc string) (*AlbumExtendedMetadata, error) { - return &AlbumExtendedMetadata{ - Genre: "Synthpop", - Label: "EMI", - Copyright: "(C) Test", - }, nil - } - fetchMusicBrainzGenreByISRC = func(isrc string) (string, error) { - musicBrainzCalled = true - return "Rock", nil - } - - genre := "" - label := "" - copyright := "" - enrichExtraMetadataByISRC("DownloadWithFallback", "TEST456", &genre, &label, ©right) - - if genre != "Synthpop" { - t.Fatalf("genre = %q, want Deezer genre", genre) - } - if label != "EMI" { - t.Fatalf("label = %q, want Deezer label", label) - } - if copyright != "(C) Test" { - t.Fatalf("copyright = %q, want Deezer copyright", copyright) - } - if musicBrainzCalled { - t.Fatal("expected MusicBrainz not to be called when Deezer already provides genre") - } -} - -func TestApplyReEnrichTrackMetadataPreservesExistingReleaseDateWhenCandidateMissing(t *testing.T) { - req := reEnrichRequest{ - SpotifyID: "source-track-id", - AlbumName: "Original Album", - ReleaseDate: "2024-01-01", - ISRC: "REQ123", - } - - applyReEnrichTrackMetadata(&req, ExtTrackMetadata{ - AlbumName: "Original Album (Deluxe)", - ReleaseDate: "", - ISRC: "", - }) - - if req.ReleaseDate != "2024-01-01" { - t.Fatalf("release date = %q, want existing value preserved", req.ReleaseDate) - } - if req.AlbumName != "Original Album (Deluxe)" { - t.Fatalf("album = %q, want updated album", req.AlbumName) - } - if req.ISRC != "REQ123" { - t.Fatalf("isrc = %q, want existing value preserved", req.ISRC) - } -} - -func TestApplyReEnrichTrackMetadataKeepsReleaseIdentityOnAlbumMismatch(t *testing.T) { - req := reEnrichRequest{ - TrackName: "Afsana", - ArtistName: "Artist Name", - AlbumName: "Original Soundtrack", - CoverURL: "https://covers/original.jpg", - TrackNumber: 3, - ReleaseDate: "2005-01-01", - } - - applyReEnrichTrackMetadata(&req, ExtTrackMetadata{ - Name: "Afsana", - Artists: "Artist Name", - AlbumName: "The Hit Machine", - CoverURL: "https://covers/compilation.jpg", - TrackNumber: 17, - ReleaseDate: "2010-01-01", - ISRC: "NEW123", - }) - - if req.AlbumName != "Original Soundtrack" { - t.Fatalf("album = %q, want original release kept", req.AlbumName) - } - if req.CoverURL != "https://covers/original.jpg" { - t.Fatalf("cover = %q, want original release cover kept", req.CoverURL) - } - if req.TrackNumber != 3 { - t.Fatalf("track number = %d, want original position kept", req.TrackNumber) - } - if req.ReleaseDate != "2005-01-01" { - t.Fatalf("release date = %q, want original date kept", req.ReleaseDate) - } - if req.ISRC != "NEW123" { - t.Fatalf("isrc = %q, want recording-level fields still enriched", req.ISRC) - } -} - -func TestApplyReEnrichTrackMetadataReplacesStalePlaylistAlbumWhenRequested(t *testing.T) { - req := reEnrichRequest{ - TrackName: "Song", - ArtistName: "Artist", - AlbumName: "Road Trip Playlist", - CoverURL: "https://covers/playlist.jpg", - TrackNumber: 42, - ReleaseDate: "", - ReplaceReleaseMetadata: true, - } - - applyReEnrichTrackMetadata(&req, ExtTrackMetadata{ - Name: "Song", - Artists: "Artist", - AlbumName: "Actual Album", - AlbumArtist: "Artist", - CoverURL: "https://covers/album.jpg", - TrackNumber: 3, - ReleaseDate: "2024-01-01", - }) - - if req.AlbumName != "Actual Album" { - t.Fatalf("album = %q, want actual album", req.AlbumName) - } - if req.AlbumArtist != "Artist" { - t.Fatalf("album artist = %q", req.AlbumArtist) - } - if req.CoverURL != "https://covers/album.jpg" { - t.Fatalf("cover = %q", req.CoverURL) - } - if req.TrackNumber != 3 { - t.Fatalf("track number = %d", req.TrackNumber) - } - if req.ReleaseDate != "2024-01-01" { - t.Fatalf("release date = %q", req.ReleaseDate) - } -} - -func TestSelectBestReEnrichTrackPrefersCandidateWithReleaseDate(t *testing.T) { - req := reEnrichRequest{ - TrackName: "Song Title", - ArtistName: "Artist Name", - AlbumName: "Album Name", - ReleaseDate: "", - DurationMs: 180000, - } - - tracks := []ExtTrackMetadata{ - { - ID: "first", - Name: "Song Title", - Artists: "Artist Name", - AlbumName: "Album Name", - DurationMS: 180000, - ReleaseDate: "", - ProviderID: "metadata-a", - }, - { - ID: "second", - Name: "Song Title", - Artists: "Artist Name", - AlbumName: "Album Name", - DurationMS: 180000, - ReleaseDate: "2024-03-09", - ProviderID: "metadata-b", - }, - } - - best := selectBestReEnrichTrack(req, tracks) - if best == nil { - t.Fatal("expected a selected track") - } - if best.ID != "second" { - t.Fatalf("selected track = %q, want candidate with release date", best.ID) - } -} - -func TestSelectBestReEnrichTrackRejectsMismatchedSearchResults(t *testing.T) { - req := reEnrichRequest{ - TrackName: "Song Title", - ArtistName: "Artist Name", - AlbumName: "Album Name", - DurationMs: 180000, - } - - tracks := []ExtTrackMetadata{ - { - ID: "wrong-rich-metadata", - Name: "Different Song", - Artists: "Different Artist", - AlbumName: "Album Name", - DurationMS: 180000, - ReleaseDate: "2024-03-09", - TrackNumber: 4, - DiscNumber: 1, - ISRC: "WRONG1234567", - ProviderID: "metadata-b", - }, - } - - if best := selectBestReEnrichTrack(req, tracks); best != nil { - t.Fatalf("selected track = %q, want no match", best.ID) - } -} - -func TestSelectBestReEnrichTrackAllowsExactISRCDespiteMetadataMismatch(t *testing.T) { - req := reEnrichRequest{ - TrackName: "Song Title", - ArtistName: "Artist Name", - ISRC: "USRC17607839", - DurationMs: 999999000, - } - - tracks := []ExtTrackMetadata{ - { - ID: "same-isrc", - Name: "Different Song", - Artists: "Different Artist", - DurationMS: 180000, - ISRC: "USRC17607839", - ProviderID: "metadata-b", - }, - } - - best := selectBestReEnrichTrack(req, tracks) - if best == nil { - t.Fatal("expected exact ISRC candidate to be selected") - } - if best.ID != "same-isrc" { - t.Fatalf("selected track = %q, want exact ISRC candidate", best.ID) - } -} - -func TestSelectBestReEnrichTrackPlaceholderFallsBackToAlbum(t *testing.T) { - req := reEnrichRequest{ - TrackName: "Unknown Title", - ArtistName: "Unknown Artist", - AlbumName: "Harry Styles", - DurationMs: 180000, - } - - tracks := []ExtTrackMetadata{ - { - ID: "album-match", - Name: "Sign of the Times", - Artists: "Harry Styles", - AlbumName: "Harry Styles", - DurationMS: 180000, - ProviderID: "metadata-b", - }, - } - - best := selectBestReEnrichTrack(req, tracks) - if best == nil { - t.Fatal("expected album-matching candidate to be selected when title/artist are placeholders") - } - if best.ID != "album-match" { - t.Fatalf("selected track = %q, want album-match", best.ID) - } -} - -func TestBuildReEnrichFFmpegMetadataOmitsEmptyFields(t *testing.T) { - req := reEnrichRequest{ - TrackName: "Song", - ArtistName: "Artist", - AlbumName: "Album", - AlbumArtist: "", - ReleaseDate: "", - TrackNumber: 0, - DiscNumber: 0, - ISRC: "", - Genre: "", - Label: "", - Copyright: "", - } - - metadata := buildReEnrichFFmpegMetadata(&req, "") - - if metadata["TITLE"] != "Song" { - t.Fatalf("title = %q", metadata["TITLE"]) - } - if metadata["ARTIST"] != "Artist" { - t.Fatalf("artist = %q", metadata["ARTIST"]) - } - if metadata["ALBUM"] != "Album" { - t.Fatalf("album = %q", metadata["ALBUM"]) - } - - for _, key := range []string{ - "ALBUMARTIST", - "DATE", - "TRACKNUMBER", - "DISCNUMBER", - "ISRC", - "GENRE", - "ORGANIZATION", - "COPYRIGHT", - "LYRICS", - "UNSYNCEDLYRICS", - } { - if _, exists := metadata[key]; exists { - t.Fatalf("did not expect key %s in metadata: %#v", key, metadata) - } - } -} - -func TestBuildReEnrichSearchQuerySkipsPlaceholderArtist(t *testing.T) { - req := reEnrichRequest{ - TrackName: "Sign of the Times", - ArtistName: "Unknown Artist", - AlbumName: "Harry Styles", - } - - query := buildReEnrichSearchQuery(req) - if query != "Sign of the Times" { - t.Fatalf("query = %q", query) - } - - req = reEnrichRequest{ - TrackName: "Unknown Title", - ArtistName: "Unknown Artist", - AlbumName: "Harry Styles", - } - query = buildReEnrichSearchQuery(req) - if query != "Harry Styles" { - t.Fatalf("fallback album query = %q", query) - } -} - -func TestApplyReEnrichTrackMetadataCopiesComposerAndTotals(t *testing.T) { - req := reEnrichRequest{} - - applyReEnrichTrackMetadata(&req, ExtTrackMetadata{ - Name: "Resolved Song", - Artists: "Resolved Artist", - TrackNumber: 7, - TotalTracks: 12, - DiscNumber: 2, - TotalDiscs: 3, - Composer: "Composer", - }) - - if req.TrackNumber != 7 || req.TotalTracks != 12 { - t.Fatalf("track metadata = %d/%d", req.TrackNumber, req.TotalTracks) - } - if req.DiscNumber != 2 || req.TotalDiscs != 3 { - t.Fatalf("disc metadata = %d/%d", req.DiscNumber, req.TotalDiscs) - } - if req.TrackName != "Resolved Song" || req.ArtistName != "Resolved Artist" { - t.Fatalf("basic tags = %q / %q", req.TrackName, req.ArtistName) - } - if req.Composer != "Composer" { - t.Fatalf("composer = %q", req.Composer) - } -} - -func TestBuildReEnrichFFmpegMetadataFormatsTotalsAndComposer(t *testing.T) { - req := reEnrichRequest{ - TrackNumber: 7, - TotalTracks: 12, - DiscNumber: 2, - TotalDiscs: 3, - Composer: "Composer", - } - - metadata := buildReEnrichFFmpegMetadata(&req, "") - - if metadata["TRACKNUMBER"] != "7/12" { - t.Fatalf("TRACKNUMBER = %q", metadata["TRACKNUMBER"]) - } - if metadata["DISCNUMBER"] != "2/3" { - t.Fatalf("DISCNUMBER = %q", metadata["DISCNUMBER"]) - } - if metadata["COMPOSER"] != "Composer" { - t.Fatalf("COMPOSER = %q", metadata["COMPOSER"]) - } -} - -func TestReEnrichGranularISRCDoesNotChangeReleaseDate(t *testing.T) { - req := reEnrichRequest{ - ReleaseDate: "2020-01-02", - ISRC: "", - UpdateFields: []string{"isrc"}, - } - - applyReEnrichTrackMetadata(&req, ExtTrackMetadata{ - ReleaseDate: "2025-04-05", - ISRC: "USRC17607839", - }) - - if req.ISRC != "USRC17607839" { - t.Fatalf("isrc = %q", req.ISRC) - } - if req.ReleaseDate != "2020-01-02" { - t.Fatalf("release date = %q, want existing value", req.ReleaseDate) - } - metadata := buildReEnrichFFmpegMetadata(&req, "") - if metadata["ISRC"] != "USRC17607839" { - t.Fatalf("ISRC metadata = %q", metadata["ISRC"]) - } - if _, exists := metadata["DATE"]; exists { - t.Fatalf("granular ISRC update unexpectedly included DATE: %#v", metadata) - } -} - -func TestBuildReEnrichResultMetadataOnlyIncludesSelectedGranularTags(t *testing.T) { - req := reEnrichRequest{ - TrackName: "Existing title", - AlbumArtist: "Resolved album artist", - ISRC: "USRC17607839", - Genre: "Rock", - UpdateFields: []string{"album_artist", "isrc"}, - } - - metadata := buildReEnrichResultMetadata(&req) - if metadata["album_artist"] != "Resolved album artist" { - t.Fatalf("album_artist = %#v", metadata["album_artist"]) - } - if metadata["isrc"] != "USRC17607839" { - t.Fatalf("isrc = %#v", metadata["isrc"]) - } - for _, key := range []string{"track_name", "release_date", "genre"} { - if _, exists := metadata[key]; exists { - t.Fatalf("unexpected key %q in preview metadata: %#v", key, metadata) - } - } -} - -func TestReEnrichPreviewReturnsBeforeTouchingAudioFile(t *testing.T) { - request, err := json.Marshal(reEnrichRequest{ - FilePath: "content://library/nonexistent.flac", - TrackName: "Song", - ArtistName: "Artist", - ISRC: "USRC17607839", - UpdateFields: []string{"isrc"}, - PreviewOnly: true, - }) - if err != nil { - t.Fatal(err) - } - - raw, err := ReEnrichFile(string(request)) - if err != nil { - t.Fatalf("preview unexpectedly touched the nonexistent file: %v", err) - } - var result map[string]any - if err := json.Unmarshal([]byte(raw), &result); err != nil { - t.Fatal(err) - } - if result["method"] != "preview" || result["success"] != true { - t.Fatalf("result = %#v", result) - } - metadata, ok := result["enriched_metadata"].(map[string]any) - if !ok || metadata["isrc"] != "USRC17607839" { - t.Fatalf("preview metadata = %#v", result["enriched_metadata"]) - } -} diff --git a/go_backend/extension_availability_verification_test.go b/go_backend/extension_availability_verification_test.go deleted file mode 100644 index a89f6744..00000000 --- a/go_backend/extension_availability_verification_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package gobackend - -import ( - "strings" - "testing" - "time" - - "github.com/dop251/goja" -) - -func TestAvailabilityPreservesCanonicalVerificationOnFailure(t *testing.T) { - for _, script := range []string{ - `throw new Error("VERIFY_REQUIRED");`, - `return null;`, - `return undefined;`, - `return {available:false};`, - } { - t.Run(script, func(t *testing.T) { - ext := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - t.Cleanup(func() { teardownExtension(ext) }) - if err := ext.ensureRuntimeReady(); err != nil { - t.Fatal(err) - } - if err := ext.VM.Set("requireChallenge", func(goja.FunctionCall) goja.Value { - ext.runtime.noteVerificationRequired("https://example.test/challenge") - return goja.Undefined() - }); err != nil { - t.Fatal(err) - } - if _, err := ext.VM.RunString(`extension.checkAvailability = function(){ requireChallenge(); ` + script + ` };`); err != nil { - t.Fatal(err) - } - _, err := newExtensionProviderWrapper(ext).CheckAvailabilityForItemID("", "Song", "Artist", "", "", "", "", 180000, "") - if err == nil || classifyDownloadErrorType(err.Error()) != "verification_required" || !strings.Contains(err.Error(), ext.ID) { - t.Fatalf("canonical verification not preserved: %v", err) - } - if ext.runtime.consumeVerificationRequired() != "" { - t.Fatal("verification evidence leaked into the next call") - } - }) - } -} - -func TestAvailabilityDoesNotPromoteUntrustedOrStaleVerification(t *testing.T) { - ext := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - t.Cleanup(func() { teardownExtension(ext) }) - if err := ext.ensureRuntimeReady(); err != nil { - t.Fatal(err) - } - if _, err := ext.VM.RunString(`extension.checkAvailability = function(){ throw new Error("VERIFY_REQUIRED"); };`); err != nil { - t.Fatal(err) - } - ext.runtime.noteVerificationRequired("https://example.test/stale-challenge") - _, err := newExtensionProviderWrapper(ext).CheckAvailabilityForItemID("", "Song", "Artist", "", "", "", "", 180000, "") - if err == nil || classifyDownloadErrorType(err.Error()) == "verification_required" { - t.Fatalf("untrusted exception inherited a previous challenge: %v", err) - } -} - -func TestExtensionVerificationErrorsRequireOwnedPendingChallenge(t *testing.T) { - for _, tc := range []struct { - name string - message string - pendingOwner string - challengeAge time.Duration - authURL string - wantChallenge bool - }{ - {"fresh", "VERIFY_REQUIRED", "coverage-ext", 0, "https://example.test/verify", true}, - {"missing", "VERIFY_REQUIRED", "", 0, "", false}, - {"other-provider", "VERIFY_REQUIRED", "other-provider", 0, "https://example.test/verify", false}, - {"expired", "VERIFY_REQUIRED", "coverage-ext", pendingAuthRequestTTL + time.Second, "https://example.test/verify", false}, - {"future", "VERIFY_REQUIRED", "coverage-ext", -time.Minute, "https://example.test/verify", false}, - {"missing-url", "VERIFY_REQUIRED", "coverage-ext", 0, "", false}, - {"network-error", "network timeout", "coverage-ext", 0, "https://example.test/verify", false}, - {"provider-auth", "PROVIDER_AUTH_FAILED: VERIFY_REQUIRED", "coverage-ext", 0, "https://example.test/verify", false}, - {"http-status", "HTTP 401: VERIFY_REQUIRED", "coverage-ext", 0, "https://example.test/verify", false}, - {"cancelled", "cancelled: VERIFY_REQUIRED", "coverage-ext", 0, "https://example.test/verify", false}, - {"throwing-getter", "ordinary failure", "coverage-ext", 0, "https://example.test/verify", false}, - } { - t.Run(tc.name, func(t *testing.T) { - ext := newTestLoadedExtension(t, ExtensionTypeMetadataProvider, ExtensionTypeDownloadProvider) - t.Cleanup(func() { - ClearPendingAuthRequest(ext.ID) - ClearPendingAuthRequest(tc.pendingOwner) - teardownExtension(ext) - }) - if err := ext.ensureRuntimeReady(); err != nil { - t.Fatal(err) - } - if err := ext.VM.Set("fixtureMessage", tc.message); err != nil { - t.Fatal(err) - } - if _, err := ext.VM.RunString(`extension.searchTracks = extension.checkAvailability = function() { throw new Error(fixtureMessage); };`); err != nil { - t.Fatal(err) - } - if tc.name == "throwing-getter" { - if _, err := ext.VM.RunString(`extension.searchTracks = extension.checkAvailability = function() { - throw {toString: function() { return "ordinary failure"; }, get message() { throw new Error("broken getter"); }}; - };`); err != nil { - t.Fatal(err) - } - } - if tc.pendingOwner != "" { - if err := registerPendingAuthRequest(&PendingAuthRequest{ - ExtensionID: tc.pendingOwner, - AuthURL: tc.authURL, - CreatedAt: time.Now().Add(-tc.challengeAge), - }); err != nil { - t.Fatal(err) - } - } - provider := newExtensionProviderWrapper(ext) - _, searchErr := provider.SearchTracks("Song Artist", 1) - _, availabilityErr := provider.CheckAvailabilityForItemID("", "Song", "Artist", "", "", "", "", 180000, "") - for _, err := range []error{searchErr, availabilityErr} { - if err == nil { - t.Fatal("extension error was lost") - } - if got := classifyDownloadErrorType(err.Error()) == "verification_required"; got != tc.wantChallenge { - t.Fatalf("verification=%v, want %v: %v", got, tc.wantChallenge, err) - } - } - if tc.pendingOwner == "" && GetPendingAuthRequest(ext.ID) != nil { - t.Fatal("error classification created a challenge") - } - }) - } -} diff --git a/go_backend/extension_cookie_jar_test.go b/go_backend/extension_cookie_jar_test.go deleted file mode 100644 index 3d3e54bd..00000000 --- a/go_backend/extension_cookie_jar_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package gobackend - -import ( - "net/http" - "net/url" - "testing" - "time" -) - -func TestSimpleCookieJarReplacesAndExpiresCookies(t *testing.T) { - jar, err := newSimpleCookieJar() - if err != nil { - t.Fatalf("newSimpleCookieJar: %v", err) - } - u, _ := url.Parse("https://api.example.com/path") - - jar.SetCookies(u, []*http.Cookie{{Name: "session", Value: "old", Path: "/"}}) - jar.SetCookies(u, []*http.Cookie{{Name: "session", Value: "new", Path: "/"}}) - cookies := jar.Cookies(u) - if len(cookies) != 1 || cookies[0].Value != "new" { - t.Fatalf("replacement cookies = %#v", cookies) - } - - jar.SetCookies(u, []*http.Cookie{{ - Name: "session", - Value: "expired", - Path: "/", - Expires: time.Now().Add(-time.Hour), - MaxAge: -1, - }}) - if cookies := jar.Cookies(u); len(cookies) != 0 { - t.Fatalf("expired cookies = %#v", cookies) - } -} - -func TestSimpleCookieJarClearAndScope(t *testing.T) { - jar, err := newSimpleCookieJar() - if err != nil { - t.Fatalf("newSimpleCookieJar: %v", err) - } - apiURL, _ := url.Parse("https://api.example.com/private/resource") - otherURL, _ := url.Parse("https://other.example.com/private/resource") - publicURL, _ := url.Parse("https://api.example.com/public") - - jar.SetCookies(apiURL, []*http.Cookie{{Name: "session", Value: "value", Path: "/private"}}) - if len(jar.Cookies(apiURL)) != 1 { - t.Fatal("expected cookie for matching host and path") - } - if len(jar.Cookies(otherURL)) != 0 || len(jar.Cookies(publicURL)) != 0 { - t.Fatal("cookie escaped its host or path scope") - } - - jar.Clear() - if len(jar.Cookies(apiURL)) != 0 { - t.Fatal("Clear retained cookies") - } -} diff --git a/go_backend/extension_download_quality.go b/go_backend/extension_download_quality.go deleted file mode 100644 index ab79d0ad..00000000 --- a/go_backend/extension_download_quality.go +++ /dev/null @@ -1,109 +0,0 @@ -package gobackend - -import ( - "fmt" - "strings" -) - -// Quality IDs belong to their declaring provider. Keep the requested audio -// kind when translating an ID instead of treating the first option as best. -func extensionQualityKind(option QualityOption, manifest *ExtensionManifest) string { - if kind := strings.ToLower(strings.TrimSpace(option.Kind)); kind != "" { - switch kind { - case "lossless", "lossy", "spatial": - return kind - } - } - // Compatibility for installed packages that predate the kind declaration. - // Use only the ID and label: descriptions may mention fallback formats. - token := strings.ToLower(strings.TrimSpace(option.ID)) - label := strings.ToLower(option.Label) - text := token + " " + label - if strings.Contains(text, "atmos") || strings.Contains(text, "dolby") || - strings.Contains(text, "surround") || token == "ac4" || token == "ac-4" || - token == "eac3" || token == "e-ac-3" || token == "ec-3" { - return "spatial" - } - if strings.Contains(text, "lossless") || strings.Contains(text, "flac") || - strings.Contains(text, "alac") || strings.Contains(text, "24-bit") || - strings.Contains(text, "16-bit") || token == "hi_res" { - return "lossless" - } - if token == "high" || token == "low" || strings.Contains(text, "mp3") || - strings.Contains(text, "aac") || strings.Contains(text, "opus") || - strings.Contains(text, "vorbis") { - return "lossy" - } - if token == "best" || token == "default" || token == "" { - if manifest != nil { - switch strings.ToLower(strings.TrimSpace(fmt.Sprint(manifest.Capabilities["downloadFallbackTier"]))) { - case "hi_res", "lossless": - return "lossless" - case "low_res": - return "lossy" - } - } - } - return "" -} - -func findExtensionQuality(manifest *ExtensionManifest, requested string) (QualityOption, bool) { - if manifest != nil && requested != "" { - for _, option := range manifest.QualityOptions { - if strings.EqualFold(strings.TrimSpace(option.ID), requested) { - return option, true - } - } - } - return QualityOption{}, false -} - -func resolveExtensionDownloadQuality(requested string, source, target *ExtensionManifest) (string, error) { - requested = strings.TrimSpace(requested) - if target == nil || len(target.QualityOptions) == 0 { - return requested, nil // Legacy providers without a quality declaration. - } - option, sourceRecognizes := findExtensionQuality(source, requested) - if !sourceRecognizes { - option = QualityOption{ID: requested} - } - kind := extensionQualityKind(option, source) - if exact, ok := findExtensionQuality(target, requested); ok { - targetKind := extensionQualityKind(exact, target) - if source == target || (kind != "" && kind == targetKind) || - (kind == "" && targetKind != "spatial") { - return strings.TrimSpace(exact.ID), nil - } - } - if kind == "" { - // An unknown foreign/default token must never opt into spatial audio. - kind = "lossless" - } - allowedKinds := []string{kind} - if kind == "spatial" || kind == "lossy" { - allowedKinds = append(allowedKinds, "lossless") - } - for _, allowed := range allowedKinds { - for _, candidate := range target.QualityOptions { - id := strings.TrimSpace(candidate.ID) - if id != "" && extensionQualityKind(candidate, target) == allowed { - return id, nil - } - } - } - return "", fmt.Errorf("provider %s has no compatible %s quality for %q", target.Name, kind, requested) -} - -func requestedQualityManifest(req DownloadRequest, manager *extensionManager) *ExtensionManifest { - if manager == nil { - return nil - } - for _, id := range []string{req.Service, req.Source} { - if ext, err := manager.GetExtension(strings.TrimSpace(id)); err == nil && ext.Manifest != nil { - if _, recognized := findExtensionQuality(ext.Manifest, strings.TrimSpace(req.Quality)); recognized { - return ext.Manifest - } - } - } - return nil -} diff --git a/go_backend/extension_download_quality_test.go b/go_backend/extension_download_quality_test.go deleted file mode 100644 index 45f4401b..00000000 --- a/go_backend/extension_download_quality_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package gobackend - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func qualityTestManifest(name string, options ...QualityOption) *ExtensionManifest { - return &ExtensionManifest{Name: name, QualityOptions: options} -} - -func TestExtensionQualityKeepsAudioKindAcrossProviders(t *testing.T) { - source := qualityTestManifest("source", QualityOption{ID: "best", Label: "FLAC Best Available"}, QualityOption{ID: "ac4", Label: "Dolby Atmos"}) - target := qualityTestManifest("target", - QualityOption{ID: "DOLBY_ATMOS", Label: "Dolby Atmos", Description: "falls back to FLAC"}, - QualityOption{ID: "HI_RES_LOSSLESS", Label: "HiRes FLAC"}, - QualityOption{ID: "LOSSLESS", Label: "Lossless"}, - QualityOption{ID: "HIGH", Label: "High"}, - ) - for _, tc := range []struct{ requested, want string }{ - {"best", "HI_RES_LOSSLESS"}, {"DEFAULT", "HI_RES_LOSSLESS"}, {"", "HI_RES_LOSSLESS"}, - {"LOSSLESS", "LOSSLESS"}, {"lossless", "LOSSLESS"}, {"HI_RES_LOSSLESS", "HI_RES_LOSSLESS"}, - {"ac4", "DOLBY_ATMOS"}, {"DOLBY_ATMOS", "DOLBY_ATMOS"}, {"HIGH", "HIGH"}, - } { - t.Run(tc.requested, func(t *testing.T) { - got, err := resolveExtensionDownloadQuality(tc.requested, source, target) - if err != nil || got != tc.want { - t.Fatalf("quality=%q err=%v; want %q", got, err, tc.want) - } - }) - } - if got, err := resolveExtensionDownloadQuality("best", source, source); err != nil || got != "best" { - t.Fatalf("same provider selection changed: %s %v", got, err) - } -} - -func TestExtensionQualityUsesDeclarationsAndRejectsLosslessDowngrade(t *testing.T) { - source := qualityTestManifest("source", QualityOption{ID: "studio", Kind: "lossless"}) - target := qualityTestManifest("target", QualityOption{ID: "studio", Kind: "spatial"}, QualityOption{ID: "pcm", Kind: "lossless"}) - if got, err := resolveExtensionDownloadQuality("studio", source, target); err != nil || got != "pcm" { - t.Fatalf("foreign ID collision: %s %v", got, err) - } - for _, target := range []*ExtensionManifest{ - qualityTestManifest("spatial-only", QualityOption{ID: "DOLBY_ATMOS"}), - qualityTestManifest("lossy-only", QualityOption{ID: "mp3_128"}), - qualityTestManifest("unknown", QualityOption{ID: "custom"}), - } { - if _, err := resolveExtensionDownloadQuality("studio", source, target); err == nil { - t.Fatalf("accepted incompatible provider %s", target.Name) - } - } - legacy := qualityTestManifest("legacy") - if got, err := resolveExtensionDownloadQuality("custom", nil, legacy); err != nil || got != "custom" { - t.Fatalf("legacy: %s %v", got, err) - } - lossless := qualityTestManifest("flac", QualityOption{ID: "flac"}) - if got, err := resolveExtensionDownloadQuality("DOLBY_ATMOS", nil, lossless); err != nil || got != "flac" { - t.Fatalf("explicit Atmos FLAC fallback: %s %v", got, err) - } -} - -func TestExtensionQualityBestUsesProviderKind(t *testing.T) { - source := qualityTestManifest("lossy-source", QualityOption{ID: "best", Label: "Best Audio"}) - source.Capabilities = map[string]any{"downloadFallbackTier": "low_res"} - target := qualityTestManifest("target", QualityOption{ID: "DOLBY_ATMOS"}, QualityOption{ID: "LOSSLESS"}, QualityOption{ID: "HIGH"}) - if got, err := resolveExtensionDownloadQuality("best", source, target); err != nil || got != "HIGH" { - t.Fatalf("lossy best: %s %v", got, err) - } - if kind := extensionQualityKind(QualityOption{ID: "DOLBY_ATMOS", Description: "best available FLAC fallback"}, target); kind != "spatial" { - t.Fatal(kind) - } -} - -func TestDownloadFallbackAndVerificationResumePreserveLosslessQuality(t *testing.T) { - source := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - source.ID, source.Manifest.Name = "quality-source", "quality-source" - source.Manifest.QualityOptions = []QualityOption{{ID: "best", Label: "FLAC Best Available"}} - target := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - target.ID, target.Manifest.Name = "quality-target", "quality-target" - target.Manifest.QualityOptions = []QualityOption{{ID: "DOLBY_ATMOS"}, {ID: "HI_RES_LOSSLESS"}, {ID: "LOSSLESS"}} - for ext, script := range map[*loadedExtension]string{ - source: `registerExtension({checkAvailability:function(){return {available:false};},download:function(){return {success:false,error_type:"not_found",error_message:"source unavailable"};}});`, - target: `registerExtension({checkAvailability:function(){return {available:true,track_id:"target-track"};},download:function(id,quality){return {success:false,error_type:"not_found",error_message:"observed-quality:"+quality};}});`, - } { - if err := os.WriteFile(filepath.Join(ext.SourceDir, "index.js"), []byte(script), 0600); err != nil { - t.Fatal(err) - } - } - manager := getExtensionManager() - manager.mu.Lock() - previous := manager.extensions - manager.extensions = map[string]*loadedExtension{source.ID: source, target.ID: target} - manager.mu.Unlock() - priority, fallback := GetProviderPriority(), GetExtensionFallbackProviderIDs() - SetProviderPriority([]string{source.ID, target.ID}) - SetExtensionFallbackProviderIDs(nil) - t.Cleanup(func() { - teardownExtension(source) - teardownExtension(target) - manager.mu.Lock() - manager.extensions = previous - manager.mu.Unlock() - SetProviderPriority(priority) - SetExtensionFallbackProviderIDs(fallback) - resetPreparedDownloadRequestCacheForTest() - }) - for _, mode := range []string{"fallback", "direct-source", "verification-resume"} { - t.Run(mode, func(t *testing.T) { - req := DownloadRequest{Service: source.ID, ItemID: "quality-" + mode, TrackName: "Song", ArtistName: "Artist", AlbumName: "Album", ISRC: "USRC17607839", ReleaseDate: "2026-01-01", OutputDir: t.TempDir(), FilenameFormat: "{title}", Quality: "best", UseFallback: true} - if mode == "direct-source" { - req.Source = source.ID - } - if mode == "verification-resume" { - cacheUnpreparedDownloadRequest(downloadPreparationKey(req), req) - } - response, err := DownloadWithExtensionFallback(req) - if err != nil || response == nil || !strings.Contains(response.Error, "observed-quality:HI_RES_LOSSLESS") { - t.Fatalf("response=%+v err=%v", response, err) - } - }) - } - // A direct/verified invocation uses the same translation, without relying - // on the normal provider-loop branch to repair the foreign token. - req := DownloadRequest{Service: source.ID, Quality: "best", OutputDir: t.TempDir(), TrackName: "Direct"} - var lastErr error - var errorType string - var retryAfter int - attemptExtensionDownload(req, target, newExtensionProviderWrapper(target), "target-track", req.Quality, target.ID, nil, false, &lastErr, &errorType, &retryAfter) - if lastErr == nil || !strings.Contains(lastErr.Error(), "observed-quality:HI_RES_LOSSLESS") { - t.Fatal(lastErr) - } -} diff --git a/go_backend/extension_fallback.go b/go_backend/extension_fallback.go deleted file mode 100644 index 0e5db510..00000000 --- a/go_backend/extension_fallback.go +++ /dev/null @@ -1,803 +0,0 @@ -package gobackend - -import ( - "errors" - "fmt" - "strings" - "time" -) - -// attemptExtensionDownload runs a single provider.Download attempt: builds the -// output path, reports progress, and on success assembles the full -// DownloadResponse (overlay, request fallbacks, optional title/artist/composer -// fallback, metadata embed, ISRC index). On failure it writes into -// lastErr/lastErrType/lastRetryAfterSeconds for the current attempt so callers -// can handle verification_required/stop-fallback without inheriting another -// provider's error or retry delay. -// cancelledOuter true means the caller must return (nil, ErrDownloadCancelled). -func attemptExtensionDownload( - req DownloadRequest, - ext *loadedExtension, - provider *extensionProviderWrapper, - trackID, quality, providerLabel string, - preparedContext map[string]any, - applyTitleFallback bool, - lastErr *error, - lastErrType *string, - lastRetryAfterSeconds *int, -) (resp *DownloadResponse, cancelledOuter bool) { - *lastErr = nil - *lastErrType = "" - *lastRetryAfterSeconds = 0 - resolvedQuality, qualityErr := resolveExtensionDownloadQuality( - quality, requestedQualityManifest(req, getExtensionManager()), ext.Manifest, - ) - if qualityErr != nil { - *lastErr = qualityErr - *lastErrType = "quality_unavailable" - *lastRetryAfterSeconds = 0 - return nil, false - } - if resolvedQuality != quality { - GoLog("[DownloadWithExtensionFallback] Provider %s maps requested quality %q to %q\n", providerLabel, quality, resolvedQuality) - } - quality = resolvedQuality - req.Quality = resolvedQuality - req.DownloadProvider = strings.TrimSpace(providerLabel) - req.ProviderTrackID = strings.TrimSpace(trackID) - preparedContext = extensionPreparedDownloadContext(req, preparedContext) - outputPath := buildOutputPathForExtension(req, ext) - if shouldReuseExistingOutput(req, outputPath) { - result := DownloadResult{FilePath: outputPath} - enrichResultQualityFromFile(&result) - built := buildDownloadSuccessResponse( - req, - result, - providerLabel, - "File already exists", - outputPath, - true, - ) - if req.ItemID != "" { - CompleteItemProgress(req.ItemID) - } - GoLog("[DownloadWithExtensionFallback] Keeping existing output instead of replacing it: %s\n", outputPath) - return &built, false - } - if req.ItemID != "" { - SetItemPreparingStage(req.ItemID, "resolving_stream") - } - - result, err := provider.DownloadPrepared(trackID, quality, outputPath, req.ItemID, preparedContext, func(percent int) { - if req.ItemID != "" { - normalized := float64(percent) / 100.0 - if normalized < 0 { - normalized = 0 - } - if normalized > 1 { - normalized = 1 - } - SetItemProgress(req.ItemID, normalized, 0, 0) - } - }) - downloadSucceeded := err == nil && result != nil && result.Success - if downloadSucceeded { - resolved := resolvedTrackInfo{ - Title: result.Title, - ArtistName: result.Artist, - AlbumName: result.Album, - ISRC: result.ISRC, - Duration: result.DurationMS / 1000, - SkipNameVerification: strings.EqualFold(strings.TrimSpace(req.Source), strings.TrimSpace(providerLabel)) || - ext.Manifest.HasCustomMatching(), - } - if !trackMatchesRequest(req, resolved, "Extension "+providerLabel) { - discardRejectedExtensionOutput(result, outputPath) - *lastErr = fmt.Errorf("provider %s returned a different track", providerLabel) - *lastErrType = "not_found" - return nil, false - } - } - if req.ItemID != "" && downloadSucceeded { - SetItemFinalizing(req.ItemID) - } - if shouldAbortCancelledFallback(req.ItemID, err) { - return nil, true - } - - if downloadSucceeded { - metadataStartedAt := time.Now() - if !ext.Manifest.SkipMetadataEnrichment { - enrichRequestExtendedMetadata(&req) - } - LogDebug( - "DownloadPipeline", - "item=%s provider=%s post-transfer metadataMs=%.1f", - req.ItemID, - providerLabel, - extensionDurationMs(time.Since(metadataStartedAt)), - ) - - normalizedResult, alreadyExists := normalizeExtensionDownloadResult(result) - message := "Downloaded from " + providerLabel - if alreadyExists { - message = "File already exists" - } - - built := buildDownloadSuccessResponse( - req, - normalizedResult, - providerLabel, - message, - normalizedResult.FilePath, - alreadyExists, - ) - overlayExtensionDownloadMetadata(&built, result) - if ext.Manifest.SkipMetadataEnrichment { - built.SkipMetadataEnrichment = true - } - applyExtensionRequestFallbacks(&built, req) - - if applyTitleFallback { - if req.TrackName != "" && built.Title == "" { - built.Title = req.TrackName - } - if req.ArtistName != "" && built.Artist == "" { - built.Artist = req.ArtistName - } - if req.Composer != "" && built.Composer == "" { - built.Composer = req.Composer - } - } - - if folderErr := finalizeDownloadAlbumFolder(req, &built); folderErr != nil { - return &DownloadResponse{Success: false, Error: folderErr.Error(), ErrorType: "file_error", Service: providerLabel}, false - } - embedExtensionDownloadMetadata(built, req, alreadyExists) - - if !alreadyExists && !isFDOutput(req.OutputFD) && strings.TrimSpace(req.OutputDir) != "" { - indexISRC := strings.TrimSpace(built.ISRC) - if indexISRC == "" { - indexISRC = strings.TrimSpace(req.ISRC) - } - if indexISRC != "" && strings.TrimSpace(built.FilePath) != "" { - AddToISRCIndex(resolvedAlbumOutputDirectory(req, firstNonEmptyTrimmed(req.AlbumName, built.Album)), indexISRC, built.FilePath) - } - } - - if req.ItemID != "" { - CompleteItemProgress(req.ItemID) - } - return &built, false - } - - if err != nil { - if errors.Is(err, ErrDownloadCancelled) { - return &DownloadResponse{ - Success: false, - Error: "Download cancelled", - ErrorType: "cancelled", - Service: providerLabel, - }, false - } - *lastErr = err - *lastErrType = "" - } else if result != nil { - *lastErr = errors.New(firstNonEmptyTrimmed(result.ErrorMessage, "extension download failed without an error message")) - *lastErrType = firstNonEmptyTrimmed(normalizeExtensionDownloadErrorType(result.ErrorType, result.ErrorMessage), "extension_error") - *lastRetryAfterSeconds = result.RetryAfterSeconds - } else { - *lastErr = fmt.Errorf("extension returned no download result") - *lastErrType = "extension_error" - } - return nil, false -} - -// attemptVerifiedResumeBeforeMetadata gives a just-verified request one direct -// chance against its selected provider before optional metadata providers run. -// Download-provider sources keep their normal precedence because they may -// dynamically lock fallback from checkAvailability. -func attemptVerifiedResumeBeforeMetadata( - req DownloadRequest, - selectedProvider string, - extManager *extensionManager, -) (*DownloadResponse, bool) { - selectedProvider = strings.TrimSpace(selectedProvider) - if selectedProvider == "" || extManager == nil { - return nil, false - } - - sourceProvider := strings.TrimSpace(req.Source) - if sourceProvider != "" && !strings.EqualFold(sourceProvider, selectedProvider) { - if sourceExt, err := extManager.GetExtension(sourceProvider); err == nil && - sourceExt != nil && sourceExt.Enabled && sourceExt.Error == "" && - sourceExt.Manifest != nil && sourceExt.Manifest.IsDownloadProvider() { - return nil, false - } - } - - ext, err := extManager.GetExtension(selectedProvider) - if err != nil || ext == nil || !ext.Enabled || ext.Error != "" || - ext.Manifest == nil || !ext.Manifest.IsDownloadProvider() { - return nil, false - } - - provider := newExtensionProviderWrapper(ext) - var availability *ExtAvailabilityResult - trackID := "" - if strings.EqualFold(sourceProvider, selectedProvider) { - trackID = resolvePreferredTrackIDForExtension(ext, req, "") - } else { - availability, err = provider.CheckAvailabilityForItemID( - req.ISRC, - req.TrackName, - req.ArtistName, - req.SpotifyID, - req.DeezerID, - req.TidalID, - req.QobuzID, - req.DurationMS, - req.ItemID, - extensionAvailabilityTrackContext(req), - ) - if shouldAbortCancelledFallback(req.ItemID, err) { - return nil, true - } - if err != nil { - if strings.EqualFold(classifyDownloadErrorType(err.Error()), "verification_required") { - return &DownloadResponse{ - Success: false, - Error: "Download failed: " + err.Error(), - ErrorType: "verification_required", - Service: selectedProvider, - }, false - } - return nil, false - } - if availability == nil || !availability.Available { - if shouldStopProviderFallback(availability) { - return buildExtensionFallbackStoppedResponse(selectedProvider, availability, nil), false - } - return nil, false - } - trackID = resolvePreferredTrackIDForExtension(ext, req, availability.TrackID) - } - - var lastErr error - var lastErrType string - var lastRetryAfterSeconds int - resp, cancelled := attemptExtensionDownload( - req, - ext, - provider, - trackID, - req.Quality, - selectedProvider, - func() map[string]any { - if availability == nil { - return nil - } - return availability.PreparedContext - }(), - strings.EqualFold(sourceProvider, selectedProvider), - &lastErr, - &lastErrType, - &lastRetryAfterSeconds, - ) - if cancelled || resp != nil { - return resp, cancelled - } - - errorType := lastErrType - if errorType == "" && lastErr != nil { - errorType = classifyDownloadErrorType(lastErr.Error()) - } - if strings.EqualFold(errorType, "verification_required") { - errorMessage := "Verification required" - if lastErr != nil { - errorMessage = "Download failed: " + lastErr.Error() - } - return &DownloadResponse{ - Success: false, - Error: errorMessage, - ErrorType: "verification_required", - RetryAfterSeconds: lastRetryAfterSeconds, - Service: selectedProvider, - }, false - } - if lastErr != nil && isOutputStorageWriteFailure(errorType, lastErr.Error()) { - return buildOutputStorageFailureResponse( - selectedProvider, - lastErr, - lastRetryAfterSeconds, - ), false - } - // A failed fast attempt may still succeed after source enrichment resolves a - // better provider-native ID. The normal path below retains strict/stop- - // fallback semantics for that prepared retry. - return nil, false -} - -func extensionAvailabilityTrackContext(req DownloadRequest) map[string]any { - return map[string]any{ - "id": req.ProviderTrackID, - "name": req.TrackName, - "artists": req.ArtistName, - "album_name": req.AlbumName, - "album_artist": req.AlbumArtist, - "cover_url": req.CoverURL, - "release_date": req.ReleaseDate, - "track_number": req.TrackNumber, - "total_tracks": req.TotalTracks, - "disc_number": req.DiscNumber, - "total_discs": req.TotalDiscs, - "duration_ms": req.DurationMS, - "isrc": req.ISRC, - "genre": req.Genre, - "label": req.Label, - "copyright": req.Copyright, - "composer": req.Composer, - "comment": req.Comment, - "explicit": req.Explicit, - "album_type": req.AlbumType, - "upc": req.UPC, - } -} - -func extensionPreparedDownloadContext(req DownloadRequest, prepared map[string]any) map[string]any { - merged := make(map[string]any, len(prepared)+1) - for key, value := range prepared { - merged[key] = value - } - merged["host_track"] = extensionAvailabilityTrackContext(req) - return merged -} - -func buildOutputStorageFailureResponse( - providerID string, - err error, - retryAfterSeconds int, -) *DownloadResponse { - message := "Output storage is not writable" - if err != nil { - message = err.Error() - } - return &DownloadResponse{ - Success: false, - Error: "Download failed: " + message, - ErrorType: "permission", - RetryAfterSeconds: retryAfterSeconds, - Service: providerID, - } -} - -func DownloadWithExtensionFallback(req DownloadRequest) (*DownloadResponse, error) { - pipelineStartedAt := time.Now() - defer func() { - LogDebug( - "DownloadPipeline", - "item=%s service=%s totalMs=%.1f", - req.ItemID, - req.Service, - extensionDurationMs(time.Since(pipelineStartedAt)), - ) - }() - preparationKey := downloadPreparationKey(req) - metadataPrepared := false - resumedAfterVerification := false - if prepared, preparedMetadata, ok := takePreparedDownloadRequest(preparationKey, req); ok { - req = prepared - metadataPrepared = preparedMetadata - resumedAfterVerification = true - GoLog("[DownloadWithExtensionFallback] Resuming item %s after verification (metadata prepared: %v)\n", req.ItemID, metadataPrepared) - } - if req.ItemID != "" { - StartItemProgress(req.ItemID) - SetItemPreparingStage(req.ItemID, "resolving_metadata") - } - - priority := GetProviderPriority() - extManager := getExtensionManager() - strictMode := !req.UseFallback - selectedProvider := strings.TrimSpace(req.Service) - - if isDownloadCancelled(req.ItemID) { - return nil, ErrDownloadCancelled - } - - if strictMode { - if selectedProvider == "" { - selectedProvider = strings.TrimSpace(req.Source) - } - if selectedProvider != "" { - priority = []string{selectedProvider} - GoLog("[DownloadWithExtensionFallback] Strict mode enabled, provider locked to: %s\n", selectedProvider) - } - } - - if !strictMode && req.Service != "" { - found := false - for _, p := range priority { - if strings.EqualFold(p, req.Service) { - found = true - break - } - } - newPriority := []string{req.Service} - for _, p := range priority { - if !strings.EqualFold(p, req.Service) { - newPriority = append(newPriority, p) - } - } - priority = newPriority - if !found { - GoLog("[DownloadWithExtensionFallback] Extension service '%s' added to priority front\n", req.Service) - } else { - GoLog("[DownloadWithExtensionFallback] Extension service '%s' moved to priority front\n", req.Service) - } - GoLog("[DownloadWithExtensionFallback] New priority order: %v\n", priority) - } - - var lastErr error - var lastErrType string - var lastErrorService string - var lastRetryAfterSeconds int - var stopProviderFallback bool - var sourceExtensionLocked bool - var sourceExtensionAvailability *ExtAvailabilityResult - var sourceExtensionTrackID string - - if resumedAfterVerification && !metadataPrepared { - GoLog("[DownloadWithExtensionFallback] Trying verified provider %s before optional metadata enrichment\n", selectedProvider) - resp, cancelled := attemptVerifiedResumeBeforeMetadata(req, selectedProvider, extManager) - if cancelled { - return nil, ErrDownloadCancelled - } - if resp != nil { - return resp, nil - } - if req.ItemID != "" { - SetItemPreparingStage(req.ItemID, "resolving_metadata") - } - } - - if req.Source != "" && selectedProvider != req.Source { - ext, err := extManager.GetExtension(req.Source) - if err == nil && ext.Enabled && ext.Error == "" && ext.Manifest.IsDownloadProvider() { - provider := newExtensionProviderWrapper(ext) - availability, availErr := provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID, extensionAvailabilityTrackContext(req)) - if shouldAbortCancelledFallback(req.ItemID, availErr) { - return nil, ErrDownloadCancelled - } - if availErr != nil { - GoLog("[DownloadWithExtensionFallback] Source extension %s preflight failed (non-fatal): %v\n", req.Source, availErr) - } else if shouldStopProviderFallback(availability) { - sourceExtensionLocked = true - sourceExtensionAvailability = availability - sourceExtensionTrackID = strings.TrimSpace(availability.TrackID) - selectedProvider = req.Source - GoLog("[DownloadWithExtensionFallback] Source extension %s requested skip_fallback (available=%v), locking download to source extension\n", req.Source, availability.Available) - } - } - } - - if !metadataPrepared { - if req.Source != "" { - ext, err := extManager.GetExtension(req.Source) - if err == nil && ext.Enabled && ext.Error == "" && ext.Manifest.IsMetadataProvider() { - GoLog("[DownloadWithExtensionFallback] Enriching track from extension '%s'...\n", req.Source) - - provider := newExtensionProviderWrapper(ext) - trackMeta := buildSourceExtensionTrackMetadata(req) - - enrichedTrack, err := provider.EnrichTrackForItemID(trackMeta, req.ItemID) - if shouldAbortCancelledFallback(req.ItemID, err) { - return nil, ErrDownloadCancelled - } - if err == nil && enrichedTrack != nil { - if enrichedTrack.ISRC != "" && enrichedTrack.ISRC != req.ISRC { - GoLog("[DownloadWithExtensionFallback] ISRC enriched: %s -> %s\n", req.ISRC, enrichedTrack.ISRC) - req.ISRC = enrichedTrack.ISRC - } - if enrichedTrack.TidalID != "" { - GoLog("[DownloadWithExtensionFallback] Tidal ID from Odesli: %s\n", enrichedTrack.TidalID) - req.TidalID = enrichedTrack.TidalID - } - if enrichedTrack.QobuzID != "" { - GoLog("[DownloadWithExtensionFallback] Qobuz ID from Odesli: %s\n", enrichedTrack.QobuzID) - req.QobuzID = enrichedTrack.QobuzID - } - if enrichedTrack.DeezerID != "" { - GoLog("[DownloadWithExtensionFallback] Deezer ID from Odesli: %s\n", enrichedTrack.DeezerID) - req.DeezerID = enrichedTrack.DeezerID - } - overlaySourceExtensionTrackIdentity(&req, *enrichedTrack) - overlayStr(&req.AlbumName, enrichedTrack.AlbumName, "AlbumName") - overlayStr(&req.AlbumArtist, enrichedTrack.AlbumArtist, "") - overlayInt(&req.DurationMS, enrichedTrack.DurationMS, "DurationMS") - overlayStr(&req.CoverURL, enrichedTrack.CoverURL, "") - overlayStr(&req.SpotifyID, enrichedTrack.ID, "Track ID") - overlayStr(&req.Label, enrichedTrack.Label, "Label") - overlayStr(&req.Copyright, enrichedTrack.Copyright, "Copyright") - overlayStr(&req.Genre, enrichedTrack.Genre, "Genre") - overlayStr(&req.ReleaseDate, enrichedTrack.ReleaseDate, "ReleaseDate") - overlayInt(&req.TrackNumber, enrichedTrack.TrackNumber, "TrackNumber") - overlayInt(&req.TotalTracks, enrichedTrack.TotalTracks, "TotalTracks") - overlayInt(&req.DiscNumber, enrichedTrack.DiscNumber, "DiscNumber") - overlayInt(&req.TotalDiscs, enrichedTrack.TotalDiscs, "TotalDiscs") - overlayStr(&req.Composer, enrichedTrack.Composer, "Composer") - overlayExtensionReleaseMetadata(&req, *enrichedTrack) - } - } - } - - if req.Source != "" && - req.TrackName != "" && req.ArtistName != "" && - (req.AlbumName == "" || req.ReleaseDate == "" || req.ISRC == "") { - - searchQuery := req.TrackName + " " + req.ArtistName - GoLog("[DownloadWithExtensionFallback] Metadata incomplete, searching providers for: %s\n", searchQuery) - - // Inspect several candidates: the first search result can be an unrelated - // same-title recording, remix, or cover. - tracks, searchErr := extManager.SearchTracksWithMetadataProvidersForItemID(searchQuery, 5, true, req.ItemID) - if shouldAbortCancelledFallback(req.ItemID, searchErr) { - return nil, ErrDownloadCancelled - } - if searchErr == nil && len(tracks) > 0 { - track := selectBestMetadataEnrichmentTrack(req, tracks) - if track == nil { - GoLog("[DownloadWithExtensionFallback] No confident metadata match; preserving source metadata\n") - } else { - GoLog("[DownloadWithExtensionFallback] Metadata match (%s): %s - %s (album: %s, date: %s, isrc: %s)\n", - track.ProviderID, track.Name, track.Artists, track.AlbumName, track.ReleaseDate, track.ISRC) - - overlayStr(&req.AlbumName, track.AlbumName, "") - overlayStr(&req.AlbumArtist, track.AlbumArtist, "") - overlayStr(&req.ReleaseDate, track.ReleaseDate, "") - overlayStr(&req.ISRC, track.ISRC, "") - overlayInt(&req.TrackNumber, track.TrackNumber, "") - overlayInt(&req.TotalTracks, track.TotalTracks, "") - overlayInt(&req.DiscNumber, track.DiscNumber, "") - overlayInt(&req.TotalDiscs, track.TotalDiscs, "") - overlayStr(&req.Composer, track.Composer, "") - overlayStr(&req.CoverURL, track.CoverURL, "") - overlayStr(&req.Genre, track.Genre, "") - overlayStr(&req.Label, track.Label, "") - overlayStr(&req.Copyright, track.Copyright, "") - overlayExtensionReleaseMetadata(&req, *track) - } - } else if searchErr != nil { - GoLog("[DownloadWithExtensionFallback] Metadata provider search failed (non-fatal): %v\n", searchErr) - } - - } - } - if req.Source != "" && selectedProvider == req.Source { - if isDownloadCancelled(req.ItemID) { - return nil, ErrDownloadCancelled - } - - if sourceExtensionLocked && (sourceExtensionAvailability == nil || !sourceExtensionAvailability.Available) { - GoLog("[DownloadWithExtensionFallback] Source extension %s stopped fallback before download (reason: %s)\n", req.Source, resolveExtensionAvailabilityReason(sourceExtensionAvailability, nil)) - return buildExtensionFallbackStoppedResponse(req.Source, sourceExtensionAvailability, nil), nil - } - - GoLog("[DownloadWithExtensionFallback] Track source is extension '%s' matching selected provider, trying it first\n", req.Source) - - ext, err := extManager.GetExtension(req.Source) - if err == nil && ext.Enabled && ext.Error == "" && ext.Manifest.IsDownloadProvider() { - stopProviderFallback = ext.Manifest.StopsProviderFallback() - - provider := newExtensionProviderWrapper(ext) - - trackID := resolvePreferredTrackIDForExtension(ext, req, sourceExtensionTrackID) - - GoLog("[DownloadWithExtensionFallback] Downloading from source extension with trackID: %s (stopProviderFallback: %v)\n", trackID, stopProviderFallback) - - var preparedContext map[string]any - if sourceExtensionAvailability != nil { - preparedContext = sourceExtensionAvailability.PreparedContext - } - resp, cancelledOuter := attemptExtensionDownload(req, ext, provider, trackID, req.Quality, req.Source, preparedContext, true, &lastErr, &lastErrType, &lastRetryAfterSeconds) - if cancelledOuter { - return nil, ErrDownloadCancelled - } - if resp != nil { - return resp, nil - } - lastErrorService = req.Source - GoLog("[DownloadWithExtensionFallback] Source extension %s failed: %v\n", req.Source, lastErr) - - sourceErrType := lastErrType - if sourceErrType == "" && lastErr != nil { - sourceErrType = classifyDownloadErrorType(lastErr.Error()) - } - if lastErr != nil && isOutputStorageWriteFailure(sourceErrType, lastErr.Error()) { - GoLog("[DownloadWithExtensionFallback] Source extension %s hit an unwritable output path; stopping provider fallback\n", req.Source) - return buildOutputStorageFailureResponse( - req.Source, - lastErr, - lastRetryAfterSeconds, - ), nil - } - if strings.EqualFold(sourceErrType, "verification_required") { - GoLog("[DownloadWithExtensionFallback] Source extension %s requires verification, not trying other providers\n", req.Source) - cachePreparedDownloadRequest(preparationKey, req) - return &DownloadResponse{ - Success: false, - Error: "Download failed: " + lastErr.Error(), - ErrorType: "verification_required", - Service: req.Source, - }, nil - } - - if stopProviderFallback || sourceExtensionLocked { - if sourceExtensionLocked { - GoLog("[DownloadWithExtensionFallback] Source extension %s requested skip_fallback, not trying other providers\n", req.Source) - return buildExtensionFallbackStoppedResponse(req.Source, sourceExtensionAvailability, lastErr), nil - } - GoLog("[DownloadWithExtensionFallback] stopProviderFallback is true, not trying other providers\n") - return &DownloadResponse{ - Success: false, - Error: "Download failed: " + lastErr.Error(), - ErrorType: firstNonEmptyTrimmed(lastErrType, "extension_error"), - RetryAfterSeconds: lastRetryAfterSeconds, - Service: req.Source, - }, nil - } - } else { - GoLog("[DownloadWithExtensionFallback] Source extension %s not available or not a download provider\n", req.Source) - } - } - - healthProtectedProvider := selectedProvider - if strings.TrimSpace(healthProtectedProvider) == "" { - healthProtectedProvider = req.Source - } - priority = prioritizeFallbackProvidersByHealth(priority, extManager, healthProtectedProvider) - priority = moveProviderToFront(priority, selectedProvider) - - for _, providerID := range priority { - if isDownloadCancelled(req.ItemID) { - return nil, ErrDownloadCancelled - } - - providerID = strings.TrimSpace(providerID) - if providerID == "" { - continue - } - // Skip the origin extension only when it differs from the explicitly - // selected provider; otherwise it must still be attempted here. - if providerID == req.Source && req.Source != selectedProvider { - continue - } - - if providerID != selectedProvider && !isExtensionFallbackAllowed(providerID) { - GoLog("[DownloadWithExtensionFallback] Skipping extension provider %s (not enabled for fallback)\n", providerID) - continue - } - - GoLog("[DownloadWithExtensionFallback] Trying provider: %s\n", providerID) - - { - ext, err := extManager.GetExtension(providerID) - if err != nil || !ext.Enabled || ext.Error != "" { - GoLog("[DownloadWithExtensionFallback] Extension %s not available\n", providerID) - continue - } - - if !ext.Manifest.IsDownloadProvider() { - continue - } - - provider := newExtensionProviderWrapper(ext) - - // Fallback providers need the same session preparation as the selected - // provider. A cached availability result may never call signedFetch, - // leaving an expired session or pending challenge invisible to the app. - var availability *ExtAvailabilityResult - verificationRequired, err := preflightExtensionDownloadSession(providerID) - if err != nil { - err = fmt.Errorf("signed-session preflight failed: %w", err) - } else if verificationRequired { - err = fmt.Errorf("verification_required: extension '%s' needs signed-session verification", providerID) - } else { - availability, err = provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID, extensionAvailabilityTrackContext(req)) - } - if shouldAbortCancelledFallback(req.ItemID, err) { - return nil, ErrDownloadCancelled - } - terminalAvailability := shouldStopProviderFallback(availability) - if err != nil || !availability.Available { - if err != nil { - lastErr = err - lastErrType = classifyDownloadErrorType(err.Error()) - lastErrorService = providerID - lastRetryAfterSeconds = 0 - GoLog("[DownloadWithExtensionFallback] %s availability failed: %v\n", providerID, err) - if strings.EqualFold(lastErrType, "verification_required") { - GoLog("[DownloadWithExtensionFallback] %s requires verification (availability); pausing fallback to open the challenge\n", providerID) - cachePreparedDownloadRequest(preparationKey, req) - return &DownloadResponse{ - Success: false, - Error: "Download failed: " + err.Error(), - ErrorType: "verification_required", - Service: providerID, - }, nil - } - } else { - GoLog("[DownloadWithExtensionFallback] %s: not available (reason: %s)\n", providerID, resolveExtensionAvailabilityReason(availability, nil)) - } - if terminalAvailability { - GoLog("[DownloadWithExtensionFallback] %s requested skip_fallback after availability check\n", providerID) - return buildExtensionFallbackStoppedResponse(providerID, availability, err), nil - } - continue - } - - req.OutputExt = "" - - resp, cancelledOuter := attemptExtensionDownload(req, ext, provider, availability.TrackID, req.Quality, providerID, availability.PreparedContext, false, &lastErr, &lastErrType, &lastRetryAfterSeconds) - if cancelledOuter { - return nil, ErrDownloadCancelled - } - if resp != nil { - return resp, nil - } - lastErrorService = providerID - GoLog("[DownloadWithExtensionFallback] %s failed: %v\n", providerID, lastErr) - - if lastErr != nil { - effType := lastErrType - if effType == "" { - effType = classifyDownloadErrorType(lastErr.Error()) - } - if isOutputStorageWriteFailure(effType, lastErr.Error()) { - GoLog("[DownloadWithExtensionFallback] %s hit an unwritable output path; stopping provider fallback\n", providerID) - return buildOutputStorageFailureResponse( - providerID, - lastErr, - lastRetryAfterSeconds, - ), nil - } - if strings.EqualFold(effType, "verification_required") { - GoLog("[DownloadWithExtensionFallback] %s requires verification; pausing fallback to open the challenge\n", providerID) - cachePreparedDownloadRequest(preparationKey, req) - return &DownloadResponse{ - Success: false, - Error: "Download failed: " + lastErr.Error(), - ErrorType: "verification_required", - RetryAfterSeconds: lastRetryAfterSeconds, - Service: providerID, - }, nil - } - } - - if terminalAvailability { - GoLog("[DownloadWithExtensionFallback] %s requested skip_fallback after download failure\n", providerID) - return buildExtensionFallbackStoppedResponse(providerID, availability, lastErr), nil - } - } - } - - if lastErr != nil { - errorType := firstNonEmptyTrimmed(lastErrType, classifyDownloadErrorType(lastErr.Error())) - if errorType == "unknown" { - errorType = "not_found" - } - return &DownloadResponse{ - Success: false, - Error: "All providers failed. Last error: " + lastErr.Error(), - ErrorType: errorType, - RetryAfterSeconds: lastRetryAfterSeconds, - Service: lastErrorService, - }, nil - } - - return &DownloadResponse{ - Success: false, - Error: "No extension download providers available", - ErrorType: "not_found", - }, nil -} diff --git a/go_backend/extension_fallback_helpers.go b/go_backend/extension_fallback_helpers.go deleted file mode 100644 index fa8d4798..00000000 --- a/go_backend/extension_fallback_helpers.go +++ /dev/null @@ -1,521 +0,0 @@ -package gobackend - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" -) - -func manifestCapabilityStringList(manifest *ExtensionManifest, key string) []string { - if manifest == nil || manifest.Capabilities == nil { - return nil - } - - raw, ok := manifest.Capabilities[key] - if !ok { - return nil - } - - values, ok := raw.([]any) - if !ok { - return nil - } - - result := make([]string, 0, len(values)) - for _, value := range values { - str, ok := value.(string) - if !ok { - continue - } - trimmed := strings.ToLower(strings.TrimSpace(str)) - if trimmed == "" { - continue - } - result = append(result, trimmed) - } - return result -} - -func extensionReplacesBuiltInProvider(ext *loadedExtension, providerID string) bool { - if ext == nil { - return false - } - - normalized := strings.ToLower(strings.TrimSpace(providerID)) - if normalized == "" { - return false - } - - for _, replaced := range manifestCapabilityStringList(ext.Manifest, "replacesBuiltInProviders") { - if replaced == normalized { - return true - } - } - - return false -} - -func trimKnownProviderPrefix(trackID, providerID string) string { - trimmedID := strings.TrimSpace(trackID) - normalizedProvider := strings.ToLower(strings.TrimSpace(providerID)) - if trimmedID == "" || normalizedProvider == "" { - return trimmedID - } - - prefix := normalizedProvider + ":" - if strings.HasPrefix(strings.ToLower(trimmedID), prefix) { - return trimmedID[len(prefix):] - } - - return trimmedID -} - -func buildSourceExtensionTrackMetadata(req DownloadRequest) *ExtTrackMetadata { - return &ExtTrackMetadata{ - ID: req.SpotifyID, - Name: req.TrackName, - Artists: req.ArtistName, - AlbumName: req.AlbumName, - AlbumArtist: req.AlbumArtist, - DurationMS: req.DurationMS, - CoverURL: req.CoverURL, - ISRC: req.ISRC, - ReleaseDate: req.ReleaseDate, - TrackNumber: req.TrackNumber, - TotalTracks: req.TotalTracks, - DiscNumber: req.DiscNumber, - TotalDiscs: req.TotalDiscs, - ProviderID: req.Source, - AlbumType: req.AlbumType, - Explicit: req.Explicit, - UPC: req.UPC, - TidalID: req.TidalID, - QobuzID: req.QobuzID, - DeezerID: req.DeezerID, - SpotifyID: req.SpotifyID, - Label: req.Label, - Copyright: req.Copyright, - Genre: req.Genre, - Composer: req.Composer, - Comment: req.Comment, - } -} - -func overlaySourceExtensionTrackIdentity(req *DownloadRequest, enrichedTrack ExtTrackMetadata) { - if req == nil { - return - } - - // The queued track describes the release selected by the user. Enrichment - // may fill missing identity fields, but must not rename it to another - // release's display title or artist spelling. - overlayStr(&req.TrackName, enrichedTrack.Name, "TrackName") - overlayStr(&req.ArtistName, enrichedTrack.Artists, "ArtistName") -} - -func resolvePreferredTrackIDForExtension(ext *loadedExtension, req DownloadRequest, explicitTrackID string) string { - candidates := make([]string, 0, 8) - appendCandidate := func(value string) { - trimmed := strings.TrimSpace(value) - if trimmed == "" { - return - } - for _, existing := range candidates { - if existing == trimmed { - return - } - } - candidates = append(candidates, trimmed) - } - - appendCandidate(explicitTrackID) - - if extensionReplacesBuiltInProvider(ext, "tidal") { - appendCandidate(req.TidalID) - appendCandidate(trimKnownProviderPrefix(req.SpotifyID, "tidal")) - } - if extensionReplacesBuiltInProvider(ext, "qobuz") { - appendCandidate(req.QobuzID) - appendCandidate(trimKnownProviderPrefix(req.SpotifyID, "qobuz")) - } - if extensionReplacesBuiltInProvider(ext, "deezer") { - appendCandidate(req.DeezerID) - appendCandidate(trimKnownProviderPrefix(req.SpotifyID, "deezer")) - } - if extensionReplacesBuiltInProvider(ext, "spotify") { - appendCandidate(trimKnownProviderPrefix(req.SpotifyID, "spotify")) - appendCandidate(req.SpotifyID) - } - - appendCandidate(req.SpotifyID) - appendCandidate(req.TidalID) - appendCandidate(req.QobuzID) - appendCandidate(req.DeezerID) - - if len(candidates) == 0 { - return "" - } - return candidates[0] -} - -func normalizeDownloadResultExtension(candidates ...string) string { - for _, candidate := range candidates { - ext := strings.TrimSpace(strings.ToLower(candidate)) - if ext == "" { - continue - } - if !strings.HasPrefix(ext, ".") { - ext = "." + ext - } - if ext == ".mp4" { - return ".m4a" - } - return ext - } - return "" -} - -// discardRejectedExtensionOutput removes only a newly downloaded file inside -// the host-selected output directory. Existing-library hits are never removed, -// nor are paths outside that narrow directory. -func discardRejectedExtensionOutput(result *ExtDownloadResult, requestedOutputPath string) { - if result == nil || result.AlreadyExists { - return - } - - resultPath := strings.TrimSpace(result.FilePath) - requestedPath := strings.TrimSpace(requestedOutputPath) - if resultPath == "" || requestedPath == "" || - strings.HasPrefix(resultPath, "content://") || - strings.HasPrefix(resultPath, "/proc/self/fd/") { - return - } - - resultAbs, resultErr := filepath.Abs(resultPath) - outputDirAbs, outputErr := filepath.Abs(filepath.Dir(requestedPath)) - if resultErr != nil || outputErr != nil { - return - } - relative, err := filepath.Rel(outputDirAbs, resultAbs) - if err != nil || relative == "." || relative == ".." || - strings.HasPrefix(relative, ".."+string(filepath.Separator)) || - filepath.IsAbs(relative) { - return - } - - if err := os.Remove(resultAbs); err != nil && !os.IsNotExist(err) { - GoLog("[DownloadWithExtensionFallback] Warning: failed to remove rejected provider output %q: %v\n", resultAbs, err) - } -} - -func normalizeExtensionDownloadResult(result *ExtDownloadResult) (DownloadResult, bool) { - if result == nil { - return DownloadResult{}, false - } - - downloadResult := DownloadResult{ - FilePath: strings.TrimSpace(result.FilePath), - BitDepth: result.BitDepth, - SampleRate: result.SampleRate, - AudioCodec: strings.TrimSpace(result.AudioCodec), - Title: result.Title, - Artist: result.Artist, - Album: result.Album, - ReleaseDate: result.ReleaseDate, - TrackNumber: result.TrackNumber, - TotalTracks: result.TotalTracks, - DiscNumber: result.DiscNumber, - TotalDiscs: result.TotalDiscs, - ISRC: result.ISRC, - CoverURL: result.CoverURL, - Genre: result.Genre, - Label: result.Label, - Copyright: result.Copyright, - Composer: result.Composer, - Comment: result.Comment, - Explicit: result.Explicit, - AlbumType: result.AlbumType, - UPC: result.UPC, - LyricsLRC: result.LyricsLRC, - DecryptionKey: result.DecryptionKey, - Decryption: normalizeDownloadDecryptionInfo(result.Decryption, result.DecryptionKey), - ActualExtension: normalizeDownloadResultExtension(result.ActualExtension, result.OutputExtension), - ActualContainer: strings.TrimSpace(result.ActualContainer), - RequiresContainerConversion: result.RequiresContainerConversion, - } - - alreadyExists := result.AlreadyExists - if strings.HasPrefix(downloadResult.FilePath, "EXISTS:") { - alreadyExists = true - downloadResult.FilePath = strings.TrimPrefix(downloadResult.FilePath, "EXISTS:") - } - - enrichResultQualityFromFile(&downloadResult) - return downloadResult, alreadyExists -} - -// overlayStr sets *dst = src when dst is empty and src is not. If field is -// non-empty it logs " from enrichment: " on overlay. -func overlayStr(dst *string, src, field string) { - if src == "" || *dst != "" { - return - } - *dst = src - if field != "" { - GoLog("[DownloadWithExtensionFallback] %s from enrichment: %s\n", field, src) - } -} - -// overlayExtensionReleaseMetadata carries release-level metadata discovered by -// the source catalog into a provider-agnostic download request. Existing -// source values always win; enrichment only fills fields that were missing. -func overlayExtensionReleaseMetadata(req *DownloadRequest, track ExtTrackMetadata) { - if req == nil { - return - } - overlayStr(&req.AlbumType, track.AlbumType, "AlbumType") - overlayStr(&req.UPC, track.UPC, "UPC") - overlayStr(&req.Comment, track.Comment, "Comment") - if !req.Explicit && track.Explicit { - req.Explicit = true - GoLog("[DownloadWithExtensionFallback] Explicit flag from enrichment\n") - } -} - -// overlayStrTrim is overlayStr but treats a whitespace-only dst as empty too. -func overlayStrTrim(dst *string, src string) { - if src == "" || strings.TrimSpace(*dst) != "" { - return - } - *dst = src -} - -// overlayInt sets *dst = src when dst is zero and src is positive. If field is -// non-empty it logs " from enrichment: " on overlay. -func overlayInt(dst *int, src int, field string) { - if src <= 0 || *dst != 0 { - return - } - *dst = src - if field != "" { - GoLog("[DownloadWithExtensionFallback] %s from enrichment: %d\n", field, src) - } -} - -func overlayExtensionDownloadMetadata(resp *DownloadResponse, result *ExtDownloadResult) { - if resp == nil || result == nil { - return - } - - overlayStrTrim(&resp.Title, result.Title) - overlayStrTrim(&resp.Artist, result.Artist) - overlayStrTrim(&resp.Album, result.Album) - overlayStrTrim(&resp.AlbumArtist, result.AlbumArtist) - overlayInt(&resp.TrackNumber, result.TrackNumber, "") - overlayInt(&resp.DiscNumber, result.DiscNumber, "") - overlayInt(&resp.TotalTracks, result.TotalTracks, "") - overlayInt(&resp.TotalDiscs, result.TotalDiscs, "") - overlayStrTrim(&resp.ReleaseDate, result.ReleaseDate) - overlayStrTrim(&resp.CoverURL, result.CoverURL) - overlayStrTrim(&resp.ISRC, result.ISRC) - overlayStrTrim(&resp.Genre, result.Genre) - overlayStrTrim(&resp.Label, result.Label) - overlayStrTrim(&resp.Copyright, result.Copyright) - overlayStrTrim(&resp.Composer, result.Composer) - overlayStrTrim(&resp.Comment, result.Comment) - overlayStrTrim(&resp.AlbumType, result.AlbumType) - overlayStrTrim(&resp.UPC, result.UPC) - if result.Explicit { - resp.Explicit = true - } - if result.LyricsLRC != "" { - resp.LyricsLRC = result.LyricsLRC - } - if result.DecryptionKey != "" { - resp.DecryptionKey = result.DecryptionKey - } - if normalized := normalizeDownloadDecryptionInfo(result.Decryption, result.DecryptionKey); normalized != nil { - resp.Decryption = normalized - } - if ext := normalizeDownloadResultExtension(result.ActualExtension, result.OutputExtension); ext != "" { - resp.ActualExtension = ext - } - if container := strings.TrimSpace(result.ActualContainer); container != "" { - resp.ActualContainer = container - } - if result.RequiresContainerConversion { - resp.RequiresContainerConversion = true - } -} - -func applyExtensionRequestFallbacks(resp *DownloadResponse, req DownloadRequest) { - if resp == nil { - return - } - - overlayStr(&resp.Album, req.AlbumName, "") - overlayStr(&resp.AlbumArtist, req.AlbumArtist, "") - overlayStr(&resp.ReleaseDate, req.ReleaseDate, "") - overlayStr(&resp.ISRC, req.ISRC, "") - overlayInt(&resp.TrackNumber, req.TrackNumber, "") - overlayInt(&resp.TotalTracks, req.TotalTracks, "") - overlayInt(&resp.DiscNumber, req.DiscNumber, "") - overlayInt(&resp.TotalDiscs, req.TotalDiscs, "") - overlayStr(&resp.CoverURL, req.CoverURL, "") - overlayStr(&resp.Comment, req.Comment, "") - overlayStr(&resp.AlbumType, req.AlbumType, "") - overlayStr(&resp.UPC, req.UPC, "") - if req.Explicit { - resp.Explicit = true - } -} - -func shouldStopProviderFallback(availability *ExtAvailabilityResult) bool { - return availability != nil && availability.SkipFallback -} - -func fallbackRuntimeHealthStatus(ext *loadedExtension) string { - if ext == nil || ext.Manifest == nil || len(ext.Manifest.ServiceHealth) == 0 { - return "unknown" - } - - health, cached := PeekExtensionHealthCached(ext) - if !cached { - staleHealth, hasStale := peekExtensionHealthStale(ext) - RefreshExtensionHealthAsync(ext) - if !hasStale { - return "unknown" - } - // An expired offline verdict must not keep suppressing a recovered - // provider while its refresh runs in the background. - if strings.EqualFold(staleHealth.Status, "offline") { - return "unknown" - } - health = staleHealth - } - status := strings.ToLower(strings.TrimSpace(health.Status)) - switch status { - case "online", "degraded", "offline": - return status - default: - return "unknown" - } -} - -func prioritizeFallbackProvidersByHealth(priority []string, extManager *extensionManager, protectedProvider string) []string { - if len(priority) == 0 || extManager == nil { - return priority - } - - online := make([]string, 0, len(priority)) - degraded := make([]string, 0, len(priority)) - unknown := make([]string, 0, len(priority)) - - for _, rawProviderID := range priority { - providerID := strings.TrimSpace(rawProviderID) - if providerID == "" { - continue - } - if strings.EqualFold(providerID, protectedProvider) || !isExtensionFallbackAllowed(providerID) { - unknown = append(unknown, providerID) - continue - } - - ext, err := extManager.GetExtension(providerID) - if err != nil || ext == nil || !ext.Enabled || ext.Error != "" || ext.Manifest == nil || !ext.Manifest.IsDownloadProvider() { - unknown = append(unknown, providerID) - continue - } - - switch fallbackRuntimeHealthStatus(ext) { - case "online": - online = append(online, providerID) - case "degraded": - degraded = append(degraded, providerID) - case "offline": - GoLog("[DownloadWithExtensionFallback] Skipping extension provider %s (service health offline)\n", providerID) - default: - unknown = append(unknown, providerID) - } - } - - result := make([]string, 0, len(online)+len(degraded)+len(unknown)) - result = append(result, online...) - result = append(result, degraded...) - result = append(result, unknown...) - return result -} - -// moveProviderToFront preserves the user's explicit provider selection after -// health-based fallback sorting. Health may order the remaining fallback -// candidates, but it must never silently replace the provider the user picked. -func moveProviderToFront(priority []string, providerID string) []string { - providerID = strings.TrimSpace(providerID) - if providerID == "" || len(priority) < 2 { - return priority - } - - selectedIndex := -1 - for i, candidate := range priority { - if strings.EqualFold(strings.TrimSpace(candidate), providerID) { - selectedIndex = i - break - } - } - if selectedIndex <= 0 { - return priority - } - - reordered := make([]string, 0, len(priority)) - reordered = append(reordered, priority[selectedIndex]) - reordered = append(reordered, priority[:selectedIndex]...) - reordered = append(reordered, priority[selectedIndex+1:]...) - return reordered -} - -func resolveExtensionAvailabilityReason(availability *ExtAvailabilityResult, err error) string { - if availability != nil { - if reason := strings.TrimSpace(availability.Reason); reason != "" { - return reason - } - } - if err != nil { - return err.Error() - } - return "extension requested no further fallback" -} - -func buildExtensionFallbackStoppedResponse(providerID string, availability *ExtAvailabilityResult, err error) *DownloadResponse { - reason := resolveExtensionAvailabilityReason(availability, err) - errorType := classifyDownloadErrorType(reason) - if errorType == "unknown" { - errorType = "extension_error" - } - return &DownloadResponse{ - Success: false, - Error: fmt.Sprintf("Fallback stopped by %s: %s", providerID, reason), - ErrorType: errorType, - Service: providerID, - } -} - -func shouldAbortCancelledFallback(itemID string, err error) bool { - if errors.Is(err, ErrDownloadCancelled) { - return true - } - return itemID != "" && isDownloadCancelled(itemID) -} - -func normalizeExtensionDownloadErrorType(errorType, message string) string { - normalized := strings.TrimSpace(errorType) - classified := classifyDownloadErrorType(message) - if classified != "" && classified != "unknown" { - switch strings.ToLower(normalized) { - case "", "unknown", "runtime_error", "api_error", "download_error", "extension_error": - return classified - } - } - return normalized -} diff --git a/go_backend/extension_fallback_metadata_test.go b/go_backend/extension_fallback_metadata_test.go deleted file mode 100644 index 605eed4d..00000000 --- a/go_backend/extension_fallback_metadata_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package gobackend - -import "testing" - -func TestOverlayExtensionReleaseMetadataFillsMissingRequestFields(t *testing.T) { - req := DownloadRequest{AlbumType: "single"} - track := ExtTrackMetadata{ - AlbumType: "album", - Explicit: true, - UPC: "4006381333931", - Comment: "https://source.example/album/1532211596", - } - - overlayExtensionReleaseMetadata(&req, track) - - if req.AlbumType != "single" { - t.Fatalf("existing album type was overwritten: %q", req.AlbumType) - } - if !req.Explicit { - t.Fatal("explicit flag from source enrichment was not retained") - } - if req.UPC != track.UPC { - t.Fatalf("UPC = %q, want %q", req.UPC, track.UPC) - } - if req.Comment != track.Comment { - t.Fatalf("comment = %q, want %q", req.Comment, track.Comment) - } - - response := buildDownloadSuccessResponse( - req, - DownloadResult{}, - "download-provider", - "downloaded", - "song.m4a", - false, - ) - if response.UPC != track.UPC || response.AlbumType != "single" || - !response.Explicit || response.Comment != track.Comment { - t.Fatalf("enriched metadata was lost in download response: %#v", response) - } -} - -func TestOverlayExtensionReleaseMetadataDoesNotEraseExistingValues(t *testing.T) { - req := DownloadRequest{ - AlbumType: "ep", - Explicit: true, - UPC: "existing-upc", - Comment: "existing-comment", - } - - overlayExtensionReleaseMetadata(&req, ExtTrackMetadata{}) - - if req.AlbumType != "ep" || !req.Explicit || - req.UPC != "existing-upc" || req.Comment != "existing-comment" { - t.Fatalf("existing release metadata changed: %#v", req) - } -} - -func TestBuildSourceExtensionTrackMetadataCarriesKnownIdentifiers(t *testing.T) { - req := DownloadRequest{ - SpotifyID: "source-id", - TidalID: "alternate-id-a", - QobuzID: "alternate-id-b", - DeezerID: "alternate-id-c", - } - - track := buildSourceExtensionTrackMetadata(req) - if track.ID != req.SpotifyID || track.SpotifyID != req.SpotifyID { - t.Fatalf("primary identifier was not propagated: %#v", track) - } - if track.TidalID != req.TidalID || track.QobuzID != req.QobuzID || track.DeezerID != req.DeezerID { - t.Fatalf("alternate identifiers were not propagated: %#v", track) - } -} - -func TestOverlaySourceExtensionTrackIdentityPreservesRequestedValues(t *testing.T) { - req := DownloadRequest{ - TrackName: "Album Display Title", - ArtistName: "Album Artist Credit", - } - - overlaySourceExtensionTrackIdentity(&req, ExtTrackMetadata{ - Name: "Single Display Title", - Artists: "Provider Artist Credit", - }) - - if req.TrackName != "Album Display Title" { - t.Fatalf("track name = %q, want requested title", req.TrackName) - } - if req.ArtistName != "Album Artist Credit" { - t.Fatalf("artist name = %q, want requested credit", req.ArtistName) - } -} - -func TestOverlaySourceExtensionTrackIdentityFillsMissingValues(t *testing.T) { - req := DownloadRequest{} - overlaySourceExtensionTrackIdentity(&req, ExtTrackMetadata{ - Name: "Resolved Title", - Artists: "Resolved Artist", - }) - - if req.TrackName != "Resolved Title" || req.ArtistName != "Resolved Artist" { - t.Fatalf("missing identity fields were not enriched: %#v", req) - } -} - -func TestExtensionPreparedDownloadContextMergesHostMetadata(t *testing.T) { - providerContext := map[string]any{ - "token": "opaque-provider-value", - "host_track": "provider-key-must-not-win", - } - req := DownloadRequest{ - ProviderTrackID: "provider-track-1", - TrackName: "Song", - ArtistName: "Artist", - AlbumName: "Album", - ISRC: "ISRC123", - } - - merged := extensionPreparedDownloadContext(req, providerContext) - if merged["token"] != "opaque-provider-value" { - t.Fatalf("provider context was not preserved: %#v", merged) - } - hostTrack, ok := merged["host_track"].(map[string]any) - if !ok { - t.Fatalf("host_track = %#v", merged["host_track"]) - } - if hostTrack["id"] != req.ProviderTrackID || hostTrack["name"] != req.TrackName || hostTrack["isrc"] != req.ISRC { - t.Fatalf("host metadata was not propagated: %#v", hostTrack) - } - if providerContext["host_track"] != "provider-key-must-not-win" { - t.Fatalf("input provider context was mutated: %#v", providerContext) - } -} diff --git a/go_backend/extension_fallback_output.go b/go_backend/extension_fallback_output.go deleted file mode 100644 index ae2b2643..00000000 --- a/go_backend/extension_fallback_output.go +++ /dev/null @@ -1,210 +0,0 @@ -package gobackend - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - -// buildDownloadFilename renders the sanitized "" filename for req -// from its template/metadata, defaulting to " - .flac". -func buildDownloadFilename(req DownloadRequest) string { - metadata := map[string]any{ - "title": req.TrackName, - "artist": req.ArtistName, - "album": req.AlbumName, - "album_artist": req.AlbumArtist, - "track": req.TrackNumber, - "track_number": req.TrackNumber, - "total_tracks": req.TotalTracks, - "playlist_position": req.PlaylistPosition, - "disc": req.DiscNumber, - "disc_number": req.DiscNumber, - "total_discs": req.TotalDiscs, - "year": extractYear(req.ReleaseDate), - "date": req.ReleaseDate, - "release_date": req.ReleaseDate, - "isrc": req.ISRC, - "provider": req.DownloadProvider, - "provider_id": req.ProviderTrackID, - "composer": req.Composer, - "quality": req.Quality, - "quality_variant": req.QualityVariant, - } - - filename := buildFilenameFromTemplate(req.FilenameFormat, metadata) - if strings.TrimSpace(filename) == "" { - filename = fmt.Sprintf("%s - %s", req.ArtistName, req.TrackName) - } - filename = sanitizeFilenamePreservingToken(filename, req.QualityVariant) - - ext := strings.TrimSpace(req.OutputExt) - if ext == "" { - ext = ".flac" - } else if !strings.HasPrefix(ext, ".") { - ext = "." + ext - } - - return filename + ext -} - -func resolvedDownloadFilename(req DownloadRequest, result DownloadResult, filePath string) string { - resolved := req - if isrc := strings.TrimSpace(result.ISRC); isrc != "" { - resolved.ISRC = isrc - } - extension := strings.TrimSpace(result.ActualExtension) - if extension == "" { - extension = filepath.Ext(strings.TrimSpace(result.FilePath)) - } - if extension == "" { - extension = filepath.Ext(strings.TrimSpace(filePath)) - } - if extension != "" { - resolved.OutputExt = extension - } - return buildDownloadFilename(resolved) -} - -func buildOutputPath(req DownloadRequest) string { - if strings.TrimSpace(req.OutputPath) != "" { - return strings.TrimSpace(req.OutputPath) - } - - outputDir := resolvedAlbumOutputDirectory(req, req.AlbumName) - if strings.TrimSpace(outputDir) == "" { - outputDir = filepath.Join(os.TempDir(), "spotiflac-downloads") - } - os.MkdirAll(outputDir, 0755) - AddAllowedDownloadDir(outputDir) - - return filepath.Join(outputDir, buildDownloadFilename(req)) -} - -func buildOutputPathForExtension(req DownloadRequest, ext *loadedExtension) string { - if strings.TrimSpace(req.OutputPath) != "" { - outputPath := strings.TrimSpace(req.OutputPath) - AddAllowedDownloadDir(filepath.Dir(outputPath)) - return outputPath - } - - // SAF downloads hand extensions a detached output FD owned by the host. - // Extensions still need a real local temp file so Android can copy it into - // the target document after provider-specific post-processing completes. - if !isFDOutput(req.OutputFD) && strings.TrimSpace(req.OutputDir) != "" { - return buildOutputPath(req) - } - - tempDir := filepath.Join(ext.DataDir, "downloads") - os.MkdirAll(tempDir, 0755) - AddAllowedDownloadDir(tempDir) - - return filepath.Join(tempDir, buildDownloadFilename(req)) -} - -func shouldReuseExistingOutput(req DownloadRequest, outputPath string) bool { - if req.AlbumFolderTemplate != "" && resolvedAlbumFolder(req, req.AlbumName) == "" { - return false - } - if req.AllowQualityVariant || isFDOutput(req.OutputFD) { - return false - } - path := strings.TrimSpace(outputPath) - if path == "" || strings.HasPrefix(path, "content://") || strings.HasPrefix(path, "/proc/self/fd/") { - return false - } - info, err := os.Stat(path) - return err == nil && info.Mode().IsRegular() && info.Size() > 0 -} - -func canEmbedGenreLabel(filePath string) bool { - path := strings.TrimSpace(filePath) - if path == "" || strings.HasPrefix(path, "content://") || strings.HasPrefix(path, "/proc/self/fd/") { - return false - } - if strings.ToLower(filepath.Ext(path)) != ".flac" { - return false - } - if !filepath.IsAbs(path) { - return false - } - info, err := os.Stat(path) - return err == nil && !info.IsDir() && info.Size() > 0 -} - -func embedExtensionDownloadMetadata(resp DownloadResponse, req DownloadRequest, alreadyExists bool) { - if alreadyExists || !req.EmbedMetadata { - return - } - - filePath := strings.TrimSpace(resp.FilePath) - if !canEmbedGenreLabel(filePath) { - if req.Genre != "" || req.Label != "" || resp.CoverURL != "" || req.CoverURL != "" { - GoLog("[DownloadWithExtensionFallback] Skipping metadata/cover embed for non-local FLAC output path: %q\n", filePath) - } - return - } - - coverURL := firstNonEmptyTrimmed(resp.CoverURL, req.CoverURL) - var coverData []byte - if coverURL != "" { - data, err := downloadCoverToMemorySized(coverURL, req.CoverMaxDimension) - if err != nil { - GoLog("[DownloadWithExtensionFallback] Warning: failed to download cover for metadata embed: %v\n", err) - } else if len(data) > 0 { - coverData = data - } - } - - metadata := Metadata{ - Title: firstNonEmptyTrimmed(resp.Title, req.TrackName), - Artist: firstNonEmptyTrimmed(resp.Artist, req.ArtistName), - Album: firstNonEmptyTrimmed(resp.Album, req.AlbumName), - AlbumArtist: firstNonEmptyTrimmed(resp.AlbumArtist, req.AlbumArtist), - ArtistTagMode: req.ArtistTagMode, - Date: firstNonEmptyTrimmed(resp.ReleaseDate, req.ReleaseDate), - TrackNumber: firstPositiveInt(resp.TrackNumber, req.TrackNumber), - TotalTracks: firstPositiveInt(resp.TotalTracks, req.TotalTracks), - DiscNumber: firstPositiveInt(resp.DiscNumber, req.DiscNumber), - TotalDiscs: firstPositiveInt(resp.TotalDiscs, req.TotalDiscs), - ISRC: firstNonEmptyTrimmed(resp.ISRC, req.ISRC), - Genre: firstNonEmptyTrimmed(resp.Genre, req.Genre), - Label: firstNonEmptyTrimmed(resp.Label, req.Label), - Copyright: firstNonEmptyTrimmed(resp.Copyright, req.Copyright), - Composer: firstNonEmptyTrimmed(resp.Composer, req.Composer), - Comment: firstNonEmptyTrimmed(resp.Comment, req.Comment), - Explicit: resp.Explicit || req.Explicit, - AlbumType: firstNonEmptyTrimmed(resp.AlbumType, req.AlbumType), - UPC: firstNonEmptyTrimmed(resp.UPC, req.UPC), - } - if req.EmbedLyrics { - metadata.Lyrics = resp.LyricsLRC - } - - var err error - if len(coverData) > 0 { - err = EmbedMetadataWithCoverData(filePath, metadata, coverData) - } else { - err = EmbedMetadata(filePath, metadata, "") - } - if err != nil { - GoLog("[DownloadWithExtensionFallback] Warning: failed to embed metadata/cover: %v\n", err) - return - } - - if len(coverData) > 0 { - GoLog("[DownloadWithExtensionFallback] Embedded metadata and cover from %q\n", coverURL) - } else { - GoLog("[DownloadWithExtensionFallback] Embedded metadata without cover\n") - } -} - -func firstPositiveInt(values ...int) int { - for _, value := range values { - if value > 0 { - return value - } - } - return 0 -} diff --git a/go_backend/extension_fallback_output_test.go b/go_backend/extension_fallback_output_test.go deleted file mode 100644 index 15da77da..00000000 --- a/go_backend/extension_fallback_output_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package gobackend - -import ( - "os" - "path/filepath" - "testing" -) - -func TestShouldReuseExistingOutputProtectsCompletedFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "Artist - Track.flac") - if err := os.WriteFile(path, []byte("existing audio"), 0o644); err != nil { - t.Fatalf("write existing output: %v", err) - } - - if !shouldReuseExistingOutput(DownloadRequest{}, path) { - t.Fatal("expected completed output to be reused when variants are disabled") - } - if shouldReuseExistingOutput( - DownloadRequest{AllowQualityVariant: true}, - path, - ) { - t.Fatal("quality-variant staging output must remain independently writable") - } -} - -func TestShouldReuseExistingOutputIgnoresEmptyStagingFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "native_saf_work.flac") - if err := os.WriteFile(path, nil, 0o644); err != nil { - t.Fatalf("write staging output: %v", err) - } - - if shouldReuseExistingOutput(DownloadRequest{}, path) { - t.Fatal("empty staging output must not be treated as a completed download") - } -} diff --git a/go_backend/extension_fallback_resume_test.go b/go_backend/extension_fallback_resume_test.go deleted file mode 100644 index 4e557173..00000000 --- a/go_backend/extension_fallback_resume_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package gobackend - -import ( - "path/filepath" - "testing" -) - -func TestVerifiedDownloadResumeTriesSelectedProviderBeforeMetadata(t *testing.T) { - resetPreparedDownloadRequestCacheForTest() - t.Cleanup(resetPreparedDownloadRequestCacheForTest) - - metadataExt := newTestLoadedExtension(t, ExtensionTypeMetadataProvider) - metadataExt.ID = "resume-metadata" - metadataExt.Manifest.Name = metadataExt.ID - downloadExt := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - downloadExt.ID = "resume-download" - downloadExt.Manifest.Name = downloadExt.ID - - manager := getExtensionManager() - manager.mu.Lock() - previousExtensions := manager.extensions - manager.extensions = map[string]*loadedExtension{ - metadataExt.ID: metadataExt, - downloadExt.ID: downloadExt, - } - manager.mu.Unlock() - t.Cleanup(func() { - teardownExtension(metadataExt) - teardownExtension(downloadExt) - manager.mu.Lock() - manager.extensions = previousExtensions - manager.mu.Unlock() - }) - - req := DownloadRequest{ - ItemID: "resume-item", - Service: downloadExt.ID, - Source: metadataExt.ID, - SpotifyID: "source-track-1", - TrackName: "Original Song", - ArtistName: "Artist", - AlbumName: "Album", - ReleaseDate: "2026-05-04", - OutputDir: t.TempDir(), - OutputExt: ".flac", - FilenameFormat: "{title}", - Quality: "LOSSLESS", - UseFallback: false, - } - key := downloadPreparationKey(req) - cacheUnpreparedDownloadRequest(key, req) - - resp, err := DownloadWithExtensionFallback(req) - if err != nil { - t.Fatalf("DownloadWithExtensionFallback: %v", err) - } - if resp == nil || !resp.Success { - t.Fatalf("resume response = %#v", resp) - } - if got := filepath.Base(resp.FilePath); got != "Original Song.flac" { - t.Fatalf("resume ran metadata enrichment before download: file = %q", got) - } -} - -func TestVerifiedDownloadResumeReusesPreparedMetadata(t *testing.T) { - resetPreparedDownloadRequestCacheForTest() - t.Cleanup(resetPreparedDownloadRequestCacheForTest) - - metadataExt := newTestLoadedExtension(t, ExtensionTypeMetadataProvider) - metadataExt.ID = "prepared-metadata" - metadataExt.Manifest.Name = metadataExt.ID - downloadExt := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - downloadExt.ID = "prepared-download" - downloadExt.Manifest.Name = downloadExt.ID - - manager := getExtensionManager() - manager.mu.Lock() - previousExtensions := manager.extensions - manager.extensions = map[string]*loadedExtension{ - metadataExt.ID: metadataExt, - downloadExt.ID: downloadExt, - } - manager.mu.Unlock() - t.Cleanup(func() { - teardownExtension(metadataExt) - teardownExtension(downloadExt) - manager.mu.Lock() - manager.extensions = previousExtensions - manager.mu.Unlock() - }) - - req := DownloadRequest{ - ItemID: "prepared-item", - Service: downloadExt.ID, - Source: metadataExt.ID, - SpotifyID: "source-track-2", - TrackName: "Original Song", - ArtistName: "Artist", - AlbumName: "Album", - ReleaseDate: "2026-05-04", - OutputDir: t.TempDir(), - OutputExt: ".flac", - FilenameFormat: "{title}", - Quality: "LOSSLESS", - UseFallback: false, - } - key := downloadPreparationKey(req) - prepared := req - prepared.TrackName = "Prepared Song" - cachePreparedDownloadRequest(key, prepared) - - resp, err := DownloadWithExtensionFallback(req) - if err != nil { - t.Fatalf("DownloadWithExtensionFallback: %v", err) - } - if resp == nil || !resp.Success { - t.Fatalf("prepared response = %#v", resp) - } - if got := filepath.Base(resp.FilePath); got != "Prepared Song.flac" { - t.Fatalf("prepared metadata was not reused: file = %q", got) - } -} diff --git a/go_backend/extension_fallback_verification_test.go b/go_backend/extension_fallback_verification_test.go deleted file mode 100644 index ce941532..00000000 --- a/go_backend/extension_fallback_verification_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package gobackend - -import ( - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "sync/atomic" - "testing" -) - -func TestFallbackKeepsPendingVerificationOwnership(t *testing.T) { - for _, mode := range []string{"fresh-response", "saved-challenge", "cached-unavailable", "cold-session", "network-error", "download-error", "empty-download-error"} { - t.Run(mode, func(t *testing.T) { - primary := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - primary.ID, primary.Manifest.Name = "primary-provider", "primary-provider" - secondary := newTestLoadedExtension(t, ExtensionTypeMetadataProvider, ExtensionTypeDownloadProvider) - secondary.ID, secondary.Manifest.Name = "secondary-provider", "secondary-provider" - secondary.Manifest.Permissions.Storage = true - secondary.Manifest.SignedSession = &SignedSessionConfig{ - Namespace: "fixture-session", BaseURL: "https://auth.example.test", - } - last := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - last.ID, last.Manifest.Name = "last-provider", "last-provider" - for ext, script := range map[*loadedExtension]string{ - primary: `registerExtension({ - checkAvailability: function() { return {available: true, track_id: "source-track"}; }, - download: function() { return {success: false, error_type: "api_error", error_message: "primary request failed", retry_after_seconds: 37}; } - });`, - secondary: `var savedChallenge; - function queryCatalog() { - var response = session.signedFetch("GET", "/catalog"); - if (response.needsVerification) { - savedChallenge = new Error("VERIFY_REQUIRED"); - throw savedChallenge; - } - throw new Error("fixture expected a challenge"); - } - registerExtension({ - searchTracks: queryCatalog, - checkAvailability: function() { - if (fixtureMode === "cached-unavailable" || fixtureMode === "cold-session") return {available: false, reason: "No verified track match found"}; - if (fixtureMode === "network-error") throw new Error("lookup network timeout"); - if (fixtureMode === "download-error" || fixtureMode === "empty-download-error") return {available: true, track_id: "matched-track"}; - if (fixtureMode === "saved-challenge") throw savedChallenge; - return queryCatalog(); - }, - download: function() { - if (fixtureMode === "download-error") throw new Error("download network timeout"); - if (fixtureMode === "empty-download-error") return {success: false}; - throw new Error("verification must finish before download"); - } - });`, - last: `registerExtension({checkAvailability: function() { recordLastProvider(); return {available: false}; }});`, - } { - if err := os.WriteFile(filepath.Join(ext.SourceDir, "index.js"), fmt.Appendf(nil, "var fixtureMode = %q;\n%s", mode, script), 0600); err != nil { - t.Fatal(err) - } - } - manager := getExtensionManager() - manager.mu.Lock() - previousExtensions := manager.extensions - manager.extensions = map[string]*loadedExtension{primary.ID: primary, secondary.ID: secondary, last.ID: last} - manager.mu.Unlock() - previousPriority, previousFallback := GetProviderPriority(), GetExtensionFallbackProviderIDs() - SetProviderPriority([]string{secondary.ID, last.ID, primary.ID}) - SetExtensionFallbackProviderIDs(nil) - t.Cleanup(func() { - for _, ext := range []*loadedExtension{primary, secondary, last} { - ClearPendingAuthRequest(ext.ID) - teardownExtension(ext) - } - manager.mu.Lock() - manager.extensions = previousExtensions - manager.mu.Unlock() - SetProviderPriority(previousPriority) - SetExtensionFallbackProviderIDs(previousFallback) - resetPreparedDownloadRequestCacheForTest() - }) - if err := secondary.ensureRuntimeReady(); err != nil { - t.Fatal(err) - } - var bootstrapCalls atomic.Int32 - secondary.runtime.httpClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - bootstrapCalls.Add(1) - return &http.Response{ - StatusCode: http.StatusOK, Header: make(http.Header), Request: req, - Body: io.NopCloser(strings.NewReader(`{"auth_url":"https://auth.example.test/verify"}`)), - }, nil - })} - expectsVerification := mode == "fresh-response" || mode == "saved-challenge" || mode == "cached-unavailable" || mode == "cold-session" - if expectsVerification && mode != "cold-session" { - _, err := newExtensionProviderWrapper(secondary).SearchTracks("Song Artist", 1) - if err == nil || GetPendingAuthRequest(secondary.ID) == nil { - t.Fatalf("metadata lookup did not create a pending challenge: %v", err) - } - } - if !expectsVerification { - saveUsableSignedSession(t, secondary.runtime, *secondary.Manifest.SignedSession, "authenticated-session") - } - if err := last.ensureRuntimeReady(); err != nil { - t.Fatal(err) - } - var laterCalls atomic.Int32 - if err := last.VM.Set("recordLastProvider", func() { laterCalls.Add(1) }); err != nil { - t.Fatal(err) - } - response, err := DownloadWithExtensionFallback(DownloadRequest{ - Service: primary.ID, Source: primary.ID, ItemID: "fallback-" + mode, - SpotifyID: "source-track", TrackName: "Song", ArtistName: "Artist", AlbumName: "Album", - ISRC: "USABC2600001", ReleaseDate: "2026-01-01", Quality: "lossless", - OutputDir: t.TempDir(), FilenameFormat: "{title}", UseFallback: true, - }) - if err != nil || response == nil { - t.Fatalf("fallback response=%+v error=%v", response, err) - } - wantType := "verification_required" - switch mode { - case "network-error": - wantType = "network" - case "download-error": - wantType = "script_error" - case "empty-download-error": - wantType = "extension_error" - } - if response.ErrorType != wantType || response.Service != secondary.ID || response.RetryAfterSeconds != 0 { - t.Fatalf("failure lost its type, owner, or retry delay: %+v", response) - } - if expectsVerification { - if laterCalls.Load() != 0 || bootstrapCalls.Load() != 1 { - t.Fatalf("pending challenge was skipped or recreated: later=%d bootstrap=%d", laterCalls.Load(), bootstrapCalls.Load()) - } - } else if laterCalls.Load() != 1 || bootstrapCalls.Load() != 0 { - t.Fatalf("authenticated failures must allow fallback without bootstrapping: later=%d bootstrap=%d", laterCalls.Load(), bootstrapCalls.Load()) - } - }) - } -} diff --git a/go_backend/extension_goja_convert.go b/go_backend/extension_goja_convert.go deleted file mode 100644 index ee7ff11a..00000000 --- a/go_backend/extension_goja_convert.go +++ /dev/null @@ -1,660 +0,0 @@ -package gobackend - -import ( - "fmt" - "strconv" - "strings" - - "github.com/dop251/goja" -) - -func gojaValueIsEmpty(value goja.Value) bool { - return value == nil || goja.IsUndefined(value) || goja.IsNull(value) -} - -func gojaObjectString(obj *goja.Object, keys ...string) string { - for _, key := range keys { - value := obj.Get(key) - if gojaValueIsEmpty(value) { - continue - } - if str, ok := value.Export().(string); ok { - return str - } - } - return "" -} - -func gojaObjectValue(obj *goja.Object, keys ...string) goja.Value { - for _, key := range keys { - value := obj.Get(key) - if !gojaValueIsEmpty(value) { - return value - } - } - return nil -} - -func gojaObjectInt(obj *goja.Object, keys ...string) int { - for _, key := range keys { - value := obj.Get(key) - if gojaValueIsEmpty(value) { - continue - } - return int(value.ToInteger()) - } - return 0 -} - -func gojaObjectInt64(obj *goja.Object, keys ...string) int64 { - for _, key := range keys { - value := obj.Get(key) - if gojaValueIsEmpty(value) { - continue - } - return value.ToInteger() - } - return 0 -} - -func gojaObjectBool(obj *goja.Object, keys ...string) bool { - for _, key := range keys { - value := obj.Get(key) - if gojaValueIsEmpty(value) { - continue - } - return value.ToBoolean() - } - return false -} - -func gojaObjectInterfaceMap(obj *goja.Object, keys ...string) map[string]any { - value := gojaObjectValue(obj, keys...) - if gojaValueIsEmpty(value) { - return nil - } - - exported, ok := value.Export().(map[string]any) - if !ok || len(exported) == 0 { - return nil - } - return exported -} - -func gojaObjectStringMap(vm *goja.Runtime, obj *goja.Object, keys ...string) map[string]string { - value := gojaObjectValue(obj, keys...) - if gojaValueIsEmpty(value) { - return nil - } - - valueObj := value.ToObject(vm) - objectKeys := valueObj.Keys() - if len(objectKeys) == 0 { - return nil - } - - result := make(map[string]string, len(objectKeys)) - for _, childKey := range objectKeys { - childValue := valueObj.Get(childKey) - if gojaValueIsEmpty(childValue) { - continue - } - result[childKey] = childValue.String() - } - if len(result) == 0 { - return nil - } - return result -} - -func gojaObjectStringSlice(obj *goja.Object, keys ...string) []string { - value := gojaObjectValue(obj, keys...) - if gojaValueIsEmpty(value) { - return nil - } - exported, ok := value.Export().([]any) - if !ok || len(exported) == 0 { - return nil - } - result := make([]string, 0, len(exported)) - for _, item := range exported { - str, ok := item.(string) - if !ok { - continue - } - str = strings.TrimSpace(str) - if str != "" { - result = append(result, str) - } - } - if len(result) == 0 { - return nil - } - return result -} - -func gojaArrayLength(value goja.Value, vm *goja.Runtime) (int, error) { - if gojaValueIsEmpty(value) { - return 0, nil - } - lengthValue := value.ToObject(vm).Get("length") - if gojaValueIsEmpty(lengthValue) { - return 0, fmt.Errorf("value is not an array") - } - length := lengthValue.ToInteger() - if length <= 0 { - return 0, nil - } - return int(length), nil -} - -func parseExtensionTrackValue(vm *goja.Runtime, value goja.Value) ExtTrackMetadata { - obj := value.ToObject(vm) - return ExtTrackMetadata{ - ID: gojaObjectString(obj, "id"), - Name: gojaObjectString(obj, "name"), - Artists: gojaObjectString(obj, "artists"), - AlbumName: gojaObjectString(obj, "album_name", "albumName"), - AlbumArtist: gojaObjectString(obj, "album_artist", "albumArtist"), - AlbumID: gojaObjectString(obj, "album_id", "albumId"), - AlbumURL: gojaObjectString(obj, "album_url", "albumUrl"), - ArtistID: gojaObjectString(obj, "artist_id", "artistId"), - ArtistURL: gojaObjectString(obj, "artist_url", "artistUrl"), - ExternalURL: gojaObjectString(obj, "external_urls", "externalUrls", "external_url", "externalUrl", "url"), - DurationMS: gojaObjectInt(obj, "duration_ms", "durationMs"), - CoverURL: gojaObjectString(obj, "cover_url", "coverUrl"), - PreviewURL: gojaObjectString(obj, "preview_url", "previewUrl"), - Images: gojaObjectString(obj, "images"), - ReleaseDate: gojaObjectString(obj, "release_date", "releaseDate"), - TrackNumber: gojaObjectInt(obj, "track_number", "trackNumber"), - TotalTracks: gojaObjectInt(obj, "total_tracks", "totalTracks"), - DiscNumber: gojaObjectInt(obj, "disc_number", "discNumber"), - TotalDiscs: gojaObjectInt(obj, "total_discs", "totalDiscs"), - ISRC: gojaObjectString(obj, "isrc"), - ProviderID: gojaObjectString(obj, "provider_id", "providerId"), - ItemType: gojaObjectString(obj, "item_type", "itemType"), - AlbumType: gojaObjectString(obj, "album_type", "albumType"), - Explicit: gojaObjectBool(obj, "explicit", "is_explicit", "isExplicit"), - UPC: gojaObjectString(obj, "upc", "barcode"), - TidalID: gojaObjectString(obj, "tidal_id", "tidalId"), - QobuzID: gojaObjectString(obj, "qobuz_id", "qobuzId"), - DeezerID: gojaObjectString(obj, "deezer_id", "deezerId"), - SpotifyID: gojaObjectString(obj, "spotify_id", "spotifyId"), - ExternalLinks: gojaObjectStringMap(vm, obj, "external_links", "externalLinks"), - Label: gojaObjectString(obj, "label"), - Copyright: gojaObjectString(obj, "copyright"), - Genre: gojaObjectString(obj, "genre"), - Composer: gojaObjectString(obj, "composer"), - Comment: gojaObjectString(obj, "comment", "comments"), - AudioQuality: gojaObjectString(obj, "audio_quality", "audioQuality"), - AudioModes: gojaObjectString(obj, "audio_modes", "audioModes"), - } -} - -func parseExtensionTrackArray(vm *goja.Runtime, value goja.Value) ([]ExtTrackMetadata, error) { - length, err := gojaArrayLength(value, vm) - if err != nil { - return nil, err - } - if length == 0 { - return []ExtTrackMetadata{}, nil - } - - arrayObj := value.ToObject(vm) - tracks := make([]ExtTrackMetadata, 0, length) - for i := 0; i < length; i++ { - trackValue := arrayObj.Get(strconv.Itoa(i)) - if gojaValueIsEmpty(trackValue) { - continue - } - tracks = append(tracks, parseExtensionTrackValue(vm, trackValue)) - } - return tracks, nil -} - -func parseExtensionAlbumValue(vm *goja.Runtime, value goja.Value) (ExtAlbumMetadata, error) { - if gojaValueIsEmpty(value) { - return ExtAlbumMetadata{}, nil - } - - obj := value.ToObject(vm) - tracks := []ExtTrackMetadata{} - if tracksValue := gojaObjectValue(obj, "tracks"); !gojaValueIsEmpty(tracksValue) { - parsedTracks, err := parseExtensionTrackArray(vm, tracksValue) - if err != nil { - return ExtAlbumMetadata{}, err - } - tracks = parsedTracks - } - - return ExtAlbumMetadata{ - ID: gojaObjectString(obj, "id"), - Name: gojaObjectString(obj, "name"), - Artists: gojaObjectString(obj, "artists"), - ArtistID: gojaObjectString(obj, "artist_id", "artistId"), - CoverURL: gojaObjectString(obj, "cover_url", "coverUrl", "images"), - HeaderImage: gojaObjectString(obj, "header_image", "headerImage"), - HeaderVideo: gojaObjectString(obj, "header_video", "headerVideo"), - ReleaseDate: gojaObjectString(obj, "release_date", "releaseDate"), - TotalTracks: gojaObjectInt(obj, "total_tracks", "totalTracks"), - AlbumType: gojaObjectString(obj, "album_type", "albumType"), - AudioTraits: gojaObjectStringSlice(obj, "audio_traits", "audioTraits"), - Tracks: tracks, - ProviderID: gojaObjectString(obj, "provider_id", "providerId"), - }.withTrackFallbacks(), nil -} - -// withTrackFallbacks fills the album-level artist and release date from the -// album's tracks when the extension did not provide them at the album level. -// This is a generic mechanism so any extension benefits, without per-extension -// special-casing in the app. -func (a ExtAlbumMetadata) withTrackFallbacks() ExtAlbumMetadata { - if strings.TrimSpace(a.Artists) == "" { - a.Artists = albumArtistFromTracks(a.Tracks) - } - if strings.TrimSpace(a.ReleaseDate) == "" { - a.ReleaseDate = albumReleaseDateFromTracks(a.Tracks) - } - if len(a.AudioTraits) == 0 { - a.AudioTraits = albumAudioTraitsFromTracks(a.Tracks) - } - return a -} - -// albumArtistFromTracks prefers an explicit per-track album artist, then falls -// back to the most common track artist across the album. -func albumArtistFromTracks(tracks []ExtTrackMetadata) string { - for _, t := range tracks { - if s := strings.TrimSpace(t.AlbumArtist); s != "" { - return s - } - } - counts := map[string]int{} - order := []string{} - for _, t := range tracks { - artist := strings.TrimSpace(t.Artists) - if artist == "" { - continue - } - if _, ok := counts[artist]; !ok { - order = append(order, artist) - } - counts[artist]++ - } - best := "" - bestCount := 0 - for _, artist := range order { - if counts[artist] > bestCount { - best = artist - bestCount = counts[artist] - } - } - return best -} - -// albumReleaseDateFromTracks returns the first non-empty track release date. -func albumReleaseDateFromTracks(tracks []ExtTrackMetadata) string { - for _, t := range tracks { - if s := strings.TrimSpace(t.ReleaseDate); s != "" { - return s - } - } - return "" -} - -// albumAudioTraitsFromTracks derives album-level audio badges (Dolby Atmos, -// Hi-Res Lossless, Lossless) from the per-track audio quality/mode fields that -// extensions like Tidal and Qobuz already provide. Tokens match what the album -// header understands ("dolby_atmos", "hi_res_lossless", "lossless"). -func albumAudioTraitsFromTracks(tracks []ExtTrackMetadata) []string { - atmos := false - hiRes := false - lossless := false - - for _, t := range tracks { - modes := strings.ToUpper(t.AudioModes) - quality := strings.ToUpper(t.AudioQuality) - if strings.Contains(modes, "ATMOS") || strings.Contains(quality, "ATMOS") { - atmos = true - } - if strings.Contains(quality, "HI_RES") || - strings.Contains(quality, "HIRES") || - strings.Contains(quality, "MASTER") || - strings.Contains(quality, "MQA") { - hiRes = true - } - if strings.Contains(quality, "LOSSLESS") || - strings.Contains(quality, "FLAC") { - lossless = true - } - if bd, sr := parseBitDepthSampleRate(quality); bd > 0 { - if bd > 16 || sr > 48 { - hiRes = true - } else { - lossless = true - } - } - } - - traits := []string{} - if atmos { - traits = append(traits, "dolby_atmos") - } - if hiRes { - traits = append(traits, "hi_res_lossless") - } else if lossless { - traits = append(traits, "lossless") - } - return traits -} - -// parseBitDepthSampleRate extracts a bit depth and sample rate (in kHz) from -// labels such as "24bit/96kHz", "16bit/44.1kHz" or "24bit". -func parseBitDepthSampleRate(quality string) (int, float64) { - lower := strings.ToLower(quality) - bitDepth := 0 - sampleRate := 0.0 - - if idx := strings.Index(lower, "bit"); idx > 0 { - j := idx - for j > 0 && lower[j-1] >= '0' && lower[j-1] <= '9' { - j-- - } - if n, err := strconv.Atoi(lower[j:idx]); err == nil { - bitDepth = n - } - } - if idx := strings.Index(lower, "khz"); idx > 0 { - j := idx - for j > 0 && ((lower[j-1] >= '0' && lower[j-1] <= '9') || lower[j-1] == '.') { - j-- - } - if f, err := strconv.ParseFloat(lower[j:idx], 64); err == nil { - sampleRate = f - } - } - return bitDepth, sampleRate -} - -func parseExtensionAlbumArray(vm *goja.Runtime, value goja.Value) ([]ExtAlbumMetadata, error) { - length, err := gojaArrayLength(value, vm) - if err != nil { - return nil, err - } - if length == 0 { - return []ExtAlbumMetadata{}, nil - } - - arrayObj := value.ToObject(vm) - albums := make([]ExtAlbumMetadata, 0, length) - for i := 0; i < length; i++ { - albumValue := arrayObj.Get(strconv.Itoa(i)) - if gojaValueIsEmpty(albumValue) { - continue - } - album, err := parseExtensionAlbumValue(vm, albumValue) - if err != nil { - return nil, err - } - albums = append(albums, album) - } - return albums, nil -} - -func parseExtensionArtistValue(vm *goja.Runtime, value goja.Value) (ExtArtistMetadata, error) { - if gojaValueIsEmpty(value) { - return ExtArtistMetadata{}, nil - } - - obj := value.ToObject(vm) - albums := []ExtAlbumMetadata{} - if albumsValue := gojaObjectValue(obj, "albums"); !gojaValueIsEmpty(albumsValue) { - parsedAlbums, err := parseExtensionAlbumArray(vm, albumsValue) - if err != nil { - return ExtArtistMetadata{}, err - } - albums = parsedAlbums - } - - releases := []ExtAlbumMetadata{} - if releasesValue := gojaObjectValue(obj, "releases"); !gojaValueIsEmpty(releasesValue) { - parsedReleases, err := parseExtensionAlbumArray(vm, releasesValue) - if err != nil { - return ExtArtistMetadata{}, err - } - releases = parsedReleases - } - - topTracks := []ExtTrackMetadata{} - if topTracksValue := gojaObjectValue(obj, "top_tracks", "topTracks", "tracks"); !gojaValueIsEmpty(topTracksValue) { - parsedTopTracks, err := parseExtensionTrackArray(vm, topTracksValue) - if err != nil { - return ExtArtistMetadata{}, err - } - topTracks = parsedTopTracks - } - - return ExtArtistMetadata{ - ID: gojaObjectString(obj, "id"), - Name: gojaObjectString(obj, "name"), - ImageURL: gojaObjectString(obj, "image_url", "imageUrl"), - HeaderImage: gojaObjectString(obj, "header_image", "headerImage"), - HeaderVideo: gojaObjectString(obj, "header_video", "headerVideo"), - Listeners: gojaObjectInt(obj, "listeners"), - Albums: albums, - Releases: releases, - TopTracks: topTracks, - ProviderID: gojaObjectString(obj, "provider_id", "providerId"), - }, nil -} - -func parseExtensionAvailabilityValue(vm *goja.Runtime, value goja.Value) ExtAvailabilityResult { - obj := value.ToObject(vm) - return ExtAvailabilityResult{ - Available: gojaObjectBool(obj, "available"), - Reason: gojaObjectString(obj, "reason"), - TrackID: gojaObjectString(obj, "track_id", "trackId"), - SkipFallback: gojaObjectBool(obj, "skip_fallback", "skipFallback"), - PreparedContext: gojaObjectInterfaceMap( - obj, - "prepared_context", - "preparedContext", - ), - } -} - -func parseExtensionDownloadDecryptionValue(vm *goja.Runtime, value goja.Value) *DownloadDecryptionInfo { - if gojaValueIsEmpty(value) { - return nil - } - - obj := value.ToObject(vm) - info := &DownloadDecryptionInfo{ - Strategy: gojaObjectString(obj, "strategy"), - Key: gojaObjectString(obj, "key"), - IV: gojaObjectString(obj, "iv"), - InputFormat: gojaObjectString(obj, "input_format", "inputFormat"), - OutputExtension: gojaObjectString(obj, "output_extension", "outputExtension"), - Options: gojaObjectInterfaceMap(obj, "options"), - } - if info.Strategy == "" && info.Key == "" && info.IV == "" && info.InputFormat == "" && info.OutputExtension == "" && len(info.Options) == 0 { - return nil - } - return info -} - -func parseExtensionDownloadResultValue(vm *goja.Runtime, value goja.Value) ExtDownloadResult { - obj := value.ToObject(vm) - return ExtDownloadResult{ - Success: gojaObjectBool(obj, "success"), - FilePath: gojaObjectString(obj, "file_path", "filePath", "path"), - AlreadyExists: gojaObjectBool(obj, "already_exists", "alreadyExists"), - BitDepth: gojaObjectInt(obj, "bit_depth", "bitDepth"), - SampleRate: gojaObjectInt(obj, "sample_rate", "sampleRate"), - AudioCodec: gojaObjectString(obj, "audio_codec", "audioCodec", "codec"), - DurationMS: gojaObjectInt(obj, "duration_ms", "durationMs"), - ErrorMessage: gojaObjectString(obj, "error_message", "errorMessage", "error"), - ErrorType: gojaObjectString(obj, "error_type", "errorType"), - RetryAfterSeconds: gojaObjectInt(obj, "retry_after_seconds", "retryAfterSeconds"), - Title: gojaObjectString(obj, "title"), - Artist: gojaObjectString(obj, "artist"), - Album: gojaObjectString(obj, "album"), - AlbumArtist: gojaObjectString(obj, "album_artist", "albumArtist"), - TrackNumber: gojaObjectInt(obj, "track_number", "trackNumber"), - DiscNumber: gojaObjectInt(obj, "disc_number", "discNumber"), - TotalTracks: gojaObjectInt(obj, "total_tracks", "totalTracks"), - TotalDiscs: gojaObjectInt(obj, "total_discs", "totalDiscs"), - ReleaseDate: gojaObjectString(obj, "release_date", "releaseDate"), - CoverURL: gojaObjectString(obj, "cover_url", "coverUrl"), - ISRC: gojaObjectString(obj, "isrc"), - Genre: gojaObjectString(obj, "genre"), - Label: gojaObjectString(obj, "label"), - Copyright: gojaObjectString(obj, "copyright"), - Composer: gojaObjectString(obj, "composer"), - Comment: gojaObjectString(obj, "comment", "comments"), - Explicit: gojaObjectBool(obj, "explicit", "is_explicit", "isExplicit"), - AlbumType: gojaObjectString(obj, "album_type", "albumType"), - UPC: gojaObjectString(obj, "upc", "barcode"), - LyricsLRC: gojaObjectString(obj, "lyrics_lrc", "lyricsLrc"), - DecryptionKey: gojaObjectString(obj, "decryption_key", "decryptionKey"), - Decryption: parseExtensionDownloadDecryptionValue(vm, gojaObjectValue(obj, "decryption")), - ActualExtension: gojaObjectString(obj, "actual_extension", "actualExtension"), - OutputExtension: gojaObjectString(obj, "output_extension", "outputExtension"), - ActualContainer: gojaObjectString(obj, "actual_container", "actualContainer", "container"), - RequiresContainerConversion: gojaObjectBool( - obj, - "requires_container_conversion", - "requiresContainerConversion", - ), - } -} - -func parseExtensionURLHandleValue(vm *goja.Runtime, value goja.Value) (ExtURLHandleResult, error) { - obj := value.ToObject(vm) - handleResult := ExtURLHandleResult{ - Type: gojaObjectString(obj, "type"), - ID: gojaObjectString(obj, "id"), - Name: gojaObjectString(obj, "name"), - CoverURL: gojaObjectString(obj, "cover_url", "coverUrl"), - HeaderImage: gojaObjectString(obj, "header_image", "headerImage"), - HeaderVideo: gojaObjectString(obj, "header_video", "headerVideo"), - } - - if trackValue := gojaObjectValue(obj, "track"); !gojaValueIsEmpty(trackValue) { - track := parseExtensionTrackValue(vm, trackValue) - handleResult.Track = &track - } - if tracksValue := gojaObjectValue(obj, "tracks"); !gojaValueIsEmpty(tracksValue) { - tracks, err := parseExtensionTrackArray(vm, tracksValue) - if err != nil { - return ExtURLHandleResult{}, err - } - handleResult.Tracks = tracks - } - if albumValue := gojaObjectValue(obj, "album"); !gojaValueIsEmpty(albumValue) { - album, err := parseExtensionAlbumValue(vm, albumValue) - if err != nil { - return ExtURLHandleResult{}, err - } - handleResult.Album = &album - } - if artistValue := gojaObjectValue(obj, "artist"); !gojaValueIsEmpty(artistValue) { - artist, err := parseExtensionArtistValue(vm, artistValue) - if err != nil { - return ExtURLHandleResult{}, err - } - handleResult.Artist = &artist - } - - return handleResult, nil -} - -func parseExtensionPostProcessValue(vm *goja.Runtime, value goja.Value) PostProcessResult { - obj := value.ToObject(vm) - return PostProcessResult{ - Success: gojaObjectBool(obj, "success"), - NewFilePath: gojaObjectString(obj, "new_file_path", "newFilePath"), - NewFileURI: gojaObjectString(obj, "new_file_uri", "newFileUri"), - Error: gojaObjectString(obj, "error"), - BitDepth: gojaObjectInt(obj, "bit_depth", "bitDepth"), - SampleRate: gojaObjectInt(obj, "sample_rate", "sampleRate"), - } -} - -func parseExtensionLyricsLineArray(vm *goja.Runtime, value goja.Value) ([]ExtLyricsLine, error) { - length, err := gojaArrayLength(value, vm) - if err != nil { - return nil, err - } - if length == 0 { - return []ExtLyricsLine{}, nil - } - - arrayObj := value.ToObject(vm) - lines := make([]ExtLyricsLine, 0, length) - for i := 0; i < length; i++ { - lineValue := arrayObj.Get(strconv.Itoa(i)) - if gojaValueIsEmpty(lineValue) { - continue - } - lineObj := lineValue.ToObject(vm) - lines = append(lines, ExtLyricsLine{ - StartTimeMs: gojaObjectInt64(lineObj, "startTimeMs", "start_time_ms"), - Words: gojaObjectString(lineObj, "words"), - EndTimeMs: gojaObjectInt64(lineObj, "endTimeMs", "end_time_ms"), - }) - } - return lines, nil -} - -func parseExtensionLyricsValue(vm *goja.Runtime, value goja.Value) (ExtLyricsResult, error) { - obj := value.ToObject(vm) - lines := []ExtLyricsLine{} - if linesValue := gojaObjectValue(obj, "lines"); !gojaValueIsEmpty(linesValue) { - parsedLines, err := parseExtensionLyricsLineArray(vm, linesValue) - if err != nil { - return ExtLyricsResult{}, err - } - lines = parsedLines - } - - return ExtLyricsResult{ - Lines: lines, - SyncType: gojaObjectString(obj, "syncType", "sync_type"), - Instrumental: gojaObjectBool(obj, "instrumental"), - PlainLyrics: gojaObjectString(obj, "plainLyrics", "plain_lyrics"), - Provider: gojaObjectString(obj, "provider"), - }, nil -} - -func parseExtensionSearchResult(vm *goja.Runtime, value goja.Value) (ExtSearchResult, error) { - if gojaValueIsEmpty(value) { - return ExtSearchResult{}, nil - } - - resultObj := value.ToObject(vm) - tracksValue := resultObj.Get("tracks") - if gojaValueIsEmpty(tracksValue) { - tracks, err := parseExtensionTrackArray(vm, value) - if err != nil { - return ExtSearchResult{}, err - } - return ExtSearchResult{ - Tracks: tracks, - Total: len(tracks), - }, nil - } - - tracks, err := parseExtensionTrackArray(vm, tracksValue) - if err != nil { - return ExtSearchResult{}, err - } - total := gojaObjectInt(resultObj, "total") - if total == 0 { - total = len(tracks) - } - return ExtSearchResult{ - Tracks: tracks, - Total: total, - }, nil -} diff --git a/go_backend/extension_goja_convert_url_handle_test.go b/go_backend/extension_goja_convert_url_handle_test.go deleted file mode 100644 index d89da14d..00000000 --- a/go_backend/extension_goja_convert_url_handle_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package gobackend - -import ( - "testing" - - "github.com/dop251/goja" -) - -func TestParseExtensionURLHandleValueCapturesID(t *testing.T) { - vm := goja.New() - value, err := vm.RunString(`({ - type: "playlist", - id: "37i9dQZF1DXcBWIGoYBM5M", - name: "Discover Weekly", - tracks: [] - })`) - if err != nil { - t.Fatalf("RunString: %v", err) - } - - result, err := parseExtensionURLHandleValue(vm, value) - if err != nil { - t.Fatalf("parseExtensionURLHandleValue: %v", err) - } - if result.Type != "playlist" { - t.Fatalf("Type = %q", result.Type) - } - if result.ID != "37i9dQZF1DXcBWIGoYBM5M" { - t.Fatalf("ID = %q, want the playlist id from the handler result", result.ID) - } - if result.Name != "Discover Weekly" { - t.Fatalf("Name = %q", result.Name) - } -} - -func TestParseExtensionURLHandleValueOmitsIDWhenAbsent(t *testing.T) { - vm := goja.New() - // Track/album/artist results carry their own ID inside their nested - // metadata, so a handler that omits the top-level "id" (as these do - // today) must not surface a stale or zero-value ID. - value, err := vm.RunString(`({ type: "track", name: "A Track" })`) - if err != nil { - t.Fatalf("RunString: %v", err) - } - - result, err := parseExtensionURLHandleValue(vm, value) - if err != nil { - t.Fatalf("parseExtensionURLHandleValue: %v", err) - } - if result.ID != "" { - t.Fatalf("ID = %q, want empty when the handler result has none", result.ID) - } -} diff --git a/go_backend/extension_health.go b/go_backend/extension_health.go deleted file mode 100644 index 100249f1..00000000 --- a/go_backend/extension_health.go +++ /dev/null @@ -1,542 +0,0 @@ -package gobackend - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "sync" - "time" -) - -const ( - extensionHealthDefaultTimeout = 4 * time.Second - extensionHealthMaxBodyBytes = 64 * 1024 - extensionHealthDefaultCache = 10 * time.Minute - extensionHealthMinCache = 60 * time.Second - extensionHealthUnknownCache = 2 * time.Minute -) - -type ExtensionHealthResult struct { - ExtensionID string `json:"extension_id"` - Status string `json:"status"` - CheckedAt string `json:"checked_at"` - Checks []ExtensionHealthCheckResult `json:"checks"` -} - -type ExtensionHealthCheckResult struct { - ID string `json:"id"` - Label string `json:"label,omitempty"` - URL string `json:"url"` - Method string `json:"method"` - ServiceKey string `json:"service_key,omitempty"` - Required bool `json:"required"` - Status string `json:"status"` - HTTPStatus int `json:"http_status,omitempty"` - LatencyMs int64 `json:"latency_ms"` - Message string `json:"message,omitempty"` - Error string `json:"error,omitempty"` - CheckedAt string `json:"checked_at"` -} - -type cachedExtensionHealthResult struct { - result ExtensionHealthResult - expiresAt time.Time -} - -var ( - extensionHealthCacheMu sync.Mutex - extensionHealthCache = map[string]cachedExtensionHealthResult{} - extensionHealthRefresh = map[string]struct{}{} - extensionHealthGeneration uint64 -) - -func clearExtensionHealthCache() { - extensionHealthCacheMu.Lock() - extensionHealthCache = map[string]cachedExtensionHealthResult{} - extensionHealthRefresh = map[string]struct{}{} - extensionHealthGeneration++ - extensionHealthCacheMu.Unlock() -} - -// PeekExtensionHealthCached returns only an already-computed, unexpired -// health snapshot. It never performs network I/O. Download provider ordering -// uses this path so an informational health endpoint can never delay the real -// provider attempt. -func PeekExtensionHealthCached(ext *loadedExtension) (ExtensionHealthResult, bool) { - if ext == nil || ext.Manifest == nil || len(ext.Manifest.ServiceHealth) == 0 { - return ExtensionHealthResult{}, false - } - cacheKey := strings.TrimSpace(ext.ID) - if cacheKey == "" { - return ExtensionHealthResult{}, false - } - - now := time.Now() - extensionHealthCacheMu.Lock() - defer extensionHealthCacheMu.Unlock() - cached, ok := extensionHealthCache[cacheKey] - if !ok || !now.Before(cached.expiresAt) { - return ExtensionHealthResult{}, false - } - return cached.result, true -} - -func peekExtensionHealthStale(ext *loadedExtension) (ExtensionHealthResult, bool) { - if ext == nil { - return ExtensionHealthResult{}, false - } - cacheKey := strings.TrimSpace(ext.ID) - if cacheKey == "" { - return ExtensionHealthResult{}, false - } - extensionHealthCacheMu.Lock() - defer extensionHealthCacheMu.Unlock() - cached, ok := extensionHealthCache[cacheKey] - return cached.result, ok -} - -// RefreshExtensionHealthAsync coalesces background refreshes per extension. -// Callers deliberately do not wait: stale or missing health data must not be -// on the latency-critical download path. -func RefreshExtensionHealthAsync(ext *loadedExtension) { - if ext == nil || ext.Manifest == nil || len(ext.Manifest.ServiceHealth) == 0 { - return - } - cacheKey := strings.TrimSpace(ext.ID) - if cacheKey == "" { - return - } - - extensionHealthCacheMu.Lock() - if _, refreshing := extensionHealthRefresh[cacheKey]; refreshing { - extensionHealthCacheMu.Unlock() - return - } - extensionHealthRefresh[cacheKey] = struct{}{} - generation := extensionHealthGeneration - extensionHealthCacheMu.Unlock() - - go func() { - result := CheckExtensionHealth(ext) - extensionHealthCacheMu.Lock() - if generation == extensionHealthGeneration { - cacheExtensionHealthResultLocked(ext, result) - delete(extensionHealthRefresh, cacheKey) - } - extensionHealthCacheMu.Unlock() - }() -} - -func CheckExtensionHealthJSON(extensionID string) (string, error) { - manager := getExtensionManager() - ext, err := manager.GetExtension(extensionID) - if err != nil { - return "", err - } - - result := CheckExtensionHealth(ext) - cacheExtensionHealthResult(ext, result) - bytes, err := json.Marshal(result) - if err != nil { - return "", err - } - return string(bytes), nil -} - -func CheckExtensionHealthCached(ext *loadedExtension) ExtensionHealthResult { - if ext == nil || ext.Manifest == nil || len(ext.Manifest.ServiceHealth) == 0 { - return CheckExtensionHealth(ext) - } - - cacheKey := strings.TrimSpace(ext.ID) - if cacheKey == "" { - return CheckExtensionHealth(ext) - } - - if cached, ok := PeekExtensionHealthCached(ext); ok { - return cached - } - - result := CheckExtensionHealth(ext) - cacheExtensionHealthResult(ext, result) - return result -} - -func cacheExtensionHealthResult(ext *loadedExtension, result ExtensionHealthResult) { - if ext == nil || ext.Manifest == nil || len(ext.Manifest.ServiceHealth) == 0 { - return - } - - cacheKey := strings.TrimSpace(ext.ID) - if cacheKey == "" { - return - } - - extensionHealthCacheMu.Lock() - cacheExtensionHealthResultLocked(ext, result) - extensionHealthCacheMu.Unlock() -} - -// cacheExtensionHealthResultLocked stores a result while the caller owns -// extensionHealthCacheMu. Keeping this small helper avoids a clear-vs-refresh -// race without taking the same mutex recursively. -func cacheExtensionHealthResultLocked(ext *loadedExtension, result ExtensionHealthResult) { - cacheKey := strings.TrimSpace(ext.ID) - - ttl := extensionHealthCacheTTL(ext.Manifest.ServiceHealth) - if result.Status == "unknown" && ttl > extensionHealthUnknownCache { - ttl = extensionHealthUnknownCache - } - - extensionHealthCache[cacheKey] = cachedExtensionHealthResult{ - result: result, - expiresAt: time.Now().Add(ttl), - } -} - -func CheckExtensionHealth(ext *loadedExtension) ExtensionHealthResult { - now := time.Now().UTC().Format(time.RFC3339) - result := ExtensionHealthResult{ - ExtensionID: "", - Status: "unsupported", - CheckedAt: now, - Checks: []ExtensionHealthCheckResult{}, - } - if ext == nil || ext.Manifest == nil { - result.Status = "offline" - return result - } - - result.ExtensionID = ext.ID - checks := ext.Manifest.ServiceHealth - if len(checks) == 0 { - return result - } - - result.Status = "online" - for _, check := range checks { - checkResult := runExtensionHealthCheck(ext.Manifest, check) - result.Checks = append(result.Checks, checkResult) - - switch checkResult.Status { - case "offline": - if check.Required { - result.Status = "offline" - } else if result.Status == "online" { - result.Status = "degraded" - } - case "degraded": - if result.Status == "online" { - result.Status = "degraded" - } - case "unknown": - if result.Status == "online" { - result.Status = "unknown" - } - } - } - - return result -} - -func extensionHealthCacheTTL(checks []ExtensionHealthCheck) time.Duration { - ttl := extensionHealthDefaultCache - for _, check := range checks { - if check.CacheTTLSeconds <= 0 { - continue - } - checkTTL := time.Duration(check.CacheTTLSeconds) * time.Second - if checkTTL < extensionHealthMinCache { - checkTTL = extensionHealthMinCache - } - if checkTTL < ttl { - ttl = checkTTL - } - } - return ttl -} - -func runExtensionHealthCheck(manifest *ExtensionManifest, check ExtensionHealthCheck) ExtensionHealthCheckResult { - method := strings.ToUpper(strings.TrimSpace(check.Method)) - if method == "" { - method = http.MethodGet - } - now := time.Now().UTC().Format(time.RFC3339) - result := ExtensionHealthCheckResult{ - ID: check.ID, - Label: check.Label, - URL: check.URL, - Method: method, - ServiceKey: strings.TrimSpace(check.ServiceKey), - Required: check.Required, - Status: "unknown", - CheckedAt: now, - } - - parsed, err := url.Parse(check.URL) - if err != nil { - result.Status = "offline" - result.Error = fmt.Sprintf("invalid health URL: %v", err) - return result - } - if parsed.Scheme != "https" { - result.Status = "offline" - result.Error = "health check must use https" - return result - } - host := parsed.Hostname() - if host == "" { - result.Status = "offline" - result.Error = "health check URL hostname is required" - return result - } - if isPrivateIP(host) { - result.Status = "offline" - result.Error = "private/local health check host is not allowed" - return result - } - if manifest == nil || !manifest.IsDomainAllowed(host) { - result.Status = "offline" - result.Error = fmt.Sprintf("health check host '%s' is not in extension network permissions", host) - return result - } - if method != http.MethodGet && method != http.MethodHead { - result.Status = "offline" - result.Error = "health check method must be GET or HEAD" - return result - } - - timeout := extensionHealthDefaultTimeout - if check.TimeoutMs > 0 { - timeout = time.Duration(check.TimeoutMs) * time.Millisecond - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, method, check.URL, nil) - if err != nil { - result.Status = "offline" - result.Error = err.Error() - return result - } - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", userAgentForURL(parsed)) - - start := time.Now() - resp, err := NewMetadataHTTPClient(timeout).Do(req) - result.LatencyMs = time.Since(start).Milliseconds() - if err != nil { - if isTransientExtensionHealthError(err) { - result.Status = "unknown" - } else { - result.Status = "offline" - } - result.Error = err.Error() - return result - } - defer resp.Body.Close() - - result.HTTPStatus = resp.StatusCode - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - result.Status = "offline" - result.Message = resp.Status - return result - } - - if method == http.MethodHead { - result.Status = "online" - result.Message = resp.Status - return result - } - - body, err := io.ReadAll(io.LimitReader(resp.Body, extensionHealthMaxBodyBytes)) - if err != nil { - result.Status = "degraded" - result.Error = err.Error() - return result - } - - status, message := classifyExtensionHealthBody(body, check.ServiceKey) - result.Status = status - if message == "" { - result.Message = resp.Status - } else { - result.Message = message - } - return result -} - -func isTransientExtensionHealthError(err error) bool { - return isTransientNetworkError(err) || isConnectivityFailure(err) -} - -func classifyExtensionHealthBody(body []byte, serviceKey string) (string, string) { - if len(strings.TrimSpace(string(body))) == 0 { - return "online", "" - } - - var payload map[string]any - if err := json.Unmarshal(body, &payload); err != nil { - return "online", "" - } - - serviceKey = strings.TrimSpace(serviceKey) - if serviceKey != "" { - if status, message, ok := classifyExtensionHealthService(payload, serviceKey); ok { - return status, message - } - } - - rawStatus, _ := payload["status"].(string) - normalized := strings.ToLower(strings.TrimSpace(rawStatus)) - switch normalized { - case "", "ok", "up", "online", "healthy", "operational", "pass", "passing": - return "online", rawStatus - case "degraded", "partial", "warning", "warn": - return "degraded", rawStatus - case "down", "offline", "error", "failed", "fail", "unhealthy": - if isTransientHealthStatusMessage(string(body)) { - return "unknown", rawStatus - } - return "offline", rawStatus - default: - return "online", rawStatus - } -} - -func classifyExtensionHealthService(payload map[string]any, serviceKey string) (string, string, bool) { - rawServices, ok := payload["services"] - if !ok { - return "", "", false - } - services, ok := rawServices.(map[string]any) - if !ok { - return "", "", false - } - rawService, ok := services[serviceKey] - if !ok { - return "unknown", fmt.Sprintf("service '%s' not found", serviceKey), true - } - service, ok := rawService.(map[string]any) - if !ok { - return "unknown", fmt.Sprintf("service '%s' has invalid health payload", serviceKey), true - } - - label, _ := service["label"].(string) - detail, _ := service["detail"].(string) - errText, _ := service["error"].(string) - messageParts := []string{} - if strings.TrimSpace(label) != "" { - messageParts = append(messageParts, strings.TrimSpace(label)) - } - if strings.TrimSpace(detail) != "" { - messageParts = append(messageParts, strings.TrimSpace(detail)) - } - if strings.TrimSpace(errText) != "" { - messageParts = append(messageParts, strings.TrimSpace(errText)) - } - - rawStatus, hasStatus := service["status"] - okValue, hasOK := service["ok"].(bool) - joinedMessage := strings.Join(messageParts, ": ") - transient := isTransientHealthStatusMessage(detail) || - isTransientHealthStatusMessage(errText) || - isTransientHealthStatusMessage(label) - - if statusCode, ok := healthNumber(rawStatus); ok { - if statusCode >= 200 && statusCode < 300 { - return "online", joinedMessage, true - } - if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden { - return "degraded", joinedMessage, true - } - if statusCode == http.StatusInternalServerError && hasOK && okValue { - return "online", joinedMessage, true - } - if transient || isTransientHealthStatusCode(statusCode) { - return "unknown", joinedMessage, true - } - return "offline", joinedMessage, true - } - - if isExtensionHealthAuthRequired(detail) { - return "degraded", joinedMessage, true - } - if transient { - return "unknown", joinedMessage, true - } - if hasOK { - if okValue { - return "online", joinedMessage, true - } - return "offline", joinedMessage, true - } - if !hasStatus { - return "unknown", joinedMessage, true - } - - statusString := strings.ToLower(strings.TrimSpace(fmt.Sprintf("%v", rawStatus))) - switch statusString { - case "ok", "up", "online", "healthy", "operational": - return "online", joinedMessage, true - case "degraded", "partial", "warning", "warn": - return "degraded", joinedMessage, true - case "down", "offline", "error", "failed", "fail", "unhealthy": - return "offline", joinedMessage, true - default: - return "unknown", joinedMessage, true - } -} - -func isExtensionHealthAuthRequired(detail string) bool { - switch strings.ToLower(strings.TrimSpace(detail)) { - case "auth_required", "authorization_required", "login_required", "unauthorized": - return true - default: - return false - } -} - -func isTransientHealthStatusMessage(text string) bool { - t := strings.ToLower(strings.TrimSpace(text)) - if t == "" { - return false - } - return strings.Contains(t, "context deadline exceeded") || - strings.Contains(t, "deadline exceeded") || - strings.Contains(t, "timeout") || - strings.Contains(t, "timed out") || - strings.Contains(t, "temporarily unavailable") || - strings.Contains(t, "try again") -} - -func isTransientHealthStatusCode(code int) bool { - switch code { - case http.StatusRequestTimeout, - http.StatusTooManyRequests, - http.StatusBadGateway, - http.StatusServiceUnavailable, - http.StatusGatewayTimeout: - return true - default: - return false - } -} - -func healthNumber(value any) (int, bool) { - switch v := value.(type) { - case float64: - return int(v), true - case int: - return v, true - case json.Number: - n, err := v.Int64() - return int(n), err == nil - default: - return 0, false - } -} diff --git a/go_backend/extension_health_misc_supplement_test.go b/go_backend/extension_health_misc_supplement_test.go deleted file mode 100644 index 6bcd0097..00000000 --- a/go_backend/extension_health_misc_supplement_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package gobackend - -import ( - "context" - "encoding/json" - "net" - "strings" - "testing" - "time" -) - -func TestExtensionHealthClassificationAndValidation(t *testing.T) { - if status, msg := classifyExtensionHealthBody([]byte(`{"status":"degraded"}`), ""); status != "degraded" || msg != "degraded" { - t.Fatalf("status/message = %q/%q", status, msg) - } - if status, _ := classifyExtensionHealthBody([]byte(`not-json`), ""); status != "online" { - t.Fatalf("invalid JSON status = %q", status) - } - if status, msg := classifyExtensionHealthBody([]byte(`{"services":{"provider":{"status":401,"label":"Provider","detail":"auth_required"}}}`), "provider"); status != "degraded" || !strings.Contains(msg, "Provider") { - t.Fatalf("service status/message = %q/%q", status, msg) - } - if status, msg, ok := classifyExtensionHealthService(map[string]any{"services": map[string]any{}}, "missing"); !ok || status != "unknown" || !strings.Contains(msg, "missing") { - t.Fatalf("missing service = %q/%q/%v", status, msg, ok) - } - if n, ok := healthNumber(json.Number("503")); !ok || n != 503 { - t.Fatalf("health number = %d/%v", n, ok) - } - if !isExtensionHealthAuthRequired(" unauthorized ") { - t.Fatal("expected auth required") - } - if !isTransientExtensionHealthError(context.DeadlineExceeded) || !isTransientExtensionHealthError(&net.DNSError{IsTimeout: true}) { - t.Fatal("expected timeout health errors to be transient") - } - if !isTransientExtensionHealthError(&net.DNSError{IsNotFound: true}) { - t.Fatal("expected health transport lookup errors to be indeterminate") - } - - if result := CheckExtensionHealth(nil); result.Status != "offline" { - t.Fatalf("nil health = %#v", result) - } - manifest := &ExtensionManifest{Permissions: ExtensionPermissions{Network: []string{"status.example.com"}}} - invalidURL := runExtensionHealthCheck(manifest, ExtensionHealthCheck{ID: "bad", URL: "://bad"}) - if invalidURL.Status != "offline" { - t.Fatalf("invalid URL = %#v", invalidURL) - } - insecure := runExtensionHealthCheck(manifest, ExtensionHealthCheck{ID: "http", URL: "http://status.example.com"}) - if insecure.Status != "offline" || !strings.Contains(insecure.Error, "https") { - t.Fatalf("insecure = %#v", insecure) - } - disallowedHost := runExtensionHealthCheck(manifest, ExtensionHealthCheck{ID: "host", URL: "https://other.example.com"}) - if disallowedHost.Status != "offline" || !strings.Contains(disallowedHost.Error, "permissions") { - t.Fatalf("host = %#v", disallowedHost) - } - badMethod := runExtensionHealthCheck(manifest, ExtensionHealthCheck{ID: "method", URL: "https://status.example.com", Method: "POST"}) - if badMethod.Status != "offline" || !strings.Contains(badMethod.Error, "method") { - t.Fatalf("method = %#v", badMethod) - } - - ext := &loadedExtension{ - ID: "health-ext", - Manifest: &ExtensionManifest{ - ServiceHealth: []ExtensionHealthCheck{ - {ID: "required", URL: "http://status.example.com", Required: true}, - {ID: "optional", URL: "http://status.example.com", Required: false}, - }, - }, - } - if result := CheckExtensionHealth(ext); result.Status != "offline" || len(result.Checks) != 2 { - t.Fatalf("extension health = %#v", result) - } -} - -func TestCoverHelpersRejectEmptyURL(t *testing.T) { - if data, err := downloadCoverToMemory(""); err == nil || data != nil { - t.Fatalf("expected empty cover error") - } -} - -func TestPeekExtensionHealthCachedNeverRefreshesSynchronously(t *testing.T) { - clearExtensionHealthCache() - ext := &loadedExtension{ - ID: "cached-health-ext", - Manifest: &ExtensionManifest{ServiceHealth: []ExtensionHealthCheck{{ - ID: "main", URL: "https://status.example.com", - }}}, - } - if _, ok := PeekExtensionHealthCached(ext); ok { - t.Fatal("unexpected cache hit") - } - - want := ExtensionHealthResult{ExtensionID: ext.ID, Status: "degraded"} - extensionHealthCacheMu.Lock() - extensionHealthCache[ext.ID] = cachedExtensionHealthResult{ - result: want, expiresAt: time.Now().Add(time.Minute), - } - extensionHealthCacheMu.Unlock() - got, ok := PeekExtensionHealthCached(ext) - if !ok || got.Status != want.Status { - t.Fatalf("cached health = %#v/%v", got, ok) - } - - extensionHealthCacheMu.Lock() - entry := extensionHealthCache[ext.ID] - entry.expiresAt = time.Now().Add(-time.Second) - extensionHealthCache[ext.ID] = entry - extensionHealthCacheMu.Unlock() - if _, ok := PeekExtensionHealthCached(ext); ok { - t.Fatal("expired cache entry was returned") - } - if stale, ok := peekExtensionHealthStale(ext); !ok || stale.Status != want.Status { - t.Fatalf("stale health snapshot = %#v/%v", stale, ok) - } -} diff --git a/go_backend/extension_lifecycle_signed_session_timeout_test.go b/go_backend/extension_lifecycle_signed_session_timeout_test.go deleted file mode 100644 index 8119f0de..00000000 --- a/go_backend/extension_lifecycle_signed_session_timeout_test.go +++ /dev/null @@ -1,265 +0,0 @@ -package gobackend - -import ( - "context" - "errors" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/dop251/goja" -) - -func TestInitializeVMLockedBoundsTopLevelScript(t *testing.T) { - previousTimeout := extensionLifecycleTimeout - extensionLifecycleTimeout = 10 * time.Millisecond - t.Cleanup(func() { extensionLifecycleTimeout = previousTimeout }) - - sourceDir := t.TempDir() - indexPath := filepath.Join(sourceDir, "index.js") - if err := os.WriteFile(indexPath, []byte("registerExtension({}); while (true) {}"), 0600); err != nil { - t.Fatal(err) - } - ext := &loadedExtension{ - ID: "lifecycle-top-level-timeout", - Manifest: &ExtensionManifest{Name: "lifecycle-top-level-timeout"}, - SourceDir: sourceDir, - DataDir: t.TempDir(), - } - - err := initializeVMLocked(ext) - if err == nil || !IsTimeoutError(err) { - t.Fatalf("initialize error = %v, want timeout", err) - } - if ext.VM != nil || ext.runtime != nil || ext.initialized { - t.Fatalf("timed-out VM was not discarded: VM=%v runtime=%v initialized=%v", ext.VM, ext.runtime, ext.initialized) - } -} - -func TestInitializeAndCleanupLifecycleCallbacksAreBounded(t *testing.T) { - previousTimeout := extensionLifecycleTimeout - extensionLifecycleTimeout = 10 * time.Millisecond - t.Cleanup(func() { extensionLifecycleTimeout = previousTimeout }) - - vm := goja.New() - if _, err := vm.RunString(`extension = { - initialize: function() { while (true) {} }, - cleanup: function() { while (true) {} } - }`); err != nil { - t.Fatal(err) - } - - if err := initializeExtensionRuntimeWithSettings(vm, "lifecycle-callback-timeout", map[string]any{"quality": "lossless"}); err == nil || !IsTimeoutError(err) { - t.Fatalf("initialize callback error = %v, want timeout", err) - } - if err := runCleanupOnVM(vm); err == nil || !IsTimeoutError(err) { - t.Fatalf("cleanup callback error = %v, want timeout", err) - } -} - -func TestLifecycleTimeoutQuarantinesUnresponsiveCleanupVM(t *testing.T) { - previousTimeout := extensionLifecycleTimeout - previousGrace := jsInterruptGracePeriod - extensionLifecycleTimeout = 10 * time.Millisecond - jsInterruptGracePeriod = 10 * time.Millisecond - t.Cleanup(func() { - extensionLifecycleTimeout = previousTimeout - jsInterruptGracePeriod = previousGrace - }) - - vm := goja.New() - release := make(chan struct{}) - if err := vm.Set("block", func() { <-release }); err != nil { - t.Fatal(err) - } - if _, err := vm.RunString(`extension = { cleanup: function() { block(); } }`); err != nil { - t.Fatal(err) - } - ext := &loadedExtension{ - ID: "lifecycle-cleanup-quarantine", - Manifest: &ExtensionManifest{Name: "lifecycle-cleanup-quarantine"}, - VM: vm, - runtime: &extensionRuntime{}, - } - - done := make(chan error, 1) - go func() { - done <- func() error { ext.VMMu.Lock(); defer ext.VMMu.Unlock(); teardownVMLocked(ext); return nil }() - }() - - select { - case err := <-done: - if err != nil { - t.Fatal(err) - } - case <-time.After(time.Second): - t.Fatal("teardown did not return after cleanup timeout") - } - ext.VMMu.Lock() - vmRemaining, runtimeRemaining := ext.VM, ext.runtime - ext.VMMu.Unlock() - if vmRemaining != nil || runtimeRemaining != nil || !hasQuarantinedRuntime(ext) { - t.Fatalf("unsafe cleanup was not quarantined: VM=%v runtime=%v quarantined=%v", vmRemaining, runtimeRemaining, hasQuarantinedRuntime(ext)) - } - close(release) - deadline := time.Now().Add(time.Second) - for hasQuarantinedRuntime(ext) && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if hasQuarantinedRuntime(ext) { - t.Fatal("quarantined cleanup runtime did not finish") - } -} - -func TestSignedSessionGrantRetryHonorsCancellationAndReleasesCoordinator(t *testing.T) { - previousWait := signedSessionRetryWaitContext - signedSessionRetryWaitContext = func(ctx context.Context, _ time.Duration) error { - <-ctx.Done() - return ctx.Err() - } - t.Cleanup(func() { signedSessionRetryWaitContext = previousWait }) - - var calls atomic.Int32 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls.Add(1) - return &http.Response{ - StatusCode: http.StatusTooManyRequests, - Header: http.Header{"Retry-After": []string{"300"}}, - Body: io.NopCloser(strings.NewReader(`{}`)), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "signed-cancel", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "signed-cancel", - BaseURL: "https://auth.example.com", - } - - ctx, cancel := context.WithCancel(context.Background()) - errCh := make(chan error, 1) - go func() { errCh <- runtime.exchangeSignedSessionGrantContext(ctx, "grant-cancel") }() - - deadline := time.Now().Add(time.Second) - for calls.Load() == 0 && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if calls.Load() == 0 { - t.Fatal("exchange request was not started") - } - - coordinator, err := runtime.signedSessionCoordinator(signedSessionConfigWithDefaults(runtime.manifest.SignedSession)) - if err != nil { - t.Fatal(err) - } - lockAcquired := make(chan struct{}) - go func() { - coordinator.mu.Lock() - defer coordinator.mu.Unlock() - close(lockAcquired) - }() - select { - case <-lockAcquired: - case <-time.After(time.Second): - t.Fatal("coordinator mutex remained held during Retry-After wait") - } - - cancel() - select { - case err := <-errCh: - if !errors.Is(err, context.Canceled) { - t.Fatalf("exchange error = %v, want context cancellation", err) - } - case <-time.After(time.Second): - t.Fatal("cancelled exchange did not return") - } - - coordinator.mu.Lock() - inFlight := coordinator.exchangeInFlight - coordinator.mu.Unlock() - if inFlight { - t.Fatal("coordinator exchange lease remained in flight after cancellation") - } -} - -func TestSignedSessionClearInvalidatesInFlightExchangeCommit(t *testing.T) { - requestStarted := make(chan struct{}) - releaseResponse := make(chan struct{}) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - close(requestStarted) - select { - case <-releaseResponse: - case <-req.Context().Done(): - return nil, req.Context().Err() - } - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"session_id":"late","session_secret":"late-secret","expires_at":"2099-01-01T00:00:00Z"}`, - )), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "signed-clear-in-flight", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "signed-clear-in-flight", - BaseURL: "https://auth.example.com", - } - - errCh := make(chan error, 1) - go func() { errCh <- runtime.exchangeSignedSessionGrant("grant-before-clear") }() - select { - case <-requestStarted: - case <-time.After(time.Second): - t.Fatal("exchange request was not started") - } - - clearResult := runtime.signedSessionClear(goja.FunctionCall{}).Export().(map[string]any) - if clearResult["success"] != true { - t.Fatalf("clear result = %#v, want success", clearResult) - } - close(releaseResponse) - select { - case err := <-errCh: - if err == nil || !strings.Contains(err.Error(), "superseded by session clear") { - t.Fatalf("exchange error = %v, want clear-generation rejection", err) - } - case <-time.After(time.Second): - t.Fatal("exchange did not finish") - } - - config := signedSessionConfigWithDefaults(runtime.manifest.SignedSession) - record, err := runtime.loadSignedSession(config) - if err != nil { - t.Fatal(err) - } - if record.SessionID != "" || record.SessionSecret != "" || record.ExpiresAt != "" { - t.Fatalf("cleared session was resurrected: %#v", record) - } -} - -func TestCancelAllActiveDownloadsDoesNotPoisonIdleItems(t *testing.T) { - activeContext, activeCancel := context.WithCancel(context.Background()) - idleContext, idleCancel := context.WithCancel(context.Background()) - t.Cleanup(idleCancel) - registry := &cancelRegistry{entries: map[string]*cancelEntry{ - "active": {ctx: activeContext, cancel: activeCancel, refs: 1}, - "idle": {ctx: idleContext, cancel: idleCancel, refs: 0}, - }} - - ids := registry.requestCancelActive() - if len(ids) != 1 || ids[0] != "active" { - t.Fatalf("cancelled IDs = %v, want [active]", ids) - } - if !errors.Is(activeContext.Err(), context.Canceled) { - t.Fatalf("active context error = %v, want cancellation", activeContext.Err()) - } - if idleContext.Err() != nil || registry.entries["idle"].canceled { - t.Fatal("idle entry was poisoned by active cancellation") - } -} diff --git a/go_backend/extension_manager.go b/go_backend/extension_manager.go deleted file mode 100644 index 0d07a8ce..00000000 --- a/go_backend/extension_manager.go +++ /dev/null @@ -1,1063 +0,0 @@ -package gobackend - -import ( - "archive/zip" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - - "github.com/dop251/goja" -) - -type loadedExtension struct { - ID string `json:"id"` - Manifest *ExtensionManifest `json:"manifest"` - VM *goja.Runtime `json:"-"` - VMMu sync.Mutex `json:"-"` - runtime *extensionRuntime - indexProgram *goja.Program - initialized bool - Enabled bool `json:"enabled"` - Error string `json:"error,omitempty"` - DataDir string `json:"data_dir"` - SourceDir string `json:"source_dir"` - IconPath string `json:"icon_path"` - - isolatedPoolMu sync.Mutex - isolatedPool []*isolatedRuntimeHandle - - quarantineMu sync.Mutex - quarantinedRuntimes int -} - -type isolatedRuntimeHandle struct { - vm *goja.Runtime - runtime *extensionRuntime -} - -func getExtensionInitSettings(extensionID string) map[string]any { - settings := GetExtensionSettingsStore().GetAll(extensionID) - if len(settings) == 0 { - return settings - } - - filtered := make(map[string]any, len(settings)) - for key, value := range settings { - if strings.HasPrefix(key, "_") { - continue - } - filtered[key] = value - } - return filtered -} - -func ensureRuntimeReadyLocked(ext *loadedExtension, applyStoredSettings bool) error { - if hasQuarantinedRuntime(ext) { - err := fmt.Errorf("extension runtime is still stopping after an unresponsive request") - ext.Error = err.Error() - return err - } - // Gate enabling too, so a package installed with a failed gate cannot be - // switched on anyway. - if err := validateManifestGates(ext.Manifest); err != nil { - ext.Error = err.Error() - ext.Enabled = false - return err - } - if ext.VM == nil || ext.runtime == nil { - if err := initializeVMLocked(ext); err != nil { - ext.Error = err.Error() - ext.Enabled = false - return err - } - } - - if applyStoredSettings && !ext.initialized { - settings := getExtensionInitSettings(ext.ID) - if len(settings) > 0 { - if err := initializeExtensionWithSettingsLocked(ext, settings); err != nil { - teardownVMLocked(ext) - ext.Error = err.Error() - ext.Enabled = false - return err - } - } else { - ext.initialized = true - } - } - - ext.Error = "" - return nil -} - -func (ext *loadedExtension) ensureRuntimeReady() error { - ext.VMMu.Lock() - defer ext.VMMu.Unlock() - - return ensureRuntimeReadyLocked(ext, true) -} - -func (ext *loadedExtension) lockReadyVM() (*goja.Runtime, error) { - ext.VMMu.Lock() - if err := ensureRuntimeReadyLocked(ext, true); err != nil { - ext.VMMu.Unlock() - return nil, err - } - return ext.VM, nil -} - -type extensionManager struct { - mu sync.RWMutex - // mutationMu serializes install/upgrade/remove (heavy FS + goja VM - // teardown/reload), which are not safe to run concurrently. Acquired before - // m.mu; "*Locked" helpers assume it is held. - mutationMu sync.Mutex - extensions map[string]*loadedExtension - extensionsDir string - dataDir string -} - -var ( - globalExtManager *extensionManager - globalExtManagerOnce sync.Once -) - -func getExtensionManager() *extensionManager { - globalExtManagerOnce.Do(func() { - globalExtManager = &extensionManager{ - extensions: make(map[string]*loadedExtension), - } - }) - return globalExtManager -} - -func (m *extensionManager) SetDirectories(extensionsDir, dataDir string) error { - m.mu.Lock() - defer m.mu.Unlock() - - m.extensionsDir = extensionsDir - m.dataDir = dataDir - - if err := os.MkdirAll(extensionsDir, 0755); err != nil { - return fmt.Errorf("failed to create extensions directory: %w", err) - } - if err := os.MkdirAll(dataDir, 0755); err != nil { - return fmt.Errorf("failed to create data directory: %w", err) - } - - return nil -} - -func (m *extensionManager) LoadExtensionFromFile(filePath string) (*loadedExtension, error) { - m.mutationMu.Lock() - defer m.mutationMu.Unlock() - return m.loadExtensionFromFileLocked(filePath) -} - -func (m *extensionManager) loadExtensionFromFileLocked(filePath string) (*loadedExtension, error) { - if !isExtensionPackagePath(filePath) { - return nil, fmt.Errorf("invalid file format: please select a .spotiflac-ext or .sflx file") - } - - zipReader, err := zip.OpenReader(filePath) - if err != nil { - return nil, fmt.Errorf("cannot open extension file: the file may be corrupted or not a valid extension package") - } - defer zipReader.Close() - - manifest, err := inspectExtensionPackage(zipReader.File) - if err != nil { - return nil, err - } - - m.mu.RLock() - existing, exists := m.extensions[manifest.Name] - var existingVersion string - var existingDisplayName string - if exists { - existingVersion = existing.Manifest.Version - existingDisplayName = existing.Manifest.DisplayName - } - m.mu.RUnlock() - - if exists { - versionCompare := compareVersions(manifest.Version, existingVersion) - if versionCompare > 0 { - return m.upgradeExtensionLocked(filePath) - } else if versionCompare == 0 { - return nil, fmt.Errorf("extension '%s' v%s is already installed", existingDisplayName, existingVersion) - } else { - return nil, fmt.Errorf("cannot downgrade '%s' from v%s to v%s", existingDisplayName, existingVersion, manifest.Version) - } - } - - m.mu.Lock() - if _, exists := m.extensions[manifest.Name]; exists { - m.mu.Unlock() - return nil, fmt.Errorf("extension '%s' was installed by another process", manifest.DisplayName) - } - - extensionsDir := m.extensionsDir - dataDir := m.dataDir - extDir, err := managedExtensionPath(extensionsDir, manifest.Name) - if err != nil { - m.mu.Unlock() - return nil, err - } - if _, err := os.Lstat(extDir); err == nil { - m.mu.Unlock() - return nil, fmt.Errorf("extension directory already exists for %q", manifest.Name) - } else if !os.IsNotExist(err) { - m.mu.Unlock() - return nil, fmt.Errorf("failed to inspect extension directory: %w", err) - } - m.mu.Unlock() - - stagingDir, err := os.MkdirTemp(extensionsDir, "."+manifest.Name+"-install-*") - if err != nil { - return nil, fmt.Errorf("failed to create extension staging directory: %w", err) - } - stagingCommitted := false - defer func() { - if !stagingCommitted { - _ = os.RemoveAll(stagingDir) - } - }() - if err := extractExtensionArchive(zipReader, stagingDir); err != nil { - return nil, err - } - - extDataDir, err := managedExtensionPath(dataDir, manifest.Name) - if err != nil { - return nil, err - } - if err := os.MkdirAll(extDataDir, 0755); err != nil { - return nil, fmt.Errorf("failed to create extension data directory: %w", err) - } - - ext := &loadedExtension{ - ID: manifest.Name, - Manifest: manifest, - Enabled: false, // New extensions start disabled - DataDir: extDataDir, - SourceDir: stagingDir, - } - - if err := validateExtensionLoad(ext); err != nil { - ext.Error = err.Error() - ext.Enabled = false - GoLog("[Extension] Failed to validate extension %s: %v\n", manifest.Name, err) - } - if err := os.Rename(stagingDir, extDir); err != nil { - return nil, fmt.Errorf("failed to activate extension: %w", err) - } - stagingCommitted = true - ext.SourceDir = extDir - - m.mu.Lock() - if _, exists := m.extensions[manifest.Name]; exists { - m.mu.Unlock() - teardownExtension(ext) - _ = os.RemoveAll(extDir) - return nil, fmt.Errorf("extension '%s' was installed by another process", manifest.DisplayName) - } - m.extensions[manifest.Name] = ext - m.mu.Unlock() - GoLog("[Extension] Loaded extension: %s v%s\n", manifest.DisplayName, manifest.Version) - - return ext, nil -} - -var supportedRuntimeFeatures = map[string]int{ - "signedSession": 3, - "sessionRefresh": 1, - "sessionGrant": 1, - "globalAction": 1, - "webviewAuth": 1, - "downloadSegments": 1, - "patternedFileTransform": 1, - "preparedContext": 1, -} - -// validateManifestGates enforces minAppVersion and requiredRuntimeFeatures -// on every load path (.sflx install, upgrade, directory load); the Store UI -// check alone never covered manual installs. An empty app version (tests, -// dev harnesses) skips the version gate. -func validateManifestGates(manifest *ExtensionManifest) error { - if manifest == nil { - return nil - } - minVersion := strings.TrimSpace(manifest.MinAppVersion) - appVersion := strings.TrimSpace(GetAppVersion()) - if minVersion != "" && appVersion != "" && compareVersions(appVersion, minVersion) < 0 { - return fmt.Errorf("requires app %s or later (installed: %s)", minVersion, appVersion) - } - for _, raw := range manifest.RequiredRuntimeFeatures { - name := strings.TrimSpace(raw) - if name == "" { - continue - } - wantVersion := 1 - if at := strings.LastIndex(name, "@"); at > 0 { - if v, err := strconv.Atoi(name[at+1:]); err == nil && v > 0 { - wantVersion = v - } - name = name[:at] - } - have, ok := supportedRuntimeFeatures[name] - if !ok { - return fmt.Errorf("requires runtime feature %q this app build does not provide", name) - } - if have < wantVersion { - return fmt.Errorf("requires runtime feature %s@%d (app provides @%d)", name, wantVersion, have) - } - } - return nil -} - -func validateExtensionLoad(ext *loadedExtension) error { - if err := validateManifestGates(ext.Manifest); err != nil { - return err - } - - ext.VMMu.Lock() - defer ext.VMMu.Unlock() - - if err := initializeVMLocked(ext); err != nil { - return err - } - teardownVMLocked(ext) - return nil -} - -func teardownExtension(ext *loadedExtension) { - if ext == nil { - return - } - ext.Enabled = false - ext.VMMu.Lock() - teardownVMLocked(ext) - ext.VMMu.Unlock() -} - -func (m *extensionManager) UnloadExtension(extensionID string) error { - m.mu.Lock() - ext, exists := m.extensions[extensionID] - if !exists { - m.mu.Unlock() - return fmt.Errorf("extension not found") - } - - ext.Enabled = false - // Remove the extension from the manager before running user cleanup. New - // operations can no longer acquire it, while existing operations keep their - // per-extension VMMu lease and are allowed to finish before teardown. - delete(m.extensions, extensionID) - m.mu.Unlock() - - ext.VMMu.Lock() - teardownVMLocked(ext) - ext.VMMu.Unlock() - - GoLog("[Extension] Unloaded extension: %s\n", extensionID) - - return nil -} - -func (m *extensionManager) GetExtension(extensionID string) (*loadedExtension, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - ext, exists := m.extensions[extensionID] - if !exists { - return nil, fmt.Errorf("extension not found") - } - return ext, nil -} - -func (m *extensionManager) GetAllExtensions() []*loadedExtension { - m.mu.RLock() - defer m.mu.RUnlock() - - result := make([]*loadedExtension, 0, len(m.extensions)) - for _, ext := range m.extensions { - result = append(result, ext) - } - return result -} - -func (m *extensionManager) SetExtensionEnabled(extensionID string, enabled bool) error { - m.mu.Lock() - ext, exists := m.extensions[extensionID] - if !exists { - m.mu.Unlock() - return fmt.Errorf("extension not found") - } - - if enabled { - ext.Enabled = true - } else { - ext.Enabled = false - ext.Error = "" - } - m.mu.Unlock() - - if enabled { - ext.VMMu.Lock() - if !m.isManagedExtensionEnabled(extensionID, ext) { - ext.VMMu.Unlock() - return fmt.Errorf("extension is no longer installed") - } - err := ensureRuntimeReadyLocked(ext, true) - ext.VMMu.Unlock() - if err != nil { - m.mu.Lock() - if m.extensions[extensionID] == ext { - ext.Enabled = false - } - m.mu.Unlock() - store := GetExtensionSettingsStore() - _ = store.Set(extensionID, "_enabled", false) - return err - } - } else { - ext.VMMu.Lock() - teardownVMLocked(ext) - ext.VMMu.Unlock() - } - GoLog("[Extension] %s %s\n", extensionID, map[bool]string{true: "enabled", false: "disabled"}[enabled]) - - store := GetExtensionSettingsStore() - if err := store.Set(extensionID, "_enabled", enabled); err != nil { - GoLog("[Extension] Failed to persist enabled state for %s: %v\n", extensionID, err) - } - - return nil -} - -// isManagedExtensionEnabled validates an operation lease after it has acquired -// the per-extension VM lock. The manager lock is deliberately not held while -// lifecycle/user JavaScript runs, so list/unload operations remain responsive. -func (m *extensionManager) isManagedExtensionEnabled(extensionID string, ext *loadedExtension) bool { - m.mu.RLock() - defer m.mu.RUnlock() - return m.extensions[extensionID] == ext && ext.Enabled -} - -func (m *extensionManager) isManagedExtension(extensionID string, ext *loadedExtension) bool { - m.mu.RLock() - defer m.mu.RUnlock() - return m.extensions[extensionID] == ext -} - -func (m *extensionManager) LoadExtensionsFromDirectory(dirPath string) ([]string, []error) { - var loaded []string - var errors []error - - entries, err := os.ReadDir(dirPath) - if err != nil { - if os.IsNotExist(err) { - return loaded, errors - } - return nil, []error{fmt.Errorf("failed to read extensions directory: %w", err)} - } - - for _, entry := range entries { - if entry.IsDir() { - manifestPath := filepath.Join(dirPath, entry.Name(), "manifest.json") - if _, err := os.Stat(manifestPath); err == nil { - ext, err := m.loadExtensionFromDirectory(filepath.Join(dirPath, entry.Name())) - if err != nil { - GoLog("[Extension] Failed to load %s: %v\n", entry.Name(), err) - errors = append(errors, fmt.Errorf("%s: %w", entry.Name(), err)) - } else { - loaded = append(loaded, ext.ID) - } - } - } else if isExtensionPackagePath(entry.Name()) { - ext, err := m.LoadExtensionFromFile(filepath.Join(dirPath, entry.Name())) - if err != nil { - GoLog("[Extension] Failed to load %s: %v\n", entry.Name(), err) - errors = append(errors, fmt.Errorf("%s: %w", entry.Name(), err)) - } else { - loaded = append(loaded, ext.ID) - } - } - } - - return loaded, errors -} - -func (m *extensionManager) loadExtensionFromDirectory(dirPath string) (*loadedExtension, error) { - m.mu.Lock() - locked := true - defer func() { - if locked { - m.mu.Unlock() - } - }() - - manifestPath := filepath.Join(dirPath, "manifest.json") - manifestData, err := os.ReadFile(manifestPath) - if err != nil { - return nil, fmt.Errorf("failed to read manifest.json: %w", err) - } - - manifest, err := ParseManifest(manifestData) - if err != nil { - return nil, fmt.Errorf("invalid extension manifest: %w", err) - } - - indexPath := filepath.Join(dirPath, "index.js") - if _, err := os.Stat(indexPath); os.IsNotExist(err) { - return nil, fmt.Errorf("extension is missing index.js file") - } - - if existing, exists := m.extensions[manifest.Name]; exists { - GoLog("[Extension] Extension '%s' already loaded, skipping\n", manifest.DisplayName) - return existing, nil - } - - expectedSourceDir, err := managedExtensionPath(m.extensionsDir, manifest.Name) - if err != nil || filepath.Clean(dirPath) != filepath.Clean(expectedSourceDir) { - return nil, fmt.Errorf("extension directory name must match manifest name %q", manifest.Name) - } - extDataDir, err := managedExtensionPath(m.dataDir, manifest.Name) - if err != nil { - return nil, err - } - if err := os.MkdirAll(extDataDir, 0755); err != nil { - return nil, fmt.Errorf("failed to create extension data directory: %w", err) - } - - ext := &loadedExtension{ - ID: manifest.Name, - Manifest: manifest, - Enabled: false, // Will be restored from settings store - DataDir: extDataDir, - SourceDir: dirPath, - } - m.mu.Unlock() - locked = false - - store := GetExtensionSettingsStore() - if enabledVal, err := store.Get(manifest.Name, "_enabled"); err == nil { - if enabled, ok := enabledVal.(bool); ok { - ext.Enabled = enabled - GoLog("[Extension] Restored enabled state for %s: %v\n", manifest.Name, enabled) - } - } - - if err := validateExtensionLoad(ext); err != nil { - ext.Error = err.Error() - ext.Enabled = false - GoLog("[Extension] Failed to validate extension %s: %v\n", manifest.Name, err) - } - - m.mu.Lock() - locked = true - if _, exists := m.extensions[manifest.Name]; exists { - m.mu.Unlock() - locked = false - teardownExtension(ext) - return nil, fmt.Errorf("extension '%s' was installed by another process", manifest.DisplayName) - } - m.extensions[manifest.Name] = ext - m.mu.Unlock() - locked = false - GoLog("[Extension] Loaded extension: %s v%s\n", manifest.DisplayName, manifest.Version) - - return ext, nil -} - -func (m *extensionManager) RemoveExtension(extensionID string) error { - m.mutationMu.Lock() - defer m.mutationMu.Unlock() - - ext, err := m.GetExtension(extensionID) - if err != nil { - return err - } - - sourceDir, err := managedExtensionPath(m.extensionsDir, ext.ID) - if err != nil || !isPathWithinBase(m.extensionsDir, ext.SourceDir) || filepath.Clean(ext.SourceDir) != filepath.Clean(sourceDir) { - return fmt.Errorf("refusing to remove extension outside the managed source directory") - } - dataDir, err := managedExtensionPath(m.dataDir, ext.ID) - if err != nil || !isPathWithinBase(m.dataDir, ext.DataDir) || filepath.Clean(ext.DataDir) != filepath.Clean(dataDir) { - return fmt.Errorf("refusing to remove extension outside the managed data directory") - } - - if err := m.UnloadExtension(extensionID); err != nil { - return err - } - - if err := os.RemoveAll(sourceDir); err != nil { - GoLog("[Extension] Warning: failed to remove source dir: %v\n", err) - } - - // Uninstall means gone: storage.json and encrypted credentials must not - // linger on disk after the extension is removed. - if err := os.RemoveAll(dataDir); err != nil { - GoLog("[Extension] Warning: failed to remove data dir: %v\n", err) - } - - return nil -} - -// Only allows upgrades (new version > current version), not downgrades -func (m *extensionManager) UpgradeExtension(filePath string) (*loadedExtension, error) { - m.mutationMu.Lock() - defer m.mutationMu.Unlock() - return m.upgradeExtensionLocked(filePath) -} - -func (m *extensionManager) upgradeExtensionLocked(filePath string) (*loadedExtension, error) { - if !isExtensionPackagePath(filePath) { - return nil, fmt.Errorf("invalid file format: please select a .spotiflac-ext or .sflx file") - } - - zipReader, err := zip.OpenReader(filePath) - if err != nil { - return nil, fmt.Errorf("cannot open extension file: the file may be corrupted or not a valid extension package") - } - defer zipReader.Close() - - newManifest, err := inspectExtensionPackage(zipReader.File) - if err != nil { - return nil, err - } - - m.mu.RLock() - existing, exists := m.extensions[newManifest.Name] - m.mu.RUnlock() - - if !exists { - return nil, fmt.Errorf("extension '%s' is not installed; use install instead of upgrade", newManifest.DisplayName) - } - - versionCompare := compareVersions(newManifest.Version, existing.Manifest.Version) - if versionCompare < 0 { - return nil, fmt.Errorf("cannot downgrade extension: current version: %s, new version: %s", existing.Manifest.Version, newManifest.Version) - } - if versionCompare == 0 { - return nil, fmt.Errorf("extension is already at version %s", existing.Manifest.Version) - } - - GoLog("[Extension] Upgrading %s from v%s to v%s\n", newManifest.DisplayName, existing.Manifest.Version, newManifest.Version) - - extDataDir, err := managedExtensionPath(m.dataDir, newManifest.Name) - if err != nil || filepath.Clean(existing.DataDir) != filepath.Clean(extDataDir) { - return nil, fmt.Errorf("installed extension has an invalid data directory") - } - extDir, err := managedExtensionPath(m.extensionsDir, newManifest.Name) - if err != nil || filepath.Clean(existing.SourceDir) != filepath.Clean(extDir) { - return nil, fmt.Errorf("installed extension has an invalid source directory") - } - wasEnabled := existing.Enabled - - stagingDir, err := os.MkdirTemp(m.extensionsDir, "."+newManifest.Name+"-upgrade-*") - if err != nil { - return nil, fmt.Errorf("failed to create upgrade staging directory: %w", err) - } - stagingActive := true - defer func() { - if stagingActive { - _ = os.RemoveAll(stagingDir) - } - }() - if err := extractExtensionArchive(zipReader, stagingDir); err != nil { - return nil, err - } - - ext := &loadedExtension{ - ID: newManifest.Name, - Manifest: newManifest, - Enabled: wasEnabled, // Preserve enabled state from before upgrade - DataDir: extDataDir, - SourceDir: stagingDir, - } - - if wasEnabled { - if err := ext.ensureRuntimeReady(); err != nil { - return nil, fmt.Errorf("upgraded extension failed validation: %w", err) - } - } else if err := validateExtensionLoad(ext); err != nil { - return nil, fmt.Errorf("upgraded extension failed validation: %w", err) - } - - backupDir, err := os.MkdirTemp(m.extensionsDir, "."+newManifest.Name+"-backup-*") - if err != nil { - teardownExtension(ext) - return nil, fmt.Errorf("failed to prepare upgrade backup: %w", err) - } - if err := os.Remove(backupDir); err != nil { - teardownExtension(ext) - return nil, fmt.Errorf("failed to prepare upgrade backup: %w", err) - } - if err := os.Rename(extDir, backupDir); err != nil { - teardownExtension(ext) - return nil, fmt.Errorf("failed to preserve current extension: %w", err) - } - if err := os.Rename(stagingDir, extDir); err != nil { - _ = os.Rename(backupDir, extDir) - teardownExtension(ext) - return nil, fmt.Errorf("failed to activate upgraded extension: %w", err) - } - stagingActive = false - ext.SourceDir = extDir - - existing.Enabled = false - if err := m.UnloadExtension(existing.ID); err != nil { - _ = os.RemoveAll(extDir) - _ = os.Rename(backupDir, extDir) - existing.Enabled = wasEnabled - teardownExtension(ext) - return nil, fmt.Errorf("failed to unload current extension: %w", err) - } - - m.mu.Lock() - m.extensions[newManifest.Name] = ext - m.mu.Unlock() - if err := os.RemoveAll(backupDir); err != nil { - GoLog("[Extension] Warning: failed to remove upgrade backup: %v\n", err) - } - - GoLog("[Extension] Upgraded extension: %s to v%s\n", newManifest.DisplayName, newManifest.Version) - - return ext, nil -} - -type ExtensionUpgradeInfo struct { - ExtensionID string `json:"extension_id"` - CurrentVersion string `json:"current_version"` - NewVersion string `json:"new_version"` - CanUpgrade bool `json:"can_upgrade"` - IsInstalled bool `json:"is_installed"` -} - -func (m *extensionManager) checkExtensionUpgradeInternal(filePath string) (*ExtensionUpgradeInfo, error) { - if !isExtensionPackagePath(filePath) { - return nil, fmt.Errorf("invalid file format: please select a .spotiflac-ext or .sflx file") - } - - zipReader, err := zip.OpenReader(filePath) - - if err != nil { - return nil, fmt.Errorf("cannot open extension file") - } - defer zipReader.Close() - - newManifest, err := inspectExtensionPackage(zipReader.File) - if err != nil { - return nil, err - } - - m.mu.RLock() - existing, exists := m.extensions[newManifest.Name] - m.mu.RUnlock() - - info := &ExtensionUpgradeInfo{ - ExtensionID: newManifest.Name, - NewVersion: newManifest.Version, - IsInstalled: exists, - } - - if !exists { - info.CurrentVersion = "" - info.CanUpgrade = false - } else { - info.CurrentVersion = existing.Manifest.Version - info.CanUpgrade = compareVersions(newManifest.Version, existing.Manifest.Version) > 0 - } - - return info, nil -} - -func (m *extensionManager) CheckExtensionUpgradeJSON(filePath string) (string, error) { - info, err := m.checkExtensionUpgradeInternal(filePath) - if err != nil { - return "", err - } - - jsonBytes, err := json.Marshal(info) - if err != nil { - return "", err - } - - return string(jsonBytes), nil -} - -func (m *extensionManager) GetInstalledExtensionsJSON() (string, error) { - extensions := m.GetAllExtensions() - - type ExtensionInfo struct { - ID string `json:"id"` - Name string `json:"name"` - DisplayName string `json:"display_name"` - Version string `json:"version"` - Description string `json:"description"` - Homepage string `json:"homepage,omitempty"` - IconPath string `json:"icon_path,omitempty"` - Types []ExtensionType `json:"types"` - Enabled bool `json:"enabled"` - Status string `json:"status"` - Error string `json:"error_message,omitempty"` - Settings []ExtensionSetting `json:"settings,omitempty"` - QualityOptions []QualityOption `json:"quality_options,omitempty"` - Permissions []string `json:"permissions"` - HasMetadataProvider bool `json:"has_metadata_provider"` - HasDownloadProvider bool `json:"has_download_provider"` - HasLyricsProvider bool `json:"has_lyrics_provider"` - SkipMetadataEnrichment bool `json:"skip_metadata_enrichment"` - SkipLyrics bool `json:"skip_lyrics"` - StopProviderFallback bool `json:"stop_provider_fallback"` - SearchBehavior *SearchBehaviorConfig `json:"search_behavior,omitempty"` - TrackMatching *TrackMatchingConfig `json:"track_matching,omitempty"` - PostProcessing *PostProcessingConfig `json:"post_processing,omitempty"` - ServiceHealth []ExtensionHealthCheck `json:"service_health,omitempty"` - Capabilities map[string]any `json:"capabilities,omitempty"` - } - - infos := make([]ExtensionInfo, len(extensions)) - for i, ext := range extensions { - permissions := []string{} - for _, domain := range ext.Manifest.Permissions.Network { - permissions = append(permissions, "network:"+domain) - } - if ext.Manifest.Permissions.Storage { - permissions = append(permissions, "storage:enabled") - } - if ext.Manifest.Permissions.File { - permissions = append(permissions, "file:enabled") - } - if ext.Manifest.Permissions.AllowHTTP { - permissions = append(permissions, "network:http") - } - if ext.Manifest.HasCapability("rawFfmpeg") { - permissions = append(permissions, "ffmpeg:raw") - } - - status := "loaded" - if ext.Error != "" { - status = "error" - } else if !ext.Enabled { - status = "disabled" - } - - iconPath := "" - if ext.Manifest.Icon != "" && ext.SourceDir != "" { - possibleIcon, safe := safeExtensionAssetPath(ext.SourceDir, ext.Manifest.Icon) - if safe { - if _, err := os.Stat(possibleIcon); err == nil { - iconPath = possibleIcon - } - } - } - if iconPath == "" && ext.SourceDir != "" { - possibleIcon, safe := safeExtensionAssetPath(ext.SourceDir, "icon.png") - if safe { - if _, err := os.Stat(possibleIcon); err == nil { - iconPath = possibleIcon - } - } - } - - infos[i] = ExtensionInfo{ - ID: ext.ID, - Name: ext.Manifest.Name, - DisplayName: ext.Manifest.DisplayName, - Version: ext.Manifest.Version, - Description: ext.Manifest.Description, - Homepage: ext.Manifest.Homepage, - IconPath: iconPath, - Types: ext.Manifest.Types, - Enabled: ext.Enabled, - Status: status, - Error: ext.Error, - Settings: ext.Manifest.Settings, - QualityOptions: ext.Manifest.QualityOptions, - Permissions: permissions, - HasMetadataProvider: ext.Manifest.IsMetadataProvider(), - HasDownloadProvider: ext.Manifest.IsDownloadProvider(), - HasLyricsProvider: ext.Manifest.IsLyricsProvider(), - SkipMetadataEnrichment: ext.Manifest.SkipMetadataEnrichment, - SkipLyrics: ext.Manifest.SkipLyrics, - StopProviderFallback: ext.Manifest.StopsProviderFallback(), - SearchBehavior: ext.Manifest.SearchBehavior, - TrackMatching: ext.Manifest.TrackMatching, - PostProcessing: ext.Manifest.PostProcessing, - ServiceHealth: ext.Manifest.ServiceHealth, - Capabilities: ext.Manifest.Capabilities, - } - } - - jsonBytes, err := json.Marshal(infos) - if err != nil { - return "", err - } - - return string(jsonBytes), nil -} - -func (m *extensionManager) InitializeExtension(extensionID string, settings map[string]any) error { - m.mu.RLock() - ext, exists := m.extensions[extensionID] - m.mu.RUnlock() - if !exists { - return fmt.Errorf("extension not found") - } - - ext.VMMu.Lock() - defer ext.VMMu.Unlock() - if !m.isManagedExtension(extensionID, ext) { - return fmt.Errorf("extension is no longer installed") - } - - if err := ensureRuntimeReadyLocked(ext, false); err != nil { - return err - } - return initializeExtensionWithSettingsLocked(ext, settings) -} - -func (m *extensionManager) CleanupExtension(extensionID string) error { - m.mu.RLock() - ext, exists := m.extensions[extensionID] - m.mu.RUnlock() - if !exists { - return fmt.Errorf("extension not found") - } - - ext.VMMu.Lock() - defer ext.VMMu.Unlock() - if !m.isManagedExtension(extensionID, ext) { - return fmt.Errorf("extension is no longer installed") - } - if ext.VM == nil { - return nil - } - if err := runCleanupLocked(ext); err != nil { - if IsRuntimeUnsafeError(err) { - quarantineRuntimeLocked(ext, ext.VM, err) - } - GoLog("[Extension] Cleanup error for %s: %v\n", extensionID, err) - return err - } - GoLog("[Extension] Cleaned up %s\n", extensionID) - return nil -} - -func (m *extensionManager) UnloadAllExtensions() { - m.mu.Lock() - extensionIDs := make([]string, 0, len(m.extensions)) - for id := range m.extensions { - extensionIDs = append(extensionIDs, id) - } - m.mu.Unlock() - - for _, id := range extensionIDs { - m.UnloadExtension(id) - } - - GoLog("[Extension] All extensions unloaded\n") -} - -func (m *extensionManager) InvokeAction(extensionID string, actionName string) (map[string]any, error) { - m.mu.RLock() - ext, exists := m.extensions[extensionID] - enabled := exists && ext.Enabled - m.mu.RUnlock() - if !exists { - return nil, fmt.Errorf("extension not found: %s", extensionID) - } - - if !enabled { - return nil, fmt.Errorf("extension is disabled") - } - ext.VMMu.Lock() - if !m.isManagedExtensionEnabled(extensionID, ext) { - ext.VMMu.Unlock() - return nil, fmt.Errorf("extension is disabled or no longer installed") - } - if err := ensureRuntimeReadyLocked(ext, true); err != nil { - ext.VMMu.Unlock() - return nil, err - } - vm := ext.VM - defer ext.VMMu.Unlock() - - // Merge extension return values onto the top-level JSON object so Flutter can read - // message, open_auth_url, setting_updates without unwrapping a nested "result" key. - actionNameLiteral := strconv.Quote(actionName) - script := fmt.Sprintf(` - (function() { - var actionName = %s; - function runAction(fn) { - try { - var result = fn(); - if (result && typeof result.then === 'function') { - return { success: true, pending: true, message: 'Action started' }; - } - if (result !== null && result !== undefined && typeof result === 'object') { - var isArr = false; - if (typeof Array !== 'undefined' && Array.isArray) { - isArr = Array.isArray(result); - } - if (!isArr) { - var out = { success: true }; - for (var k in result) { - out[k] = result[k]; - } - return out; - } - } - return { success: true, result: result }; - } catch (e) { - return { success: false, error: e.toString() }; - } - } - if (typeof extension !== 'undefined' && extension && typeof extension[actionName] === 'function') { - return runAction(function() { return extension[actionName](); }); - } - if (actionName === 'completeGrant' && typeof session !== 'undefined' && session && typeof session.completeGrant === 'function') { - return runAction(function() { return session.completeGrant(); }); - } - return { success: false, error: 'Action function not found: ' + actionName }; - })() - `, actionNameLiteral) - - result, err := RunWithTimeoutAndRecover(vm, script, DefaultJSTimeout) - if err != nil { - if IsRuntimeUnsafeError(err) { - quarantineRuntimeLocked(ext, vm, err) - } - GoLog("[Extension] InvokeAction error for %s.%s: %v\n", extensionID, actionName, err) - return nil, fmt.Errorf("action failed: %v", err) - } - - if result == nil || goja.IsUndefined(result) { - return map[string]any{"success": true}, nil - } - - exported := result.Export() - if resultMap, ok := exported.(map[string]any); ok { - status := "unspecified" - if success, present := resultMap["success"].(bool); present { - status = strconv.FormatBool(success) - } - GoLog( - "[Extension] InvokeAction %s.%s completed (success=%s, fields=%d)\n", - extensionID, - actionName, - status, - len(resultMap), - ) - return resultMap, nil - } - - return map[string]any{"success": true, "result": exported}, nil -} diff --git a/go_backend/extension_manager_package.go b/go_backend/extension_manager_package.go deleted file mode 100644 index 501da06f..00000000 --- a/go_backend/extension_manager_package.go +++ /dev/null @@ -1,212 +0,0 @@ -package gobackend - -import ( - "archive/zip" - "fmt" - "io" - "os" - "path" - "path/filepath" - "strconv" - "strings" -) - -func compareVersions(v1, v2 string) int { - parts1 := strings.Split(strings.TrimPrefix(v1, "v"), ".") - parts2 := strings.Split(strings.TrimPrefix(v2, "v"), ".") - - maxLen := len(parts1) - if len(parts2) > maxLen { - maxLen = len(parts2) - } - - for i := 0; i < maxLen; i++ { - var n1, n2 int - if i < len(parts1) { - n1, _ = strconv.Atoi(parts1[i]) - } - if i < len(parts2) { - n2, _ = strconv.Atoi(parts2[i]) - } - - if n1 < n2 { - return -1 - } - if n1 > n2 { - return 1 - } - } - - return 0 -} - -func isExtensionPackagePath(filePath string) bool { - lowerPath := strings.ToLower(filePath) - return strings.HasSuffix(lowerPath, ".spotiflac-ext") || strings.HasSuffix(lowerPath, ".sflx") -} - -func managedExtensionPath(root, extensionID string) (string, error) { - if root == "" { - return "", fmt.Errorf("extension directory is not configured") - } - if !extensionIDPattern.MatchString(extensionID) { - return "", fmt.Errorf("invalid extension ID %q", extensionID) - } - fullPath := filepath.Join(root, extensionID) - if !isPathWithinBase(root, fullPath) { - return "", fmt.Errorf("extension path escapes its managed directory") - } - return fullPath, nil -} - -func safeExtensionAssetPath(root, assetPath string) (string, bool) { - if root == "" || assetPath == "" || filepath.IsAbs(assetPath) || strings.Contains(assetPath, `\`) { - return "", false - } - cleaned := path.Clean(assetPath) - if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { - return "", false - } - fullPath := filepath.Join(root, filepath.FromSlash(cleaned)) - return fullPath, isPathWithinBase(root, fullPath) -} - -const ( - maxExtensionArchiveEntries = 2048 - maxExtensionArchiveUncompressedBytes = 256 * 1024 * 1024 - maxExtensionManifestBytes = 1024 * 1024 -) - -func validateExtensionArchive(files []*zip.File) error { - if len(files) > maxExtensionArchiveEntries { - return fmt.Errorf( - "extension archive contains too many entries (maximum %d)", - maxExtensionArchiveEntries, - ) - } - - seenPaths := make(map[string]struct{}, len(files)) - var totalUncompressed uint64 - for _, file := range files { - if file.FileInfo().Mode()&os.ModeSymlink != 0 || strings.Contains(file.Name, `\`) { - return fmt.Errorf("unsafe path in extension archive: %s", file.Name) - } - - relPath := path.Clean(file.Name) - if relPath == "." || relPath == ".." || strings.HasPrefix(relPath, "../") || path.IsAbs(relPath) { - return fmt.Errorf("unsafe path in extension archive: %s", file.Name) - } - pathKey := strings.ToLower(relPath) - if _, exists := seenPaths[pathKey]; exists { - return fmt.Errorf("duplicate path in extension archive: %s", file.Name) - } - seenPaths[pathKey] = struct{}{} - - if file.FileInfo().IsDir() { - continue - } - if file.UncompressedSize64 > maxExtensionArchiveUncompressedBytes-totalUncompressed { - return fmt.Errorf( - "extension archive exceeds the %d MiB extracted size limit", - maxExtensionArchiveUncompressedBytes/(1024*1024), - ) - } - totalUncompressed += file.UncompressedSize64 - } - return nil -} - -func inspectExtensionPackage(files []*zip.File) (*ExtensionManifest, error) { - if err := validateExtensionArchive(files); err != nil { - return nil, err - } - - var manifestFile *zip.File - hasIndexJS := false - for _, file := range files { - switch path.Clean(file.Name) { - case "manifest.json": - manifestFile = file - case "index.js": - hasIndexJS = !file.FileInfo().IsDir() - } - } - - if manifestFile == nil || manifestFile.FileInfo().IsDir() { - return nil, fmt.Errorf("invalid extension package: root manifest.json not found") - } - if !hasIndexJS { - return nil, fmt.Errorf("invalid extension package: root index.js not found") - } - if manifestFile.UncompressedSize64 > maxExtensionManifestBytes { - return nil, fmt.Errorf("invalid extension package: manifest.json is too large") - } - - rc, err := manifestFile.Open() - if err != nil { - return nil, fmt.Errorf("failed to open manifest.json: %w", err) - } - manifestData, readErr := io.ReadAll(io.LimitReader(rc, maxExtensionManifestBytes+1)) - closeErr := rc.Close() - if readErr != nil { - return nil, fmt.Errorf("failed to read manifest.json: %w", readErr) - } - if closeErr != nil { - return nil, fmt.Errorf("failed to close manifest.json: %w", closeErr) - } - if len(manifestData) > maxExtensionManifestBytes { - return nil, fmt.Errorf("invalid extension package: manifest.json is too large") - } - - manifest, err := ParseManifest(manifestData) - if err != nil { - return nil, fmt.Errorf("invalid extension manifest: %w", err) - } - return manifest, nil -} - -func extractExtensionArchive(zipReader *zip.ReadCloser, destination string) error { - if err := validateExtensionArchive(zipReader.File); err != nil { - return err - } - for _, file := range zipReader.File { - if file.FileInfo().IsDir() { - continue - } - if file.FileInfo().Mode()&os.ModeSymlink != 0 || strings.Contains(file.Name, `\`) { - return fmt.Errorf("unsafe path in extension archive: %s", file.Name) - } - - relPath := path.Clean(file.Name) - if relPath == "." || relPath == ".." || strings.HasPrefix(relPath, "../") || path.IsAbs(relPath) { - return fmt.Errorf("unsafe path in extension archive: %s", file.Name) - } - destPath := filepath.Join(destination, filepath.FromSlash(relPath)) - if !isPathWithinBase(destination, destPath) { - return fmt.Errorf("unsafe path in extension archive: %s", file.Name) - } - - if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { - return fmt.Errorf("failed to create extension directory: %w", err) - } - destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - return fmt.Errorf("failed to create extension file: %w", err) - } - srcFile, err := file.Open() - if err != nil { - destFile.Close() - return fmt.Errorf("failed to open file in archive: %w", err) - } - _, copyErr := io.Copy(destFile, srcFile) - closeSrcErr := srcFile.Close() - closeDestErr := destFile.Close() - if copyErr != nil { - return fmt.Errorf("failed to extract extension file: %w", copyErr) - } - if closeSrcErr != nil || closeDestErr != nil { - return fmt.Errorf("failed to close extracted extension file") - } - } - return nil -} diff --git a/go_backend/extension_manager_runtime.go b/go_backend/extension_manager_runtime.go deleted file mode 100644 index 7cfb71f9..00000000 --- a/go_backend/extension_manager_runtime.go +++ /dev/null @@ -1,523 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/dop251/goja" -) - -// extensionLifecycleTimeout bounds extension code that runs outside a normal -// provider/action request. A lifecycle callback is still arbitrary extension -// JavaScript, so it needs the same interrupt/quarantine contract as a regular -// invocation. Tests may shorten this value; production keeps the normal JS -// request budget. -var extensionLifecycleTimeout = DefaultJSTimeout - -func runExtensionLifecycleCall( - vm *goja.Runtime, - call func() (goja.Value, error), -) (goja.Value, error) { - return runGojaCallWithTimeoutAndRecover(vm, call, extensionLifecycleTimeout) -} - -// discardLifecycleRuntime releases a runtime whose lifecycle execution failed. -// An interrupted Go callback may still be using the VM after the timeout -// helper returns, so unsafe runtimes must remain quarantined until the helper's -// completion signal closes. Safe failures can be closed immediately. -func discardLifecycleRuntime( - ext *loadedExtension, - vm *goja.Runtime, - runtime *extensionRuntime, - err error, -) { - if IsRuntimeUnsafeError(err) { - if ext != nil && ext.VM == vm { - quarantineRuntimeLocked(ext, vm, err) - } else { - registerQuarantinedRuntime(ext, runtime, runtimeCompletion(err)) - } - return - } - - if runtime != nil { - runtime.closeStorageFlusher() - } - if ext != nil && ext.VM == vm { - ext.VM = nil - ext.runtime = nil - ext.indexProgram = nil - ext.initialized = false - } -} - -func initializeVMLocked(ext *loadedExtension) error { - ext.VM = nil - ext.runtime = nil - ext.indexProgram = nil - ext.initialized = false - vm := goja.New() - ext.VM = vm - - indexPath := filepath.Join(ext.SourceDir, "index.js") - jsCode, err := os.ReadFile(indexPath) - if err != nil { - ext.VM = nil - return fmt.Errorf("failed to read index.js: %w", err) - } - indexProgram, err := goja.Compile(indexPath, string(jsCode), false) - if err != nil { - ext.VM = nil - return fmt.Errorf("failed to compile extension code: %w", err) - } - ext.indexProgram = indexProgram - - runtime := newExtensionRuntime(ext) - ext.runtime = runtime - runtime.RegisterAPIs(vm) - runtime.RegisterGoBackendAPIs(vm) - - console := vm.NewObject() - console.Set("log", func(call goja.FunctionCall) goja.Value { - GoLog("[Extension:%s] %s\n", ext.ID, formatExtensionLogArgs(call.Arguments)) - return goja.Undefined() - }) - vm.Set("console", console) - - var registeredExtension goja.Value - vm.Set("registerExtension", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) > 0 { - registeredExtension = call.Arguments[0] - vm.Set("extension", call.Arguments[0]) - } - return goja.Undefined() - }) - - _, err = runExtensionLifecycleCall(vm, func() (goja.Value, error) { - return vm.RunProgram(indexProgram) - }) - if err != nil { - discardLifecycleRuntime(ext, vm, runtime, err) - return fmt.Errorf("failed to execute extension code: %w", err) - } - - if registeredExtension == nil || goja.IsUndefined(registeredExtension) { - discardLifecycleRuntime(ext, vm, runtime, fmt.Errorf("extension did not call registerExtension()")) - return fmt.Errorf("extension did not call registerExtension()") - } - - return nil -} - -func newIsolatedExtensionRuntime(ext *loadedExtension) (*goja.Runtime, *extensionRuntime, error) { - vm := goja.New() - - indexProgram := ext.indexProgram - if indexProgram == nil { - indexPath := filepath.Join(ext.SourceDir, "index.js") - jsCode, err := os.ReadFile(indexPath) - if err != nil { - return nil, nil, fmt.Errorf("failed to read index.js: %w", err) - } - indexProgram, err = goja.Compile(indexPath, string(jsCode), false) - if err != nil { - return nil, nil, fmt.Errorf("failed to compile extension code: %w", err) - } - } - - runtime := &extensionRuntime{ - extensionID: ext.ID, - manifest: ext.Manifest, - settings: make(map[string]any), - cookieJar: nil, - dataDir: ext.DataDir, - vm: vm, - } - if ext.runtime != nil && ext.runtime.cookieJar != nil { - runtime.cookieJar = ext.runtime.cookieJar - } else { - jar, _ := newSimpleCookieJar() - runtime.cookieJar = jar - } - runtime.httpClient = newExtensionHTTPClient(ext, runtime.cookieJar, extensionHTTPTimeout(ext, 30*time.Second), true) - runtime.downloadClient = newExtensionHTTPClient(ext, runtime.cookieJar, DownloadTimeout, false) - runtime.RegisterAPIs(vm) - runtime.RegisterGoBackendAPIs(vm) - - console := vm.NewObject() - console.Set("log", func(call goja.FunctionCall) goja.Value { - GoLog("[Extension:%s] %s\n", ext.ID, formatExtensionLogArgs(call.Arguments)) - return goja.Undefined() - }) - vm.Set("console", console) - - var registeredExtension goja.Value - vm.Set("registerExtension", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) > 0 { - registeredExtension = call.Arguments[0] - vm.Set("extension", call.Arguments[0]) - } - return goja.Undefined() - }) - - if _, err := runExtensionLifecycleCall(vm, func() (goja.Value, error) { - return vm.RunProgram(indexProgram) - }); err != nil { - discardLifecycleRuntime(ext, vm, runtime, err) - return nil, nil, fmt.Errorf("failed to execute extension code: %w", err) - } - - if registeredExtension == nil || goja.IsUndefined(registeredExtension) { - discardLifecycleRuntime(ext, vm, runtime, fmt.Errorf("extension did not call registerExtension()")) - return nil, nil, fmt.Errorf("extension did not call registerExtension()") - } - - settings := getExtensionInitSettings(ext.ID) - if len(settings) > 0 { - if err := initializeExtensionRuntimeWithSettings(vm, ext.ID, settings); err != nil { - discardLifecycleRuntime(ext, vm, runtime, err) - return nil, nil, err - } - } - - return vm, runtime, nil -} - -// A goja runtime plus an executed extension program is several MB of live -// heap; rebuilding one per download multiplies that by the number of tracks. -// Extensions already serve many calls on the persistent shared VM, so reusing -// an initialized isolated runtime for consecutive downloads is the same -// lifecycle contract. -const maxIdleIsolatedRuntimes = 1 - -// acquireIsolatedExtensionRuntime pops an idle pooled runtime or builds one. -func acquireIsolatedExtensionRuntime(ext *loadedExtension) (*goja.Runtime, *extensionRuntime, error) { - if hasQuarantinedRuntime(ext) { - return nil, nil, fmt.Errorf("extension runtime is still stopping after an unresponsive request") - } - ext.isolatedPoolMu.Lock() - if n := len(ext.isolatedPool); n > 0 { - handle := ext.isolatedPool[n-1] - // Reslicing alone retains this handle in the backing array. If the - // borrowed VM is retired after an error, its heap would stay alive. - ext.isolatedPool[n-1] = nil - ext.isolatedPool = ext.isolatedPool[:n-1] - ext.isolatedPoolMu.Unlock() - return handle.vm, handle.runtime, nil - } - ext.isolatedPoolMu.Unlock() - - ext.VMMu.Lock() - defer ext.VMMu.Unlock() - if hasQuarantinedRuntime(ext) { - return nil, nil, fmt.Errorf("extension runtime is still stopping after an unresponsive request") - } - return newIsolatedExtensionRuntime(ext) -} - -// releaseIsolatedExtensionRuntime pools a healthy runtime for reuse or tears -// it down. Pass healthy=false after an interrupt/timeout/script error, whose -// VM state can't be trusted for reuse. -func releaseIsolatedExtensionRuntime( - ext *loadedExtension, - vm *goja.Runtime, - runtime *extensionRuntime, - healthy, cleanupSafe bool, - unsafeDone <-chan struct{}, -) { - if !cleanupSafe { - registerQuarantinedRuntime(ext, runtime, unsafeDone) - return - } - - if runtime != nil { - if err := runtime.flushStorageNow(); err != nil { - GoLog("[Extension:%s] isolated download storage flush failed: %v\n", ext.ID, err) - } - } - - if healthy && vm != nil && runtime != nil && ext.Enabled { - ext.isolatedPoolMu.Lock() - if len(ext.isolatedPool) < maxIdleIsolatedRuntimes { - ext.isolatedPool = append(ext.isolatedPool, &isolatedRuntimeHandle{vm: vm, runtime: runtime}) - ext.isolatedPoolMu.Unlock() - return - } - ext.isolatedPoolMu.Unlock() - } - - if cleanupSafe { - cleanupErr := runCleanupOnVM(vm) - if IsRuntimeUnsafeError(cleanupErr) { - registerQuarantinedRuntime(ext, runtime, runtimeCompletion(cleanupErr)) - return - } - if cleanupErr != nil { - GoLog("[Extension:%s] isolated download cleanup failed: %v\n", ext.ID, cleanupErr) - } - } - if runtime != nil { - runtime.closeStorageFlusher() - } -} - -func hasQuarantinedRuntime(ext *loadedExtension) bool { - if ext == nil { - return false - } - ext.quarantineMu.Lock() - defer ext.quarantineMu.Unlock() - return ext.quarantinedRuntimes > 0 -} - -// registerQuarantinedRuntime keeps the extension gated until the interrupted -// goroutine actually exits. A Go goroutine cannot be killed safely, so allowing -// a replacement VM immediately would let a broken extension accumulate an -// unbounded number of runtimes. The runtime is touched only after completion. -func registerQuarantinedRuntime(ext *loadedExtension, runtime *extensionRuntime, done <-chan struct{}) { - if ext == nil { - return - } - ext.quarantineMu.Lock() - ext.quarantinedRuntimes++ - ext.quarantineMu.Unlock() - - if done == nil { - GoLog("[Extension:%s] quarantined runtime has no completion signal; keeping extension gated\n", ext.ID) - return - } - go func() { - <-done - if runtime != nil { - runtime.closeStorageFlusher() - } - ext.quarantineMu.Lock() - if ext.quarantinedRuntimes > 0 { - ext.quarantinedRuntimes-- - } - ext.quarantineMu.Unlock() - }() -} - -// quarantineRuntimeLocked detaches a VM that remained busy after interrupt. -// The caller holds VMMu. Touching or cleaning up that VM would race its stuck -// goroutine; replacement calls remain gated until that goroutine exits. -func quarantineRuntimeLocked(ext *loadedExtension, vm *goja.Runtime, err error) { - if ext == nil || ext.VM != vm { - return - } - runtime := ext.runtime - ext.VM = nil - ext.runtime = nil - ext.initialized = false - ext.Error = "extension runtime was quarantined after an unresponsive script" - registerQuarantinedRuntime(ext, runtime, runtimeCompletion(err)) -} - -// drainIsolatedRuntimePool tears down idle isolated runtimes. Called on -// extension teardown and on app-wide memory release. -func drainIsolatedRuntimePool(ext *loadedExtension) { - ext.isolatedPoolMu.Lock() - pool := ext.isolatedPool - ext.isolatedPool = nil - ext.isolatedPoolMu.Unlock() - - for _, handle := range pool { - cleanupErr := runCleanupOnVM(handle.vm) - if IsRuntimeUnsafeError(cleanupErr) { - registerQuarantinedRuntime(ext, handle.runtime, runtimeCompletion(cleanupErr)) - continue - } - if cleanupErr != nil { - GoLog("[Extension:%s] isolated pool cleanup failed: %v\n", ext.ID, cleanupErr) - } - if handle.runtime != nil { - if err := handle.runtime.flushStorageNow(); err != nil { - GoLog("[Extension:%s] isolated pool storage flush failed: %v\n", ext.ID, err) - } - handle.runtime.closeStorageFlusher() - } - } -} - -// drainAllIsolatedRuntimePools releases every extension's idle isolated -// runtimes (memory-pressure hook). -func drainAllIsolatedRuntimePools() { - m := getExtensionManager() - m.mu.RLock() - exts := make([]*loadedExtension, 0, len(m.extensions)) - for _, ext := range m.extensions { - exts = append(exts, ext) - } - m.mu.RUnlock() - - for _, ext := range exts { - drainIsolatedRuntimePool(ext) - } -} - -func (m *extensionManager) initializeVM(ext *loadedExtension) error { - ext.VMMu.Lock() - defer ext.VMMu.Unlock() - return initializeVMLocked(ext) -} - -func initializeExtensionRuntimeWithSettings( - vm *goja.Runtime, - extensionID string, - settings map[string]any, -) error { - settingsJSON, err := json.Marshal(settings) - if err != nil { - return fmt.Errorf("failed to save settings") - } - - script := fmt.Sprintf(` - (function() { - var settings = %s; - if (typeof extension !== 'undefined' && typeof extension.initialize === 'function') { - try { - extension.initialize(settings); - return { success: true }; - } catch (e) { - return { success: false, error: e.toString() }; - } - } - return { success: true, message: 'no initialize function' }; - })() - `, string(settingsJSON)) - - result, err := runExtensionLifecycleCall(vm, func() (goja.Value, error) { - return vm.RunString(script) - }) - if err != nil { - GoLog("[Extension] Initialize error for %s: %v\n", extensionID, err) - return err - } - - if result != nil && !goja.IsUndefined(result) { - exported := result.Export() - if resultMap, ok := exported.(map[string]any); ok { - if success, ok := resultMap["success"].(bool); ok && !success { - errMsg := "unknown error" - if e, ok := resultMap["error"].(string); ok { - errMsg = e - } - GoLog("[Extension] Initialize failed for %s: %s\n", extensionID, errMsg) - return fmt.Errorf("initialize failed: %s", errMsg) - } - } - } - - return nil -} - -func initializeExtensionWithSettingsLocked( - ext *loadedExtension, - settings map[string]any, -) error { - if ext.VM == nil { - return fmt.Errorf("extension failed to load: please reinstall the extension") - } - - if err := initializeExtensionRuntimeWithSettings(ext.VM, ext.ID, settings); err != nil { - ext.Error = err.Error() - ext.Enabled = false - if IsRuntimeUnsafeError(err) { - quarantineRuntimeLocked(ext, ext.VM, err) - } - return err - } - - ext.initialized = true - GoLog("[Extension] Initialized %s\n", ext.ID) - return nil -} - -func runCleanupLocked(ext *loadedExtension) error { - if ext.VM != nil { - if err := runCleanupOnVM(ext.VM); err != nil { - return err - } - if ext.VM.Get("extension") != nil { - GoLog("[Extension] Cleanup called for %s\n", ext.ID) - } - } - return nil -} - -func runCleanupOnVM(vm *goja.Runtime) error { - if vm == nil { - return nil - } - - script := ` - (function() { - if (typeof extension !== 'undefined' && typeof extension.cleanup === 'function') { - try { - extension.cleanup(); - return { success: true }; - } catch (e) { - return { success: false, error: e.toString() }; - } - } - return { success: true, message: 'no cleanup function' }; - })() - ` - - result, err := runExtensionLifecycleCall(vm, func() (goja.Value, error) { - return vm.RunString(script) - }) - if err != nil { - return err - } - - if result != nil && !goja.IsUndefined(result) { - exported := result.Export() - if resultMap, ok := exported.(map[string]any); ok { - if success, ok := resultMap["success"].(bool); ok && !success { - errMsg := "unknown error" - if e, ok := resultMap["error"].(string); ok { - errMsg = e - } - return fmt.Errorf("cleanup failed: %s", errMsg) - } - } - } - - return nil -} - -func teardownVMLocked(ext *loadedExtension) { - drainIsolatedRuntimePool(ext) - // Preserve writes made before cleanup even when the cleanup callback becomes - // unresponsive and its VM has to remain quarantined. - if ext.runtime != nil { - if err := ext.runtime.flushStorageNow(); err != nil { - GoLog("[Extension] Failed to flush storage before cleanup for %s: %v\n", ext.ID, err) - } - } - vm := ext.VM - if err := runCleanupLocked(ext); err != nil { - GoLog("[Extension] Error calling cleanup for %s: %v\n", ext.ID, err) - if IsRuntimeUnsafeError(err) { - quarantineRuntimeLocked(ext, vm, err) - return - } - } - if ext.runtime != nil { - if err := ext.runtime.flushStorageNow(); err != nil { - GoLog("[Extension] Failed to flush storage for %s: %v\n", ext.ID, err) - } - ext.runtime.closeStorageFlusher() - } - ext.runtime = nil - ext.VM = nil - ext.initialized = false -} - -// supportedRuntimeFeatures maps every feature name the goja runtime provides -// to its current contract version (documented in SIGNED_SESSION_GUIDE.md). diff --git a/go_backend/extension_manager_supplement_test.go b/go_backend/extension_manager_supplement_test.go deleted file mode 100644 index caae8496..00000000 --- a/go_backend/extension_manager_supplement_test.go +++ /dev/null @@ -1,195 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" -) - -func TestExtensionManagerPackageLifecycle(t *testing.T) { - dir := t.TempDir() - extensionsDir := filepath.Join(dir, "extensions") - dataDir := filepath.Join(dir, "data") - manager := &extensionManager{extensions: map[string]*loadedExtension{}} - if err := manager.SetDirectories(extensionsDir, dataDir); err != nil { - t.Fatalf("SetDirectories: %v", err) - } - if err := GetExtensionSettingsStore().SetDataDir(dataDir); err != nil { - t.Fatalf("settings data dir: %v", err) - } - - js := ` -var cleaned = false; -registerExtension({ - initialize: function(settings) { this.settings = settings || {}; }, - cleanup: function() { cleaned = true; }, - doAction: function() { return { message: "done", setting_updates: { quality: "lossless" } }; }, - getHomeFeed: function() { return [{ id: "home", title: "Home" }]; }, - getBrowseCategories: function() { return [{ id: "cat", title: "Category" }]; }, - searchTracks: function() { return { tracks: [], total: 0 }; }, - fetchLyrics: function() { return { syncType: "UNSYNCED", lines: [{ words: "hello" }] }; }, - getDownloadUrl: function() { return { url: "https://example.test/a.flac" }; } -}); -` - pkgV1 := filepath.Join(dir, "manager-ext-v1.spotiflac-ext") - createTestExtensionPackage(t, pkgV1, "manager-ext", "1.0.0", js, nil) - pkgV2 := filepath.Join(dir, "manager-ext-v2.spotiflac-ext") - createTestExtensionPackage(t, pkgV2, "manager-ext", "1.1.0", js, nil) - brokenUpgrade := filepath.Join(dir, "manager-ext-broken.spotiflac-ext") - createTestExtensionPackage(t, brokenUpgrade, "manager-ext", "1.0.1", `registerExtension({`, nil) - unsafePkg := filepath.Join(dir, "unsafe.spotiflac-ext") - createTestExtensionPackage(t, unsafePkg, "manager-ext", "1.0.0", js, map[string]string{"../unsafe.txt": "blocked"}) - - if compareVersions("v1.2.0", "1.1.9") <= 0 || compareVersions("1.0.0", "1.0") != 0 || compareVersions("1.0.0", "1.0.1") >= 0 { - t.Fatal("compareVersions mismatch") - } - if _, err := manager.LoadExtensionFromFile(filepath.Join(dir, "bad.txt")); err == nil { - t.Fatal("expected bad extension suffix error") - } - if _, err := manager.LoadExtensionFromFile(filepath.Join(dir, "missing.spotiflac-ext")); err == nil { - t.Fatal("expected invalid package error") - } - if _, err := manager.LoadExtensionFromFile(unsafePkg); err == nil { - t.Fatal("expected unsafe archive path to reject the package") - } - - ext, err := manager.LoadExtensionFromFile(pkgV1) - if err != nil { - t.Fatalf("LoadExtensionFromFile: %v", err) - } - if ext.ID != "manager-ext" || ext.Enabled || ext.SourceDir == "" { - t.Fatalf("loaded extension = %#v", ext) - } - if _, err := manager.LoadExtensionFromFile(pkgV1); err == nil { - t.Fatal("expected duplicate version error") - } - - installedJSON, err := manager.GetInstalledExtensionsJSON() - if err != nil || !strings.Contains(installedJSON, "manager-ext") || !strings.Contains(installedJSON, "icon_path") { - t.Fatalf("GetInstalledExtensionsJSON = %q/%v", installedJSON, err) - } - var installed []map[string]any - if err := json.Unmarshal([]byte(installedJSON), &installed); err != nil || len(installed) != 1 { - t.Fatalf("decode installed = %#v/%v", installed, err) - } - - if err := GetExtensionSettingsStore().Set("manager-ext", "quality", "lossless"); err != nil { - t.Fatalf("settings Set: %v", err) - } - if err := manager.SetExtensionEnabled("manager-ext", true); err != nil { - t.Fatalf("enable extension: %v", err) - } - if !ext.Enabled || ext.VM == nil || !ext.initialized { - t.Fatalf("enabled extension = %#v", ext) - } - if err := manager.InitializeExtension("manager-ext", map[string]any{"quality": "hires"}); err != nil { - t.Fatalf("InitializeExtension: %v", err) - } - action, err := manager.InvokeAction("manager-ext", "doAction") - if err != nil || action["success"] != true || action["message"] != "done" { - t.Fatalf("InvokeAction = %#v/%v", action, err) - } - if err := manager.CleanupExtension("manager-ext"); err != nil { - t.Fatalf("CleanupExtension: %v", err) - } - if err := manager.SetExtensionEnabled("manager-ext", false); err != nil { - t.Fatalf("disable extension: %v", err) - } - if _, err := manager.UpgradeExtension(brokenUpgrade); err == nil { - t.Fatal("expected invalid upgrade to be rejected") - } - stillInstalled, err := manager.GetExtension("manager-ext") - if err != nil || stillInstalled.Manifest.Version != "1.0.0" { - t.Fatalf("failed upgrade replaced the installed version: %#v/%v", stillInstalled, err) - } - if _, err := os.Stat(filepath.Join(stillInstalled.SourceDir, "index.js")); err != nil { - t.Fatalf("failed upgrade removed the working source: %v", err) - } - if ext.VM != nil || ext.initialized { - t.Fatalf("expected VM teardown, got %#v", ext) - } - if _, err := manager.InvokeAction("manager-ext", "doAction"); err == nil { - t.Fatal("expected disabled action error") - } - - upgradeJSON, err := manager.CheckExtensionUpgradeJSON(pkgV2) - if err != nil || !strings.Contains(upgradeJSON, `"can_upgrade":true`) { - t.Fatalf("CheckExtensionUpgradeJSON = %q/%v", upgradeJSON, err) - } - upgraded, err := manager.UpgradeExtension(pkgV2) - if err != nil { - t.Fatalf("UpgradeExtension: %v", err) - } - if upgraded.Manifest.Version != "1.1.0" { - t.Fatalf("upgraded = %#v", upgraded.Manifest) - } - if _, err := manager.UpgradeExtension(pkgV1); err == nil { - t.Fatal("expected downgrade error") - } - if err := manager.RemoveExtension("manager-ext"); err != nil { - t.Fatalf("RemoveExtension: %v", err) - } - if _, err := manager.GetExtension("manager-ext"); err == nil { - t.Fatal("expected removed extension missing") - } - - dirExt := filepath.Join(extensionsDir, "dir-ext") - if err := os.MkdirAll(dirExt, 0755); err != nil { - t.Fatal(err) - } - manifest := `{"name":"dir-ext","displayName":"dir-ext","version":"1.0.0","description":"Directory extension","type":["metadata_provider"],"permissions":{}}` - if err := os.WriteFile(filepath.Join(dirExt, "manifest.json"), []byte(manifest), 0600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dirExt, "index.js"), []byte(`registerExtension({searchTracks:function(){return {tracks:[], total:0};}});`), 0600); err != nil { - t.Fatal(err) - } - loaded, loadErrs := manager.LoadExtensionsFromDirectory(extensionsDir) - if len(loadErrs) != 0 || len(loaded) != 1 || loaded[0] != "dir-ext" { - t.Fatalf("LoadExtensionsFromDirectory = %#v/%#v", loaded, loadErrs) - } - manager.UnloadAllExtensions() - if len(manager.GetAllExtensions()) != 0 { - t.Fatal("expected all extensions unloaded") - } -} - -func TestValidateManifestGates(t *testing.T) { - originalVersion := GetAppVersion() - defer SetAppVersion(originalVersion) - SetAppVersion("4.5.0") - - if err := validateManifestGates(nil); err != nil { - t.Fatalf("nil manifest = %v", err) - } - if err := validateManifestGates(&ExtensionManifest{MinAppVersion: "4.9.0"}); err == nil { - t.Fatal("expected minAppVersion gate to fail") - } - if err := validateManifestGates(&ExtensionManifest{MinAppVersion: "4.5.0"}); err != nil { - t.Fatalf("equal version should pass: %v", err) - } - SetAppVersion("") - if err := validateManifestGates(&ExtensionManifest{MinAppVersion: "9.9.9"}); err != nil { - t.Fatalf("empty app version must skip the gate: %v", err) - } - SetAppVersion("4.5.0") - - pass := &ExtensionManifest{ - RequiredRuntimeFeatures: []string{"signedSession@3", "sessionGrant"}, - } - if err := validateManifestGates(pass); err != nil { - t.Fatalf("supported features should pass: %v", err) - } - if err := validateManifestGates(&ExtensionManifest{ - RequiredRuntimeFeatures: []string{"quantumDecrypt"}, - }); err == nil { - t.Fatal("unknown feature must fail") - } - if err := validateManifestGates(&ExtensionManifest{ - RequiredRuntimeFeatures: []string{"signedSession@99"}, - }); err == nil { - t.Fatal("future contract version must fail") - } -} diff --git a/go_backend/extension_manifest.go b/go_backend/extension_manifest.go deleted file mode 100644 index 6f2d7d1e..00000000 --- a/go_backend/extension_manifest.go +++ /dev/null @@ -1,453 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "net/url" - "regexp" - "strings" -) - -var extensionIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,127}$`) - -type ExtensionType string - -const ( - ExtensionTypeMetadataProvider ExtensionType = "metadata_provider" - ExtensionTypeDownloadProvider ExtensionType = "download_provider" - ExtensionTypeLyricsProvider ExtensionType = "lyrics_provider" -) - -type SettingType string - -const ( - SettingTypeString SettingType = "string" - SettingTypeNumber SettingType = "number" - SettingTypeBool SettingType = "boolean" - SettingTypeSelect SettingType = "select" - SettingTypeButton SettingType = "button" // Action button that calls a JS function -) - -type ExtensionPermissions struct { - Network []string `json:"network"` - Storage bool `json:"storage"` - File bool `json:"file"` - AllowHTTP bool `json:"allowHttp,omitempty"` -} - -type ExtensionSetting struct { - Key string `json:"key"` - Type SettingType `json:"type"` - Label string `json:"label"` - Description string `json:"description,omitempty"` - Required bool `json:"required,omitempty"` - Secret bool `json:"secret,omitempty"` - Default any `json:"default,omitempty"` - Options []string `json:"options,omitempty"` - Action string `json:"action,omitempty"` -} - -type QualityOption struct { - ID string `json:"id"` - Kind string `json:"kind,omitempty"` - Label string `json:"label"` - Description string `json:"description"` - Settings []QualitySpecificSetting `json:"settings,omitempty"` -} - -type QualitySpecificSetting struct { - Key string `json:"key"` - Type SettingType `json:"type"` - Label string `json:"label"` - Description string `json:"description,omitempty"` - Required bool `json:"required,omitempty"` - Secret bool `json:"secret,omitempty"` - Default any `json:"default,omitempty"` - Options []string `json:"options,omitempty"` -} - -type SearchFilter struct { - ID string `json:"id"` - Label string `json:"label,omitempty"` - Icon string `json:"icon,omitempty"` -} - -type SearchBehaviorConfig struct { - Enabled bool `json:"enabled"` - Placeholder string `json:"placeholder,omitempty"` - Primary bool `json:"primary,omitempty"` - Icon string `json:"icon,omitempty"` - ThumbnailRatio string `json:"thumbnailRatio,omitempty"` - ThumbnailWidth int `json:"thumbnailWidth,omitempty"` - ThumbnailHeight int `json:"thumbnailHeight,omitempty"` - Filters []SearchFilter `json:"filters,omitempty"` -} - -type URLHandlerConfig struct { - Enabled bool `json:"enabled"` - Patterns []string `json:"patterns,omitempty"` -} - -type TrackMatchingConfig struct { - CustomMatching bool `json:"customMatching"` - Strategy string `json:"strategy,omitempty"` - DurationTolerance int `json:"durationTolerance,omitempty"` -} - -type PostProcessingHook struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - DefaultEnabled bool `json:"defaultEnabled,omitempty"` - SupportedFormats []string `json:"supportedFormats,omitempty"` -} - -type PostProcessingConfig struct { - Enabled bool `json:"enabled"` - Hooks []PostProcessingHook `json:"hooks,omitempty"` -} - -type ExtensionHealthCheck struct { - ID string `json:"id"` - Label string `json:"label,omitempty"` - URL string `json:"url"` - Method string `json:"method,omitempty"` - ServiceKey string `json:"serviceKey,omitempty"` - TimeoutMs int `json:"timeoutMs,omitempty"` - CacheTTLSeconds int `json:"cacheTtlSeconds,omitempty"` - Required bool `json:"required,omitempty"` -} - -type SignedSessionEndpoints struct { - Bootstrap string `json:"bootstrap,omitempty"` - Challenge string `json:"challenge,omitempty"` - Exchange string `json:"exchange,omitempty"` - Refresh string `json:"refresh,omitempty"` -} - -type SignedSessionConfig struct { - Namespace string `json:"namespace"` - BaseURL string `json:"baseUrl"` - AppVersion string `json:"appVersion,omitempty"` - Platform string `json:"platform,omitempty"` - CallbackURL string `json:"callbackUrl,omitempty"` - SchemeLabel string `json:"schemeLabel,omitempty"` - HeaderPrefix string `json:"headerPrefix,omitempty"` - TimeWindowSeconds int `json:"timeWindowSeconds,omitempty"` - Endpoints SignedSessionEndpoints `json:"endpoints,omitempty"` -} - -type ExtensionManifest struct { - Name string `json:"name"` - DisplayName string `json:"displayName"` - Version string `json:"version"` - Description string `json:"description"` - Homepage string `json:"homepage,omitempty"` - Icon string `json:"icon,omitempty"` - Types []ExtensionType `json:"type"` - Permissions ExtensionPermissions `json:"permissions"` - Settings []ExtensionSetting `json:"settings,omitempty"` - QualityOptions []QualityOption `json:"qualityOptions,omitempty"` - MinAppVersion string `json:"minAppVersion,omitempty"` - SkipMetadataEnrichment bool `json:"skipMetadataEnrichment,omitempty"` - SkipLyrics bool `json:"skipLyrics,omitempty"` - StopProviderFallback bool `json:"stopProviderFallback,omitempty"` - SkipBuiltInFallback bool `json:"skipBuiltInFallback,omitempty"` - SearchBehavior *SearchBehaviorConfig `json:"searchBehavior,omitempty"` - URLHandler *URLHandlerConfig `json:"urlHandler,omitempty"` - TrackMatching *TrackMatchingConfig `json:"trackMatching,omitempty"` - PostProcessing *PostProcessingConfig `json:"postProcessing,omitempty"` - ServiceHealth []ExtensionHealthCheck `json:"serviceHealth,omitempty"` - SignedSession *SignedSessionConfig `json:"signedSession,omitempty"` - RequiredRuntimeFeatures []string `json:"requiredRuntimeFeatures,omitempty"` - Capabilities map[string]any `json:"capabilities,omitempty"` -} - -type ManifestValidationError struct { - Field string - Message string -} - -func (e *ManifestValidationError) Error() string { - return fmt.Sprintf("manifest validation error: %s - %s", e.Field, e.Message) -} - -func ParseManifest(data []byte) (*ExtensionManifest, error) { - var manifest ExtensionManifest - if err := json.Unmarshal(data, &manifest); err != nil { - return nil, fmt.Errorf("failed to parse manifest JSON: %w", err) - } - - if err := manifest.Validate(); err != nil { - return nil, err - } - - return &manifest, nil -} - -func (m *ExtensionManifest) Validate() error { - if strings.TrimSpace(m.Name) == "" { - return &ManifestValidationError{Field: "name", Message: "name is required"} - } - if !extensionIDPattern.MatchString(m.Name) { - return &ManifestValidationError{ - Field: "name", - Message: "name must be a lowercase extension ID containing only letters, numbers, '.', '_' or '-'", - } - } - - if strings.TrimSpace(m.Version) == "" { - return &ManifestValidationError{Field: "version", Message: "version is required"} - } - - if strings.TrimSpace(m.Description) == "" { - return &ManifestValidationError{Field: "description", Message: "description is required"} - } - - if len(m.Types) == 0 { - return &ManifestValidationError{Field: "type", Message: "at least one type is required"} - } - - for _, t := range m.Types { - if t != ExtensionTypeMetadataProvider && t != ExtensionTypeDownloadProvider && t != ExtensionTypeLyricsProvider { - return &ManifestValidationError{ - Field: "type", - Message: fmt.Sprintf("invalid extension type: %s (must be 'metadata_provider', 'download_provider', or 'lyrics_provider')", t), - } - } - } - - for i, setting := range m.Settings { - if strings.TrimSpace(setting.Key) == "" { - return &ManifestValidationError{ - Field: fmt.Sprintf("settings[%d].key", i), - Message: "setting key is required", - } - } - - if setting.Type == "" { - return &ManifestValidationError{ - Field: fmt.Sprintf("settings[%d].type", i), - Message: "setting type is required", - } - } - - if setting.Type == SettingTypeSelect && len(setting.Options) == 0 { - return &ManifestValidationError{ - Field: fmt.Sprintf("settings[%d].options", i), - Message: "select type requires options", - } - } - - if setting.Type == SettingTypeButton && setting.Action == "" { - return &ManifestValidationError{ - Field: fmt.Sprintf("settings[%d].action", i), - Message: "button type requires action (JS function name)", - } - } - } - - for i, quality := range m.QualityOptions { - switch quality.Kind { - case "", "lossless", "lossy", "spatial": - default: - return &ManifestValidationError{ - Field: fmt.Sprintf("qualityOptions[%d].kind", i), - Message: "quality kind must be lossless, lossy, or spatial", - } - } - } - - for i, check := range m.ServiceHealth { - if strings.TrimSpace(check.ID) == "" { - return &ManifestValidationError{ - Field: fmt.Sprintf("serviceHealth[%d].id", i), - Message: "health check id is required", - } - } - if strings.TrimSpace(check.URL) == "" { - return &ManifestValidationError{ - Field: fmt.Sprintf("serviceHealth[%d].url", i), - Message: "health check url is required", - } - } - method := strings.ToUpper(strings.TrimSpace(check.Method)) - if method != "" && method != "GET" && method != "HEAD" { - return &ManifestValidationError{ - Field: fmt.Sprintf("serviceHealth[%d].method", i), - Message: "health check method must be GET or HEAD", - } - } - } - - if m.SignedSession != nil { - if !m.Permissions.Storage { - return &ManifestValidationError{Field: "permissions.storage", Message: "signedSession requires storage permission"} - } - if strings.TrimSpace(m.SignedSession.Namespace) == "" { - return &ManifestValidationError{Field: "signedSession.namespace", Message: "namespace is required"} - } - baseURL := strings.TrimSpace(m.SignedSession.BaseURL) - if baseURL == "" { - return &ManifestValidationError{Field: "signedSession.baseUrl", Message: "baseUrl is required"} - } - if !strings.HasPrefix(strings.ToLower(baseURL), "https://") { - return &ManifestValidationError{Field: "signedSession.baseUrl", Message: "baseUrl must use https"} - } - parsed, err := url.Parse(baseURL) - if err != nil || parsed.Hostname() == "" { - return &ManifestValidationError{Field: "signedSession.baseUrl", Message: "baseUrl is invalid"} - } - if !m.IsDomainAllowed(parsed.Hostname()) { - return &ManifestValidationError{Field: "signedSession.baseUrl", Message: "baseUrl host must be listed in permissions.network"} - } - } - if m.HasCapability("rawFfmpeg") && !m.Permissions.File { - return &ManifestValidationError{Field: "permissions.file", Message: "rawFfmpeg capability requires file permission"} - } - if err := validateDownloadTransferCapability(m.Capabilities); err != nil { - return &ManifestValidationError{Field: "capabilities.downloadTransfer", Message: err.Error()} - } - - return nil -} - -func (m *ExtensionManifest) HasCapability(name string) bool { - if m == nil || m.Capabilities == nil { - return false - } - value, ok := m.Capabilities[name] - if !ok { - return false - } - enabled, ok := value.(bool) - return ok && enabled -} - -func (m *ExtensionManifest) HasType(t ExtensionType) bool { - for _, et := range m.Types { - if et == t { - return true - } - } - return false -} - -func (m *ExtensionManifest) IsMetadataProvider() bool { - return m.HasType(ExtensionTypeMetadataProvider) -} - -func (m *ExtensionManifest) IsDownloadProvider() bool { - return m.HasType(ExtensionTypeDownloadProvider) -} - -func (m *ExtensionManifest) IsLyricsProvider() bool { - return m.HasType(ExtensionTypeLyricsProvider) -} - -func (m *ExtensionManifest) StopsProviderFallback() bool { - if m == nil { - return false - } - return m.StopProviderFallback || m.SkipBuiltInFallback -} - -func (m *ExtensionManifest) IsDomainAllowed(domain string) bool { - domain = strings.ToLower(strings.TrimSpace(domain)) - for _, allowed := range m.Permissions.Network { - allowed = strings.ToLower(strings.TrimSpace(allowed)) - if allowed == domain { - return true - } - // Support wildcard subdomains (e.g., *.example.com) - if strings.HasPrefix(allowed, "*.") { - suffix := allowed[1:] - if strings.HasSuffix(domain, suffix) { - return true - } - } - } - return false -} - -func (m *ExtensionManifest) HasCustomSearch() bool { - return m.SearchBehavior != nil && m.SearchBehavior.Enabled -} - -func (m *ExtensionManifest) HasCustomMatching() bool { - return m.TrackMatching != nil && m.TrackMatching.CustomMatching -} - -func (m *ExtensionManifest) HasPostProcessing() bool { - return m.PostProcessing != nil && m.PostProcessing.Enabled -} - -func (m *ExtensionManifest) HasURLHandler() bool { - return m.URLHandler != nil && m.URLHandler.Enabled && len(m.URLHandler.Patterns) > 0 -} - -// MatchesURL reports whether one of the handler's patterns matches the URL. -// Web patterns are anchored to the URL's host (exact domain or subdomain, -// optional path prefix) — never matched as a raw substring, so "spotify.com" -// cannot match a URL that merely embeds it in a query parameter. Patterns -// ending in ":" (e.g. "spotify:") match custom URI schemes as prefixes. -func (m *ExtensionManifest) MatchesURL(urlStr string) bool { - if !m.HasURLHandler() { - return false - } - - urlStr = strings.ToLower(strings.TrimSpace(urlStr)) - parsed, parseErr := url.Parse(urlStr) - - for _, pattern := range m.URLHandler.Patterns { - pattern = strings.ToLower(strings.TrimSpace(pattern)) - if pattern == "" { - continue - } - - // Scheme patterns anchor to the front of the URI. - if !strings.Contains(pattern, "/") && strings.HasSuffix(pattern, ":") { - if strings.HasPrefix(urlStr, pattern) { - return true - } - continue - } - - if parseErr != nil || parsed.Host == "" { - continue - } - host := parsed.Hostname() - urlPath := parsed.Path - if urlPath == "" { - urlPath = "/" - } - - if idx := strings.Index(pattern, "://"); idx >= 0 { - pattern = pattern[idx+3:] - } - patternHost, patternPath, hasPath := strings.Cut(pattern, "/") - if patternHost == "" { - continue - } - if host != patternHost && !strings.HasSuffix(host, "."+patternHost) { - continue - } - if hasPath && patternPath != "" && - !strings.HasPrefix(urlPath, "/"+patternPath) { - continue - } - return true - } - return false -} - -func (m *ExtensionManifest) GetPostProcessingHooks() []PostProcessingHook { - if m.PostProcessing == nil { - return nil - } - return m.PostProcessing.Hooks -} - -func (m *ExtensionManifest) ToJSON() ([]byte, error) { - return json.Marshal(m) -} diff --git a/go_backend/extension_package_security_test.go b/go_backend/extension_package_security_test.go deleted file mode 100644 index 93cb6f6b..00000000 --- a/go_backend/extension_package_security_test.go +++ /dev/null @@ -1,184 +0,0 @@ -package gobackend - -import ( - "archive/zip" - "bytes" - "crypto/sha256" - "fmt" - "os" - "path/filepath" - "strings" - "testing" -) - -func TestWriteVerifiedExtensionPackageAcceptsMatchingSHA256(t *testing.T) { - dir := t.TempDir() - dest := filepath.Join(dir, "verified.spotiflac-ext") - payload := []byte("extension package") - checksum := fmt.Sprintf("%x", sha256.Sum256(payload)) - - if err := writeVerifiedExtensionPackage(bytes.NewReader(payload), dest, checksum); err != nil { - t.Fatalf("writeVerifiedExtensionPackage: %v", err) - } - got, err := os.ReadFile(dest) - if err != nil { - t.Fatalf("read verified package: %v", err) - } - if !bytes.Equal(got, payload) { - t.Fatalf("verified package = %q, want %q", got, payload) - } -} - -func TestWriteVerifiedExtensionPackageRejectsMismatchBeforeReplace(t *testing.T) { - dir := t.TempDir() - dest := filepath.Join(dir, "cached.spotiflac-ext") - if err := os.WriteFile(dest, []byte("existing"), 0600); err != nil { - t.Fatal(err) - } - - err := writeVerifiedExtensionPackage( - bytes.NewReader([]byte("tampered")), - dest, - strings.Repeat("0", sha256.Size*2), - ) - if err == nil || !strings.Contains(err.Error(), "SHA-256 mismatch") { - t.Fatalf("expected checksum mismatch, got %v", err) - } - got, readErr := os.ReadFile(dest) - if readErr != nil { - t.Fatal(readErr) - } - if string(got) != "existing" { - t.Fatalf("checksum failure replaced existing package with %q", got) - } -} - -func TestRegistrySkipsOnlyExtensionWithMalformedChecksum(t *testing.T) { - checksum := strings.Repeat("a", sha256.Size*2) - registry, err := parseRegistryBody([]byte( - `{"version":1,"extensions":[` + - `{"id":"bad","name":"bad","version":"1.0.0","sha256":"not-a-hash"},` + - `{"id":"verified","name":"verified","version":"1.0.0","checksumSha256":"sha256:` + - checksum + - `"},` + - `{"id":"legacy","name":"legacy","version":"1.0.0"}` + - `]}`, - )) - if err != nil { - t.Fatalf("parse registry: %v", err) - } - if len(registry.Extensions) != 2 { - t.Fatalf("registry extensions = %#v, want valid entries only", registry.Extensions) - } - if registry.Extensions[0].ID != "verified" || registry.Extensions[1].ID != "legacy" { - t.Fatalf("registry extension order = %#v", registry.Extensions) - } - if got := registry.Extensions[0].getSHA256(); got != checksum { - t.Fatalf("normalized checksum = %q, want %q", got, checksum) - } -} - -func TestExtensionPackageRequiresUniqueRootEntrypoints(t *testing.T) { - dir := t.TempDir() - duplicate := filepath.Join(dir, "duplicate.spotiflac-ext") - createTestExtensionPackage( - t, - duplicate, - "duplicate-ext", - "1.0.0", - `registerExtension({});`, - map[string]string{"MANIFEST.JSON": "{}"}, - ) - - reader, err := zip.OpenReader(duplicate) - if err != nil { - t.Fatal(err) - } - _, inspectErr := inspectExtensionPackage(reader.File) - _ = reader.Close() - if inspectErr == nil || !strings.Contains(inspectErr.Error(), "duplicate path") { - t.Fatalf("expected duplicate archive path error, got %v", inspectErr) - } - - nested := filepath.Join(dir, "nested.spotiflac-ext") - writeTestZip(t, nested, map[string]string{ - "nested/manifest.json": validSecurityTestManifest("nested-ext"), - "nested/index.js": `registerExtension({});`, - }) - reader, err = zip.OpenReader(nested) - if err != nil { - t.Fatal(err) - } - _, inspectErr = inspectExtensionPackage(reader.File) - _ = reader.Close() - if inspectErr == nil || !strings.Contains(inspectErr.Error(), "root manifest.json") { - t.Fatalf("expected root entrypoint error, got %v", inspectErr) - } -} - -func TestExtensionPackageRejectsUnsafeAndOversizedEntries(t *testing.T) { - for _, unsafePath := range []string{ - "../outside.js", - `nested\outside.js`, - "/absolute.js", - } { - t.Run(unsafePath, func(t *testing.T) { - archivePath := filepath.Join(t.TempDir(), "unsafe.spotiflac-ext") - writeTestZip(t, archivePath, map[string]string{ - "manifest.json": validSecurityTestManifest("unsafe-ext"), - "index.js": `registerExtension({});`, - unsafePath: "unsafe", - }) - reader, err := zip.OpenReader(archivePath) - if err != nil { - t.Fatal(err) - } - _, inspectErr := inspectExtensionPackage(reader.File) - _ = reader.Close() - if inspectErr == nil || !strings.Contains(inspectErr.Error(), "unsafe path") { - t.Fatalf("expected unsafe archive path error, got %v", inspectErr) - } - }) - } - - oversized := &zip.File{FileHeader: zip.FileHeader{ - Name: "payload.bin", - UncompressedSize64: maxExtensionArchiveUncompressedBytes + 1, - }} - if err := validateExtensionArchive([]*zip.File{oversized}); err == nil || - !strings.Contains(err.Error(), "extracted size limit") { - t.Fatalf("expected extracted size error, got %v", err) - } -} - -func validSecurityTestManifest(name string) string { - return fmt.Sprintf( - `{"name":%q,"displayName":%q,"version":"1.0.0","description":"test","type":["metadata_provider"],"permissions":{}}`, - name, - name, - ) -} - -func writeTestZip(t *testing.T, filePath string, files map[string]string) { - t.Helper() - output, err := os.Create(filePath) - if err != nil { - t.Fatal(err) - } - archive := zip.NewWriter(output) - for name, content := range files { - writer, createErr := archive.Create(name) - if createErr != nil { - t.Fatal(createErr) - } - if _, writeErr := writer.Write([]byte(content)); writeErr != nil { - t.Fatal(writeErr) - } - } - if err := archive.Close(); err != nil { - t.Fatal(err) - } - if err := output.Close(); err != nil { - t.Fatal(err) - } -} diff --git a/go_backend/extension_perf.go b/go_backend/extension_perf.go deleted file mode 100644 index b7f13f90..00000000 --- a/go_backend/extension_perf.go +++ /dev/null @@ -1,119 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "time" - - "github.com/dop251/goja" -) - -type extensionCallPerf struct { - extensionID string - operation string - startedAt time.Time - initMs float64 - jsMs float64 - parseMs float64 - items int - payloadBytes int -} - -func newExtensionCallPerf(extensionID, operation string) *extensionCallPerf { - if !GetLogBuffer().IsLoggingEnabled() { - return nil - } - return &extensionCallPerf{ - extensionID: extensionID, - operation: operation, - startedAt: time.Now(), - } -} - -func extensionDurationMs(duration time.Duration) float64 { - return float64(duration.Microseconds()) / 1000.0 -} - -func (p *extensionCallPerf) recordInit(duration time.Duration) { - if p == nil { - return - } - p.initMs += extensionDurationMs(duration) -} - -func (p *extensionCallPerf) recordJS(duration time.Duration) { - if p == nil { - return - } - p.jsMs += extensionDurationMs(duration) -} - -func (p *extensionCallPerf) recordParse(duration time.Duration) { - if p == nil { - return - } - p.parseMs += extensionDurationMs(duration) -} - -func (p *extensionCallPerf) recordPayload(value goja.Value) { - if p == nil || gojaValueIsEmpty(value) { - return - } - if payload, err := json.Marshal(value); err == nil { - p.payloadBytes = len(payload) - } -} - -func (p *extensionCallPerf) setPayloadBytes(payloadBytes int) { - if p == nil { - return - } - p.payloadBytes = payloadBytes -} - -func (p *extensionCallPerf) setItems(items int) { - if p == nil { - return - } - p.items = items -} - -func (p *extensionCallPerf) finish() { - if p == nil { - return - } - LogDebug( - "ExtensionPerf", - "extension=%s op=%s totalMs=%.1f initMs=%.1f jsMs=%.1f parseMs=%.1f items=%d payloadBytes=%d", - p.extensionID, - p.operation, - extensionDurationMs(time.Since(p.startedAt)), - p.initMs, - p.jsMs, - p.parseMs, - p.items, - p.payloadBytes, - ) -} - -func countExtensionTopLevelItems(vm *goja.Runtime, value goja.Value) int { - if gojaValueIsEmpty(value) { - return 0 - } - - if length, err := gojaArrayLength(value, vm); err == nil && length > 0 { - return length - } - - obj := value.ToObject(vm) - for _, key := range []string{"items", "tracks", "sections", "albums", "artists", "playlists", "results"} { - child := obj.Get(key) - if gojaValueIsEmpty(child) { - continue - } - if length, err := gojaArrayLength(child, vm); err == nil && length > 0 { - return length - } - } - - return 1 -} diff --git a/go_backend/extension_priority.go b/go_backend/extension_priority.go deleted file mode 100644 index 2a2b0832..00000000 --- a/go_backend/extension_priority.go +++ /dev/null @@ -1,201 +0,0 @@ -package gobackend - -import ( - "strings" - "sync" -) - -var providerPriority []string -var providerPriorityMu sync.RWMutex - -var extensionFallbackProviderIDs []string -var extensionFallbackProviderIDsMu sync.RWMutex - -var metadataProviderPriority []string -var metadataProviderPriorityMu sync.RWMutex - -func SetProviderPriority(providerIDs []string) { - providerPriorityMu.Lock() - defer providerPriorityMu.Unlock() - providerPriority = sanitizeDownloadProviderPriority(providerIDs) - GoLog("[Extension] Download provider priority set: %v\n", providerPriority) -} - -func GetProviderPriority() []string { - providerPriorityMu.RLock() - defer providerPriorityMu.RUnlock() - - if len(providerPriority) == 0 { - return []string{} - } - - result := make([]string, len(providerPriority)) - copy(result, providerPriority) - return result -} - -func sanitizeDownloadProviderPriority(providerIDs []string) []string { - sanitized := make([]string, 0, len(providerIDs)) - seen := map[string]struct{}{} - - for _, providerID := range providerIDs { - providerID = strings.TrimSpace(providerID) - if providerID == "" { - continue - } - - if isRetiredBuiltInDownloadProvider(providerID) { - continue - } - - seenKey := strings.ToLower(providerID) - if _, exists := seen[seenKey]; exists { - continue - } - seen[seenKey] = struct{}{} - sanitized = append(sanitized, providerID) - } - - return sanitized -} - -func isRetiredBuiltInDownloadProvider(providerID string) bool { - normalized := strings.ToLower(strings.TrimSpace(providerID)) - if normalized == "" { - return false - } - switch normalized { - case "deezer", "qobuz", "tidal": - return !hasEnabledExtensionProvider(normalized, func(manifest *ExtensionManifest) bool { - return manifest.IsDownloadProvider() - }) - default: - return false - } -} - -func isRetiredBuiltInMetadataProvider(providerID string) bool { - normalized := strings.ToLower(strings.TrimSpace(providerID)) - if normalized == "" { - return false - } - switch normalized { - case "deezer", "spotify", "qobuz", "tidal": - return !hasEnabledExtensionProvider(normalized, func(manifest *ExtensionManifest) bool { - return manifest.IsMetadataProvider() - }) - default: - return false - } -} - -func hasEnabledExtensionProvider(providerID string, matches func(*ExtensionManifest) bool) bool { - if providerID == "" || matches == nil { - return false - } - - manager := getExtensionManager() - manager.mu.RLock() - defer manager.mu.RUnlock() - - for id, ext := range manager.extensions { - if !strings.EqualFold(strings.TrimSpace(id), providerID) { - continue - } - if ext == nil || !ext.Enabled || ext.Error != "" || ext.Manifest == nil { - return false - } - return matches(ext.Manifest) - } - - return false -} - -func SetExtensionFallbackProviderIDs(providerIDs []string) { - extensionFallbackProviderIDsMu.Lock() - defer extensionFallbackProviderIDsMu.Unlock() - - if providerIDs == nil { - extensionFallbackProviderIDs = nil - GoLog("[Extension] Extension fallback providers reset to default (all enabled download extensions)\n") - return - } - - sanitized := make([]string, 0, len(providerIDs)) - seen := map[string]struct{}{} - for _, providerID := range providerIDs { - providerID = strings.TrimSpace(providerID) - if providerID == "" { - continue - } - if _, exists := seen[providerID]; exists { - continue - } - seen[providerID] = struct{}{} - sanitized = append(sanitized, providerID) - } - - extensionFallbackProviderIDs = sanitized - GoLog("[Extension] Extension fallback providers set: %v\n", sanitized) -} - -func GetExtensionFallbackProviderIDs() []string { - extensionFallbackProviderIDsMu.RLock() - defer extensionFallbackProviderIDsMu.RUnlock() - - if extensionFallbackProviderIDs == nil { - return nil - } - - result := make([]string, len(extensionFallbackProviderIDs)) - copy(result, extensionFallbackProviderIDs) - return result -} - -func isExtensionFallbackAllowed(providerID string) bool { - allowed := GetExtensionFallbackProviderIDs() - if allowed == nil { - return true - } - - for _, allowedProviderID := range allowed { - if allowedProviderID == providerID { - return true - } - } - return false -} - -func SetMetadataProviderPriority(providerIDs []string) { - metadataProviderPriorityMu.Lock() - defer metadataProviderPriorityMu.Unlock() - - sanitized := make([]string, 0, len(providerIDs)) - seen := map[string]struct{}{} - for _, providerID := range providerIDs { - providerID = strings.TrimSpace(providerID) - if providerID == "" || isRetiredBuiltInMetadataProvider(providerID) { - continue - } - if _, exists := seen[providerID]; exists { - continue - } - seen[providerID] = struct{}{} - sanitized = append(sanitized, providerID) - } - metadataProviderPriority = sanitized - GoLog("[Extension] Metadata provider priority set: %v\n", sanitized) -} - -func GetMetadataProviderPriority() []string { - metadataProviderPriorityMu.RLock() - defer metadataProviderPriorityMu.RUnlock() - - if len(metadataProviderPriority) == 0 { - return []string{} - } - - result := make([]string, len(metadataProviderPriority)) - copy(result, metadataProviderPriority) - return result -} diff --git a/go_backend/extension_provider_supplement_test.go b/go_backend/extension_provider_supplement_test.go deleted file mode 100644 index 23ddb569..00000000 --- a/go_backend/extension_provider_supplement_test.go +++ /dev/null @@ -1,214 +0,0 @@ -package gobackend - -import ( - "errors" - "os" - "path/filepath" - "testing" - "time" -) - -func TestExtensionDownloadCancellationInterruptsBusyJavaScript(t *testing.T) { - ext := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - if err := os.WriteFile(filepath.Join(ext.SourceDir, "index.js"), []byte(` -registerExtension({ - download: function() { - while (true) {} - } -}); -`), 0600); err != nil { - t.Fatalf("write busy extension: %v", err) - } - provider := newExtensionProviderWrapper(ext) - const itemID = "busy-download-cancel" - outputPath := filepath.Join(t.TempDir(), "busy.flac") - done := make(chan error, 1) - go func() { - _, err := provider.DownloadPrepared( - "track", - "best", - outputPath, - itemID, - nil, - nil, - ) - done <- err - }() - - deadline := time.Now().Add(time.Second) - for { - downloadCancels.mu.Lock() - entry := downloadCancels.entries[itemID] - ready := entry != nil && entry.refs > 0 - downloadCancels.mu.Unlock() - if ready { - break - } - if time.Now().After(deadline) { - t.Fatal("download cancellation context was not initialized") - } - time.Sleep(time.Millisecond) - } - cancelDownload(itemID) - select { - case err := <-done: - if !errors.Is(err, ErrDownloadCancelled) { - t.Fatalf("DownloadPrepared error = %v, want ErrDownloadCancelled", err) - } - case <-time.After(time.Second): - t.Fatal("busy extension did not stop after cancellation") - } -} - -func TestExtensionProviderWrapperFullSurface(t *testing.T) { - ext := newTestLoadedExtension(t, ExtensionTypeMetadataProvider, ExtensionTypeDownloadProvider, ExtensionTypeLyricsProvider) - provider := newExtensionProviderWrapper(ext) - - search, err := provider.SearchTracks("query", 5) - if err != nil { - t.Fatalf("SearchTracks: %v", err) - } - if search.Total != 1 || search.Tracks[0].ProviderID != ext.ID || search.Tracks[0].ExternalLinks["provider"] == "" { - t.Fatalf("search = %#v", search) - } - - track, err := provider.GetTrack("track-1") - if err != nil { - t.Fatalf("GetTrack: %v", err) - } - if track.Name != "Track track-1" || track.ProviderID != ext.ID || track.AudioQuality == "" || track.Comment != "https://example.test/album/1" || !track.Explicit || track.UPC != "0012345678901" { - t.Fatalf("track = %#v", track) - } - - album, err := provider.GetAlbum("album-1") - if err != nil { - t.Fatalf("GetAlbum: %v", err) - } - if album.ProviderID != ext.ID || len(album.Tracks) != 1 || album.Tracks[0].ProviderID != ext.ID { - t.Fatalf("album = %#v", album) - } - - playlist, err := provider.GetPlaylist("playlist-1") - if err != nil { - t.Fatalf("GetPlaylist: %v", err) - } - if playlist.Name != "Playlist playlist-1" || playlist.ProviderID != ext.ID { - t.Fatalf("playlist = %#v", playlist) - } - - artist, err := provider.GetArtist("artist-1") - if err != nil { - t.Fatalf("GetArtist: %v", err) - } - if artist.ProviderID != ext.ID || len(artist.Releases) != 1 || artist.Releases[0].ProviderID != ext.ID { - t.Fatalf("artist = %#v", artist) - } - - enriched, err := provider.EnrichTrack(&ExtTrackMetadata{ID: "track-1", Name: "Old", ProviderID: ext.ID}) - if err != nil { - t.Fatalf("EnrichTrack: %v", err) - } - if enriched.Name != "Enriched" || enriched.ProviderID != ext.ID { - t.Fatalf("enriched = %#v", enriched) - } - - availability, err := provider.CheckAvailabilityForItemID("ISRC", "Song", "Artist", "source-id", "alternate-id-a", "alternate-id-b", "alternate-id-c", 0, "") - if err != nil { - t.Fatalf("CheckAvailabilityForItemID: %v", err) - } - if !availability.Available || availability.TrackID != "download-track" || !availability.SkipFallback { - t.Fatalf("availability = %#v", availability) - } - if availability.PreparedContext["token"] != "prepared" { - t.Fatalf("prepared context = %#v", availability.PreparedContext) - } - - progress := []int{} - download, err := provider.Download("track-1", "LOSSLESS", filepath.Join(t.TempDir(), "song.flac"), "", func(percent int) { - progress = append(progress, percent) - }) - if err != nil { - t.Fatalf("Download: %v", err) - } - if !download.Success || download.Decryption == nil || download.DecryptionKey != "001122" || download.Comment != "https://example.test/album/1" || !download.Explicit || download.AlbumType != "compilation" || download.UPC != "0012345678901" || len(progress) != 1 || progress[0] != 100 { - t.Fatalf("download = %#v progress=%v", download, progress) - } - preparedDownload, err := provider.DownloadPrepared( - "track-1", - "LOSSLESS", - filepath.Join(t.TempDir(), "prepared.flac"), - "", - availability.PreparedContext, - nil, - ) - if err != nil || preparedDownload == nil || preparedDownload.Title != "prepared" { - t.Fatalf("prepared download = %#v, err=%v", preparedDownload, err) - } - - lyrics, err := provider.FetchLyrics("Song", "Artist", "Album", 180) - if err != nil { - t.Fatalf("GetLyrics: %v", err) - } - if lyrics.Provider != ext.ID || len(lyrics.Lines) != 1 || lyrics.Lines[0].Words != "Hello" { - t.Fatalf("lyrics = %#v", lyrics) - } - - urlResult, err := provider.HandleURL("https://example.test/track/1") - if err != nil { - t.Fatalf("HandleURL: %v", err) - } - if urlResult.Track == nil || urlResult.Track.Name == "" || len(urlResult.Tracks) != 1 || urlResult.Album == nil || urlResult.Artist == nil { - t.Fatalf("url result = %#v", urlResult) - } - - post, err := provider.PostProcessV2(PostProcessInput{Path: filepath.Join(t.TempDir(), "song.flac")}, map[string]any{"title": "Song"}, "hook") - if err != nil { - t.Fatalf("PostProcessV2: %v", err) - } - if !post.Success || post.BitDepth != 24 || post.SampleRate != 96000 { - t.Fatalf("post = %#v", post) - } -} - -func TestExtensionProviderAndManagerSelectionHelpers(t *testing.T) { - manifest := &ExtensionManifest{Capabilities: map[string]any{ - "replacesBuiltInProviders": []any{" Legacy-Provider ", 7, ""}, - }} - if values := manifestCapabilityStringList(manifest, "replacesBuiltInProviders"); len(values) != 1 || values[0] != "legacy-provider" { - t.Fatalf("capability list = %#v", values) - } - if !extensionReplacesBuiltInProvider(&loadedExtension{Manifest: manifest}, "legacy-provider") || extensionReplacesBuiltInProvider(nil, "legacy-provider") { - t.Fatal("extension replacement mismatch") - } - if trimKnownProviderPrefix("Legacy-Provider:101", "legacy-provider") != "101" || trimKnownProviderPrefix("101", "legacy-provider") != "101" { - t.Fatal("trimKnownProviderPrefix mismatch") - } - if metadataTrackDedupKey(ExtTrackMetadata{ISRC: "usrc"}) != "isrc:USRC" || - metadataTrackDedupKey(ExtTrackMetadata{SpotifyID: "sp"}) != "spotify:sp" || - metadataTrackDedupKey(ExtTrackMetadata{ProviderID: "p", ID: "1"}) != "p:1" { - t.Fatal("metadata dedup key mismatch") - } - - manager := &extensionManager{extensions: map[string]*loadedExtension{}} - downloadExt := newTestLoadedExtension(t, ExtensionTypeDownloadProvider, ExtensionTypeMetadataProvider) - manager.extensions[downloadExt.ID] = downloadExt - if providers := manager.GetDownloadProviders(); len(providers) != 1 { - t.Fatalf("download providers = %#v", providers) - } - SetProviderPriority([]string{"deezer", "coverage-ext", "coverage-ext", " "}) - if priority := GetProviderPriority(); len(priority) != 1 || priority[0] != "coverage-ext" { - t.Fatalf("provider priority = %#v", priority) - } - SetExtensionFallbackProviderIDs([]string{"a", "a", " ", "b"}) - if ids := GetExtensionFallbackProviderIDs(); len(ids) != 2 || !isExtensionFallbackAllowed("a") || isExtensionFallbackAllowed("z") { - t.Fatalf("fallback ids = %#v", ids) - } - SetExtensionFallbackProviderIDs(nil) - if !isExtensionFallbackAllowed("z") { - t.Fatal("nil fallback list should allow all") - } - SetMetadataProviderPriority([]string{"spotify", "deezer", "coverage-ext", "coverage-ext"}) - if priority := GetMetadataProviderPriority(); len(priority) != 1 || priority[0] != "coverage-ext" { - t.Fatalf("metadata priority = %#v", priority) - } -} diff --git a/go_backend/extension_provider_types.go b/go_backend/extension_provider_types.go deleted file mode 100644 index 31e1d513..00000000 --- a/go_backend/extension_provider_types.go +++ /dev/null @@ -1,217 +0,0 @@ -package gobackend - -import "strings" - -type ExtTrackMetadata struct { - ID string `json:"id"` - Name string `json:"name"` - Artists string `json:"artists"` - AlbumName string `json:"album_name"` - AlbumArtist string `json:"album_artist,omitempty"` - AlbumID string `json:"album_id,omitempty"` - AlbumURL string `json:"album_url,omitempty"` - ArtistID string `json:"artist_id,omitempty"` - ArtistURL string `json:"artist_url,omitempty"` - ExternalURL string `json:"external_urls,omitempty"` - DurationMS int `json:"duration_ms"` - CoverURL string `json:"cover_url,omitempty"` - PreviewURL string `json:"preview_url,omitempty"` - Images string `json:"images,omitempty"` - ReleaseDate string `json:"release_date,omitempty"` - TrackNumber int `json:"track_number,omitempty"` - TotalTracks int `json:"total_tracks,omitempty"` - DiscNumber int `json:"disc_number,omitempty"` - TotalDiscs int `json:"total_discs,omitempty"` - ISRC string `json:"isrc,omitempty"` - ProviderID string `json:"provider_id"` - ItemType string `json:"item_type,omitempty"` - AlbumType string `json:"album_type,omitempty"` - Explicit bool `json:"explicit,omitempty"` - UPC string `json:"upc,omitempty"` - - TidalID string `json:"tidal_id,omitempty"` - QobuzID string `json:"qobuz_id,omitempty"` - DeezerID string `json:"deezer_id,omitempty"` - SpotifyID string `json:"spotify_id,omitempty"` - ExternalLinks map[string]string `json:"external_links,omitempty"` - - Label string `json:"label,omitempty"` - Copyright string `json:"copyright,omitempty"` - Genre string `json:"genre,omitempty"` - Composer string `json:"composer,omitempty"` - Comment string `json:"comment,omitempty"` - - AudioQuality string `json:"audio_quality,omitempty"` - AudioModes string `json:"audio_modes,omitempty"` -} - -func (t *ExtTrackMetadata) ResolvedCoverURL() string { - if t.CoverURL != "" { - return t.CoverURL - } - return t.Images -} - -type ExtAlbumMetadata struct { - ID string `json:"id"` - Name string `json:"name"` - Artists string `json:"artists"` - ArtistID string `json:"artist_id,omitempty"` - CoverURL string `json:"cover_url,omitempty"` - HeaderImage string `json:"header_image,omitempty"` - HeaderVideo string `json:"header_video,omitempty"` - ReleaseDate string `json:"release_date,omitempty"` - TotalTracks int `json:"total_tracks"` - AlbumType string `json:"album_type,omitempty"` - AudioTraits []string `json:"audio_traits,omitempty"` - Tracks []ExtTrackMetadata `json:"tracks"` - ProviderID string `json:"provider_id"` -} - -type ExtArtistMetadata struct { - ID string `json:"id"` - Name string `json:"name"` - ImageURL string `json:"image_url,omitempty"` - HeaderImage string `json:"header_image,omitempty"` - HeaderVideo string `json:"header_video,omitempty"` - Listeners int `json:"listeners,omitempty"` - Albums []ExtAlbumMetadata `json:"albums,omitempty"` - Releases []ExtAlbumMetadata `json:"releases,omitempty"` - TopTracks []ExtTrackMetadata `json:"top_tracks,omitempty"` - ProviderID string `json:"provider_id"` -} - -type ExtSearchResult struct { - Tracks []ExtTrackMetadata `json:"tracks"` - Total int `json:"total"` -} - -type ExtAvailabilityResult struct { - Available bool `json:"available"` - Reason string `json:"reason,omitempty"` - TrackID string `json:"track_id,omitempty"` - SkipFallback bool `json:"skip_fallback,omitempty"` - PreparedContext map[string]any `json:"prepared_context,omitempty"` -} - -type DownloadDecryptionInfo struct { - Strategy string `json:"strategy,omitempty"` - Key string `json:"key,omitempty"` - IV string `json:"iv,omitempty"` - InputFormat string `json:"input_format,omitempty"` - OutputExtension string `json:"output_extension,omitempty"` - Options map[string]any `json:"options,omitempty"` -} - -type ExtDownloadResult struct { - Success bool `json:"success"` - FilePath string `json:"file_path,omitempty"` - AlreadyExists bool `json:"already_exists,omitempty"` - BitDepth int `json:"bit_depth,omitempty"` - SampleRate int `json:"sample_rate,omitempty"` - AudioCodec string `json:"audio_codec,omitempty"` - DurationMS int `json:"duration_ms,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - ErrorType string `json:"error_type,omitempty"` - RetryAfterSeconds int `json:"retry_after_seconds,omitempty"` - - Title string `json:"title,omitempty"` - Artist string `json:"artist,omitempty"` - Album string `json:"album,omitempty"` - AlbumArtist string `json:"album_artist,omitempty"` - TrackNumber int `json:"track_number,omitempty"` - DiscNumber int `json:"disc_number,omitempty"` - TotalTracks int `json:"total_tracks,omitempty"` - TotalDiscs int `json:"total_discs,omitempty"` - ReleaseDate string `json:"release_date,omitempty"` - CoverURL string `json:"cover_url,omitempty"` - ISRC string `json:"isrc,omitempty"` - Genre string `json:"genre,omitempty"` - Label string `json:"label,omitempty"` - Copyright string `json:"copyright,omitempty"` - Composer string `json:"composer,omitempty"` - Comment string `json:"comment,omitempty"` - Explicit bool `json:"explicit,omitempty"` - AlbumType string `json:"album_type,omitempty"` - UPC string `json:"upc,omitempty"` - LyricsLRC string `json:"lyrics_lrc,omitempty"` - DecryptionKey string `json:"decryption_key,omitempty"` - Decryption *DownloadDecryptionInfo `json:"decryption,omitempty"` - ActualExtension string `json:"actual_extension,omitempty"` - OutputExtension string `json:"output_extension,omitempty"` - ActualContainer string `json:"actual_container,omitempty"` - RequiresContainerConversion bool `json:"requires_container_conversion,omitempty"` -} - -const genericFFmpegMOVDecryptionStrategy = "ffmpeg.mov_key" - -func cloneDownloadDecryptionInfo(info *DownloadDecryptionInfo) *DownloadDecryptionInfo { - if info == nil { - return nil - } - - cloned := &DownloadDecryptionInfo{ - Strategy: strings.TrimSpace(info.Strategy), - Key: strings.TrimSpace(info.Key), - IV: strings.TrimSpace(info.IV), - InputFormat: strings.TrimSpace(info.InputFormat), - OutputExtension: strings.TrimSpace(info.OutputExtension), - } - if len(info.Options) > 0 { - cloned.Options = make(map[string]any, len(info.Options)) - for key, value := range info.Options { - cloned.Options[key] = value - } - } - return cloned -} - -func normalizeDownloadDecryptionStrategy(strategy string) string { - switch strings.ToLower(strings.TrimSpace(strategy)) { - case "", "ffmpeg.mov_key", "ffmpeg_mov_key", "mov_decryption_key", "mp4_decryption_key", "ffmpeg.mp4_decryption_key": - return genericFFmpegMOVDecryptionStrategy - default: - return strings.TrimSpace(strategy) - } -} - -func normalizeDownloadDecryptionInfo(info *DownloadDecryptionInfo, legacyKey string) *DownloadDecryptionInfo { - normalized := cloneDownloadDecryptionInfo(info) - trimmedLegacyKey := strings.TrimSpace(legacyKey) - - if normalized == nil { - if trimmedLegacyKey == "" { - return nil - } - return &DownloadDecryptionInfo{ - Strategy: genericFFmpegMOVDecryptionStrategy, - Key: trimmedLegacyKey, - InputFormat: "mov", - } - } - - normalized.Strategy = normalizeDownloadDecryptionStrategy(normalized.Strategy) - if normalized.Key == "" && trimmedLegacyKey != "" { - normalized.Key = trimmedLegacyKey - } - if normalized.Strategy == "" && normalized.Key != "" { - normalized.Strategy = genericFFmpegMOVDecryptionStrategy - } - if normalized.Strategy == genericFFmpegMOVDecryptionStrategy && normalized.InputFormat == "" { - normalized.InputFormat = "mov" - } - if normalized.Strategy == genericFFmpegMOVDecryptionStrategy && normalized.Key == "" { - return nil - } - - return normalized -} - -func normalizedDownloadDecryptionKey(info *DownloadDecryptionInfo, legacyKey string) string { - if normalized := normalizeDownloadDecryptionInfo(info, legacyKey); normalized != nil { - if normalized.Strategy == genericFFmpegMOVDecryptionStrategy { - return normalized.Key - } - } - return strings.TrimSpace(legacyKey) -} diff --git a/go_backend/extension_provider_wrapper.go b/go_backend/extension_provider_wrapper.go deleted file mode 100644 index a73fbca3..00000000 --- a/go_backend/extension_provider_wrapper.go +++ /dev/null @@ -1,1095 +0,0 @@ -package gobackend - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "github.com/dop251/goja" -) - -type extensionProviderWrapper struct { - extension *loadedExtension - vm *goja.Runtime -} - -func newExtensionProviderWrapper(ext *loadedExtension) *extensionProviderWrapper { - return &extensionProviderWrapper{ - extension: ext, - vm: ext.VM, - } -} - -func (p *extensionProviderWrapper) lockReadyVM() error { - vm, err := p.extension.lockReadyVM() - if err != nil { - return err - } - p.vm = vm - return nil -} - -// extCallOpts configures a shared extension invocation. It covers the -// skeleton common to most extensionProviderWrapper methods: perf tracking, VM -// locking, optional download/request cancellation binding, and translating -// timeouts/cancellation into the right error. -type extCallOpts struct { - perfName string - invoke func(vm *goja.Runtime) (goja.Value, error) - timeout time.Duration - itemID string // optional: binds download-cancel + active-item tracking - requestID string // optional: binds request-cancel via context (customSearch only) - context context.Context // optional: caller lifecycle for non-download work - // beforeRun runs after lock+cancel setup, right before the invocation. Its - // returned cleanup, if any, runs after the call. - beforeRun func() func() - // timeoutMessage overrides the default "<perfName> timeout: extension took - // too long to respond". - timeoutMessage string - // rawError returns non-timeout script errors unwrapped instead of - // "<perfName> failed: %w". - rawError bool -} - -// callExtension locks the extension's VM, runs opts.invoke, and hands -// the raw result to parse while the VM lock is still held. parse is where -// each caller does its type-specific parsing, perf.recordParse/setItems, and -// any ProviderID stamping. -func callExtension[T any](p *extensionProviderWrapper, opts extCallOpts, parse func(perf *extensionCallPerf, result goja.Value) (T, error)) (T, error) { - var zero T - ctx := opts.context - if ctx == nil { - ctx = context.Background() - } - - perf := newExtensionCallPerf(p.extension.ID, opts.perfName) - defer perf.finish() - initStartedAt := time.Now() - if err := p.lockReadyVM(); err != nil { - return zero, err - } - perf.recordInit(time.Since(initStartedAt)) - defer p.extension.VMMu.Unlock() - - if opts.itemID != "" { - if p.extension.runtime != nil { - p.extension.runtime.setActiveDownloadItemID(opts.itemID) - defer p.extension.runtime.clearActiveDownloadItemID() - } - ctx = initDownloadCancel(opts.itemID) - defer clearDownloadCancel(opts.itemID) - if isDownloadCancelled(opts.itemID) { - return zero, ErrDownloadCancelled - } - } - - if opts.requestID != "" { - if p.extension.runtime != nil { - p.extension.runtime.setActiveRequestID(opts.requestID) - defer p.extension.runtime.clearActiveRequestID() - } - ctx = initExtensionRequestCancel(opts.requestID) - defer clearExtensionRequestCancel(opts.requestID) - if isExtensionRequestCancelled(opts.requestID) { - return zero, ErrExtensionRequestCancelled - } - } - - if opts.beforeRun != nil { - if cleanup := opts.beforeRun(); cleanup != nil { - defer cleanup() - } - } - - jsStartedAt := time.Now() - result, err := runGojaCallWithTimeoutContextAndRecover(ctx, p.vm, func() (goja.Value, error) { - result, err := opts.invoke(p.vm) - if err != nil { - err = p.normalizePendingVerificationError(err) - } - return result, err - }, opts.timeout) - perf.recordJS(time.Since(jsStartedAt)) - perf.recordPayload(result) - if err != nil { - if IsRuntimeUnsafeError(err) { - quarantineRuntimeLocked(p.extension, p.vm, err) - } - if opts.requestID != "" && isExtensionRequestCancelled(opts.requestID) { - return zero, ErrExtensionRequestCancelled - } - if opts.itemID != "" && isDownloadCancelled(opts.itemID) { - return zero, ErrDownloadCancelled - } - if opts.requestID != "" && errors.Is(err, ErrExtensionRequestCancelled) { - return zero, ErrExtensionRequestCancelled - } - if IsTimeoutError(err) { - if opts.timeoutMessage != "" { - return zero, errors.New(opts.timeoutMessage) - } - return zero, fmt.Errorf("%s timeout: extension took too long to respond", opts.perfName) - } - if opts.rawError { - return zero, err - } - return zero, fmt.Errorf("%s failed: %w", opts.perfName, err) - } - if opts.itemID != "" && isDownloadCancelled(opts.itemID) { - return zero, ErrDownloadCancelled - } - if opts.requestID != "" && isExtensionRequestCancelled(opts.requestID) { - return zero, ErrExtensionRequestCancelled - } - - return parse(perf, result) -} - -func (p *extensionProviderWrapper) normalizePendingVerificationError(err error) error { - var exception *goja.Exception - if !errors.As(err, &exception) { - return err - } - // A script may rethrow an earlier challenge after the per-call runtime - // marker was cleared. Only trust an existing, fresh challenge for this - // extension; the exception alone must not start verification. - pending := GetPendingAuthRequest(p.extension.ID) - if pending == nil || pending.ExtensionID != p.extension.ID || strings.TrimSpace(pending.AuthURL) == "" { - return err - } - if age := time.Since(pending.CreatedAt); age < 0 || age >= pendingAuthRequestTTL { - return err - } - value := exception.Value() - if gojaValueIsEmpty(value) { - return err - } - var message string - if extractionErr := p.vm.Try(func() { - if object, ok := value.(*goja.Object); ok { - if field := object.Get("message"); !gojaValueIsEmpty(field) { - message = field.String() - } - } else { - message = value.String() - } - }); extractionErr != nil || strings.TrimSpace(message) != "VERIFY_REQUIRED" { - return err - } - return fmt.Errorf("verification_required: extension '%s' needs signed-session verification: %w", p.extension.ID, err) -} - -func invokeExtensionMethod(vm *goja.Runtime, method string, args ...any) (goja.Value, error) { - extensionValue := vm.Get("extension") - if gojaValueIsEmpty(extensionValue) { - return goja.Null(), nil - } - - extensionObject := extensionValue.ToObject(vm) - callable, ok := goja.AssertFunction(extensionObject.Get(method)) - if !ok { - return goja.Null(), nil - } - - return callable(extensionObject, gojaArgumentValues(vm, args)...) -} - -func gojaArgumentValues(vm *goja.Runtime, args []any) []goja.Value { - values := make([]goja.Value, len(args)) - for i, arg := range args { - values[i] = gojaArgumentValue(vm, arg) - } - return values -} - -func gojaArgumentValue(vm *goja.Runtime, value any) goja.Value { - switch typed := value.(type) { - case goja.Value: - return typed - case map[string]any: - obj := vm.NewObject() - for key, child := range typed { - _ = obj.Set(key, gojaArgumentValue(vm, child)) - } - return obj - case map[string]string: - obj := vm.NewObject() - for key, child := range typed { - _ = obj.Set(key, child) - } - return obj - case []any: - children := make([]any, len(typed)) - for i, child := range typed { - children[i] = gojaArgumentValue(vm, child) - } - return vm.NewArray(children...) - case []string: - children := make([]any, len(typed)) - for i, child := range typed { - children[i] = child - } - return vm.NewArray(children...) - default: - return vm.ToValue(value) - } -} - -func extensionMethodInvocation(method string, args ...any) func(*goja.Runtime) (goja.Value, error) { - return func(vm *goja.Runtime) (goja.Value, error) { - return invokeExtensionMethod(vm, method, args...) - } -} - -func hasExtensionMethod(vm *goja.Runtime, method string) bool { - extensionValue := vm.Get("extension") - if gojaValueIsEmpty(extensionValue) { - return false - } - _, ok := goja.AssertFunction(extensionValue.ToObject(vm).Get(method)) - return ok -} - -func invokeExtensionOrGlobal(vm *goja.Runtime, method string, args ...any) (goja.Value, error) { - if hasExtensionMethod(vm, method) { - return invokeExtensionMethod(vm, method, args...) - } - callable, ok := goja.AssertFunction(vm.Get(method)) - if !ok { - return goja.Null(), nil - } - return callable(vm.GlobalObject(), gojaArgumentValues(vm, args)...) -} - -func extensionTrackInput(track *ExtTrackMetadata) map[string]any { - if track == nil { - return map[string]any{} - } - return map[string]any{ - "id": track.ID, - "name": track.Name, - "artists": track.Artists, - "album_name": track.AlbumName, - "album_artist": track.AlbumArtist, - "album_id": track.AlbumID, - "album_url": track.AlbumURL, - "artist_id": track.ArtistID, - "artist_url": track.ArtistURL, - "external_urls": track.ExternalURL, - "duration_ms": track.DurationMS, - "cover_url": track.CoverURL, - "preview_url": track.PreviewURL, - "images": track.Images, - "release_date": track.ReleaseDate, - "track_number": track.TrackNumber, - "total_tracks": track.TotalTracks, - "disc_number": track.DiscNumber, - "total_discs": track.TotalDiscs, - "isrc": track.ISRC, - "provider_id": track.ProviderID, - "item_type": track.ItemType, - "album_type": track.AlbumType, - "explicit": track.Explicit, - "tidal_id": track.TidalID, - "qobuz_id": track.QobuzID, - "deezer_id": track.DeezerID, - "spotify_id": track.SpotifyID, - "external_links": track.ExternalLinks, - "label": track.Label, - "copyright": track.Copyright, - "genre": track.Genre, - "composer": track.Composer, - "comment": track.Comment, - "audio_quality": track.AudioQuality, - "audio_modes": track.AudioModes, - "upc": track.UPC, - } -} - -func (p *extensionProviderWrapper) SearchTracks(query string, limit int) (*ExtSearchResult, error) { - return p.SearchTracksForItemID(query, limit, "") -} - -func (p *extensionProviderWrapper) SearchTracksForItemID(query string, limit int, itemID string) (*ExtSearchResult, error) { - if !p.extension.Manifest.IsMetadataProvider() { - return nil, fmt.Errorf("extension '%s' is not a metadata provider", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - - return callExtension(p, extCallOpts{ - perfName: "searchTracks", - invoke: extensionMethodInvocation("searchTracks", query, limit), - timeout: DefaultJSTimeout, - itemID: itemID, - }, func(perf *extensionCallPerf, result goja.Value) (*ExtSearchResult, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return nil, fmt.Errorf("searchTracks returned null") - } - parseStartedAt := time.Now() - searchResult, err := parseExtensionSearchResult(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - if err != nil { - return nil, fmt.Errorf("failed to parse search result: %w", err) - } - perf.setItems(len(searchResult.Tracks)) - - for i := range searchResult.Tracks { - searchResult.Tracks[i].ProviderID = p.extension.ID - } - - return &searchResult, nil - }) -} - -func (p *extensionProviderWrapper) GetTrack(trackID string) (*ExtTrackMetadata, error) { - if !p.extension.Manifest.IsMetadataProvider() { - return nil, fmt.Errorf("extension '%s' is not a metadata provider", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - - return callExtension(p, extCallOpts{ - perfName: "getTrack", - invoke: extensionMethodInvocation("getTrack", trackID), - timeout: DefaultJSTimeout, - }, func(perf *extensionCallPerf, result goja.Value) (*ExtTrackMetadata, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return nil, fmt.Errorf("getTrack returned null") - } - parseStartedAt := time.Now() - track := parseExtensionTrackValue(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - perf.setItems(1) - track.ProviderID = p.extension.ID - return &track, nil - }) -} - -func (p *extensionProviderWrapper) GetAlbum(albumID string) (*ExtAlbumMetadata, error) { - if !p.extension.Manifest.IsMetadataProvider() { - return nil, fmt.Errorf("extension '%s' is not a metadata provider", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - - return callExtension(p, extCallOpts{ - perfName: "getAlbum", - invoke: extensionMethodInvocation("getAlbum", albumID), - timeout: DefaultJSTimeout, - }, func(perf *extensionCallPerf, result goja.Value) (*ExtAlbumMetadata, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return nil, fmt.Errorf("getAlbum returned null") - } - parseStartedAt := time.Now() - album, err := parseExtensionAlbumValue(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - if err != nil { - return nil, fmt.Errorf("failed to parse album: %w", err) - } - perf.setItems(len(album.Tracks)) - - album.ProviderID = p.extension.ID - for i := range album.Tracks { - album.Tracks[i].ProviderID = p.extension.ID - } - return &album, nil - }) -} - -func (p *extensionProviderWrapper) GetPlaylist(playlistID string) (*ExtAlbumMetadata, error) { - if !p.extension.Manifest.IsMetadataProvider() { - return nil, fmt.Errorf("extension '%s' is not a metadata provider", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - - return callExtension(p, extCallOpts{ - perfName: "getPlaylist", - invoke: func(vm *goja.Runtime) (goja.Value, error) { - if hasExtensionMethod(vm, "getPlaylist") { - return invokeExtensionMethod(vm, "getPlaylist", playlistID) - } - return invokeExtensionMethod(vm, "getAlbum", playlistID) - }, - timeout: DefaultJSTimeout, - }, func(perf *extensionCallPerf, result goja.Value) (*ExtAlbumMetadata, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return nil, fmt.Errorf("getPlaylist returned null") - } - parseStartedAt := time.Now() - playlist, err := parseExtensionAlbumValue(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - if err != nil { - return nil, fmt.Errorf("failed to parse playlist: %w", err) - } - perf.setItems(len(playlist.Tracks)) - - playlist.ProviderID = p.extension.ID - for i := range playlist.Tracks { - playlist.Tracks[i].ProviderID = p.extension.ID - } - return &playlist, nil - }) -} - -func (p *extensionProviderWrapper) GetArtist(artistID string) (*ExtArtistMetadata, error) { - if !p.extension.Manifest.IsMetadataProvider() { - return nil, fmt.Errorf("extension '%s' is not a metadata provider", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - - return callExtension(p, extCallOpts{ - perfName: "getArtist", - invoke: extensionMethodInvocation("getArtist", artistID), - timeout: DefaultJSTimeout, - }, func(perf *extensionCallPerf, result goja.Value) (*ExtArtistMetadata, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return nil, fmt.Errorf("getArtist returned null") - } - parseStartedAt := time.Now() - artist, err := parseExtensionArtistValue(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - if err != nil { - return nil, fmt.Errorf("failed to parse artist: %w", err) - } - perf.setItems(len(artist.Albums) + len(artist.Releases) + len(artist.TopTracks)) - - artist.ProviderID = p.extension.ID - for i := range artist.Releases { - artist.Releases[i].ProviderID = p.extension.ID - for j := range artist.Releases[i].Tracks { - artist.Releases[i].Tracks[j].ProviderID = p.extension.ID - } - } - return &artist, nil - }) -} - -func (p *extensionProviderWrapper) EnrichTrack(track *ExtTrackMetadata) (*ExtTrackMetadata, error) { - return p.EnrichTrackForItemID(track, "") -} - -// EnrichTrackForItemID is excluded from the shared callExtension helper: -// unlike the other providers it must return the original track (not an error) -// on every failure path, which doesn't fit the helper's error-returning shape. -func (p *extensionProviderWrapper) EnrichTrackForItemID(track *ExtTrackMetadata, itemID string) (*ExtTrackMetadata, error) { - if !p.extension.Manifest.IsMetadataProvider() { - return track, nil - } - - if !p.extension.Enabled { - return track, nil - } - perf := newExtensionCallPerf(p.extension.ID, "enrichTrack") - defer perf.finish() - initStartedAt := time.Now() - if err := p.lockReadyVM(); err != nil { - GoLog("[Extension] EnrichTrack init error for %s: %v\n", p.extension.ID, err) - return track, nil - } - perf.recordInit(time.Since(initStartedAt)) - defer p.extension.VMMu.Unlock() - downloadCtx := context.Background() - if itemID != "" { - if p.extension.runtime != nil { - p.extension.runtime.setActiveDownloadItemID(itemID) - defer p.extension.runtime.clearActiveDownloadItemID() - } - downloadCtx = initDownloadCancel(itemID) - defer clearDownloadCancel(itemID) - if isDownloadCancelled(itemID) { - return track, ErrDownloadCancelled - } - } - - jsStartedAt := time.Now() - result, err := runGojaCallWithTimeoutContextAndRecover(downloadCtx, p.vm, func() (goja.Value, error) { - return invokeExtensionMethod(p.vm, "enrichTrack", extensionTrackInput(track)) - }, DefaultJSTimeout) - perf.recordJS(time.Since(jsStartedAt)) - perf.recordPayload(result) - if err != nil { - if IsRuntimeUnsafeError(err) { - quarantineRuntimeLocked(p.extension, p.vm, err) - } - if isDownloadCancelled(itemID) { - return track, ErrDownloadCancelled - } - if IsTimeoutError(err) { - GoLog("[Extension] EnrichTrack timeout for %s\n", p.extension.ID) - } else { - GoLog("[Extension] EnrichTrack error for %s: %v\n", p.extension.ID, err) - } - return track, nil - } - if isDownloadCancelled(itemID) { - return track, ErrDownloadCancelled - } - - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return track, nil - } - - parseStartedAt := time.Now() - enrichedTrack := parseExtensionTrackValue(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - perf.setItems(1) - enrichedTrack.ProviderID = track.ProviderID - - return &enrichedTrack, nil -} - -func (p *extensionProviderWrapper) CheckAvailabilityForItemID(isrc, trackName, artistName, spotifyID, deezerID, tidalID, qobuzID string, durationMS int, itemID string, trackContexts ...map[string]any) (*ExtAvailabilityResult, error) { - if !p.extension.Manifest.IsDownloadProvider() { - return nil, fmt.Errorf("extension '%s' is not a download provider", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - - availabilityOptions := map[string]any{ - "spotify_id": spotifyID, - "deezer_id": deezerID, - "tidal_id": tidalID, - "qobuz_id": qobuzID, - "duration_ms": durationMS, - } - if len(trackContexts) > 0 && len(trackContexts[0]) > 0 { - availabilityOptions["track"] = trackContexts[0] - } - - var availabilityRuntime *extensionRuntime - consumeVerificationError := func() error { - if availabilityRuntime != nil && availabilityRuntime.consumeVerificationRequired() != "" { - return fmt.Errorf( - "verification_required: extension '%s' needs signed-session verification", - p.extension.ID, - ) - } - return nil - } - - return callExtension(p, extCallOpts{ - perfName: "checkAvailability", - invoke: func(vm *goja.Runtime) (goja.Value, error) { - result, err := invokeExtensionMethod(vm, "checkAvailability", isrc, trackName, artistName, availabilityOptions) - // A thrown JS error must preserve the same canonical runtime evidence - // as a returned unavailable result. Cancellation and timeout still - // take precedence in callExtension. - if err != nil { - if verificationErr := consumeVerificationError(); verificationErr != nil { - return nil, verificationErr - } - } - return result, err - }, - timeout: DefaultJSTimeout, - itemID: itemID, - beforeRun: func() func() { - // Drop any stale flag so the post-run check below only sees - // verification requested by THIS call. - availabilityRuntime = p.extension.runtime - if availabilityRuntime != nil { - availabilityRuntime.consumeVerificationRequired() - } - return nil - }, - }, func(perf *extensionCallPerf, result goja.Value) (*ExtAvailabilityResult, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - if err := consumeVerificationError(); err != nil { - return nil, err - } - return &ExtAvailabilityResult{Available: false, Reason: "not implemented"}, nil - } - parseStartedAt := time.Now() - availability := parseExtensionAvailabilityValue(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - perf.setItems(1) - // A signed-session call inside checkAvailability required - // verification. Extensions often swallow that and report a plain - // "not available", which would silently skip this provider's - // challenge; surface it as an error so the fallback loop pauses and - // opens the challenge instead. - if !availability.Available { - if err := consumeVerificationError(); err != nil { - return nil, err - } - } - return &availability, nil - }) -} - -const ExtDownloadTimeout = DownloadTimeout - -// Download is excluded from the shared callExtension helper: it runs in -// an isolated VM/runtime (not p.vm/p.extension.VMMu) with a progress -// callback, which the helper's lock+perf model doesn't cover. -func (p *extensionProviderWrapper) Download(trackID, quality, outputPath, itemID string, onProgress func(percent int)) (*ExtDownloadResult, error) { - return p.DownloadPrepared( - trackID, - quality, - outputPath, - itemID, - nil, - onProgress, - ) -} - -// DownloadPrepared passes the opaque context returned by checkAvailability to -// the isolated download runtime. Existing extensions remain compatible because -// JavaScript ignores the additional options argument; extensions that opt in -// can reuse already-resolved metadata or stream preparation. -func (p *extensionProviderWrapper) DownloadPrepared( - trackID, quality, outputPath, itemID string, - preparedContext map[string]any, - onProgress func(percent int), -) (*ExtDownloadResult, error) { - if !p.extension.Manifest.IsDownloadProvider() { - return nil, fmt.Errorf("extension '%s' is not a download provider", p.extension.ID) - } - - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - perf := newExtensionCallPerf(p.extension.ID, "download") - defer perf.finish() - initStartedAt := time.Now() - vm, runtime, err := acquireIsolatedExtensionRuntime(p.extension) - perf.recordInit(time.Since(initStartedAt)) - if err != nil { - return &ExtDownloadResult{ - Success: false, - ErrorMessage: err.Error(), - ErrorType: "init_error", - }, nil - } - vmHealthy := false - cleanupSafe := true - var unsafeDone <-chan struct{} - defer func() { - releaseIsolatedExtensionRuntime( - p.extension, - vm, - runtime, - vmHealthy, - cleanupSafe, - unsafeDone, - ) - }() - if runtime != nil { - runtime.setActiveDownloadItemID(itemID) - defer runtime.clearActiveDownloadItemID() - } - downloadCtx := context.Background() - if itemID != "" { - downloadCtx = initDownloadCancel(itemID) - defer clearDownloadCancel(itemID) - SetItemPreparing(itemID) - } - - if runtime != nil { - var finishResolution func() - downloadCtx, finishResolution = runtime.beginResolutionBudget(downloadCtx, extensionResolutionTimeout) - defer finishResolution() - } - - progressCallback := vm.ToValue(func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) > 0 { - percent := int(call.Arguments[0].ToInteger()) - if percent < 0 { - percent = 0 - } - if percent > 100 { - percent = 100 - } - if onProgress != nil { - onProgress(percent) - } - } - return goja.Undefined() - }) - - if runtime != nil { - // Drop any stale flag (pooled runtimes survive across downloads) so - // the post-run check only sees verification from THIS call. - runtime.consumeVerificationRequired() - } - - jsStartedAt := time.Now() - downloadOptions := map[string]any{"resolutionTimeoutMs": extensionResolutionTimeout.Milliseconds()} - if len(preparedContext) > 0 { - downloadOptions["preparedContext"] = preparedContext - } - result, err := runGojaCallWithTimeoutContextAndRecover(downloadCtx, vm, func() (goja.Value, error) { - return invokeExtensionMethod( - vm, - "download", - trackID, - quality, - outputPath, - progressCallback, - downloadOptions, - ) - }, ExtDownloadTimeout) - perf.recordJS(time.Since(jsStartedAt)) - perf.recordPayload(result) - vmHealthy = err == nil - cleanupSafe = !IsRuntimeUnsafeError(err) - unsafeDone = runtimeCompletion(err) - if err != nil { - if itemID != "" && isDownloadCancelled(itemID) { - return nil, ErrDownloadCancelled - } - errMsg := err.Error() - errType := "script_error" - if IsTimeoutError(err) { - errMsg = "download timeout: extension took too long to complete" - if context.Cause(downloadCtx) == context.DeadlineExceeded { - errMsg = "stream resolution timeout: extension took too long to resolve an audio stream" - } - errType = "timeout" - } - return &ExtDownloadResult{ - Success: false, - ErrorMessage: errMsg, - ErrorType: errType, - }, nil - } - - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return &ExtDownloadResult{ - Success: false, - ErrorMessage: "download returned null", - ErrorType: "not_implemented", - }, nil - } - - parseStartedAt := time.Now() - downloadResult := parseExtensionDownloadResultValue(vm, result) - perf.recordParse(time.Since(parseStartedAt)) - perf.setItems(1) - downloadResult.Decryption = normalizeDownloadDecryptionInfo( - downloadResult.Decryption, - downloadResult.DecryptionKey, - ) - downloadResult.DecryptionKey = normalizedDownloadDecryptionKey( - downloadResult.Decryption, - downloadResult.DecryptionKey, - ) - - // A signed-session call inside download() required verification but the - // script reported a generic failure; tag the result so the fallback loop - // pauses and opens this provider's challenge instead of skipping it. - if runtime != nil && !downloadResult.Success { - if runtime.consumeVerificationRequired() != "" && - !strings.EqualFold(downloadResult.ErrorType, "verification_required") { - downloadResult.ErrorType = "verification_required" - if downloadResult.ErrorMessage == "" { - downloadResult.ErrorMessage = "Verification required" - } - } - } - - return &downloadResult, nil -} - -func (p *extensionProviderWrapper) CustomSearch(query string, options map[string]any) ([]ExtTrackMetadata, error) { - return p.customSearch(query, options, "", "") -} - -func (p *extensionProviderWrapper) CustomSearchForRequestID(query string, options map[string]any, requestID string) ([]ExtTrackMetadata, error) { - return p.customSearch(query, options, "", requestID) -} - -func (p *extensionProviderWrapper) customSearch(query string, options map[string]any, itemID, requestID string) ([]ExtTrackMetadata, error) { - if !p.extension.Manifest.HasCustomSearch() { - return nil, fmt.Errorf("extension '%s' does not support custom search", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - if options == nil { - options = map[string]any{} - } - - return callExtension(p, extCallOpts{ - perfName: "customSearch", - invoke: extensionMethodInvocation("customSearch", query, options), - timeout: DefaultJSTimeout, - itemID: itemID, - requestID: requestID, - }, func(perf *extensionCallPerf, result goja.Value) ([]ExtTrackMetadata, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return []ExtTrackMetadata{}, nil - } - parseStartedAt := time.Now() - tracks, err := parseExtensionTrackArray(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - if err != nil { - return nil, fmt.Errorf("failed to parse search result: %w", err) - } - perf.setItems(len(tracks)) - - for i := range tracks { - tracks[i].ProviderID = p.extension.ID - } - - return tracks, nil - }) -} - -type ExtURLHandleResult struct { - Type string `json:"type"` - // ID identifies the handled resource itself (e.g. a playlist ID). Track, - // album, and artist results already carry their own ID inside their - // nested metadata; this covers result types with no such nested object. - ID string `json:"id,omitempty"` - Track *ExtTrackMetadata `json:"track,omitempty"` - Tracks []ExtTrackMetadata `json:"tracks,omitempty"` - Album *ExtAlbumMetadata `json:"album,omitempty"` - Artist *ExtArtistMetadata `json:"artist,omitempty"` - Name string `json:"name,omitempty"` - CoverURL string `json:"cover_url,omitempty"` - HeaderImage string `json:"header_image,omitempty"` - HeaderVideo string `json:"header_video,omitempty"` -} - -func (p *extensionProviderWrapper) HandleURL(url string) (*ExtURLHandleResult, error) { - if !p.extension.Manifest.HasURLHandler() { - return nil, fmt.Errorf("extension '%s' does not support URL handling", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - - return callExtension(p, extCallOpts{ - perfName: "handleUrl", - invoke: extensionMethodInvocation("handleUrl", url), - timeout: DefaultJSTimeout, - }, func(perf *extensionCallPerf, result goja.Value) (*ExtURLHandleResult, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return nil, fmt.Errorf("handleUrl returned null - URL not recognized") - } - parseStartedAt := time.Now() - handleResult, err := parseExtensionURLHandleValue(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - if err != nil { - return nil, fmt.Errorf("failed to parse URL handle result: %w", err) - } - urlItems := len(handleResult.Tracks) - if handleResult.Track != nil { - urlItems++ - } - if handleResult.Album != nil { - urlItems += 1 + len(handleResult.Album.Tracks) - } - if handleResult.Artist != nil { - urlItems += 1 + len(handleResult.Artist.Albums) + len(handleResult.Artist.Releases) + len(handleResult.Artist.TopTracks) - } - perf.setItems(urlItems) - - if handleResult.Track != nil { - handleResult.Track.ProviderID = p.extension.ID - } - for i := range handleResult.Tracks { - handleResult.Tracks[i].ProviderID = p.extension.ID - } - if handleResult.Album != nil { - handleResult.Album.ProviderID = p.extension.ID - for i := range handleResult.Album.Tracks { - handleResult.Album.Tracks[i].ProviderID = p.extension.ID - } - } - if handleResult.Artist != nil { - handleResult.Artist.ProviderID = p.extension.ID - for i := range handleResult.Artist.Albums { - handleResult.Artist.Albums[i].ProviderID = p.extension.ID - for j := range handleResult.Artist.Albums[i].Tracks { - handleResult.Artist.Albums[i].Tracks[j].ProviderID = p.extension.ID - } - } - for i := range handleResult.Artist.Releases { - handleResult.Artist.Releases[i].ProviderID = p.extension.ID - for j := range handleResult.Artist.Releases[i].Tracks { - handleResult.Artist.Releases[i].Tracks[j].ProviderID = p.extension.ID - } - } - for i := range handleResult.Artist.TopTracks { - handleResult.Artist.TopTracks[i].ProviderID = p.extension.ID - } - } - - return &handleResult, nil - }) -} - -type PostProcessResult struct { - Success bool `json:"success"` - NewFilePath string `json:"new_file_path,omitempty"` - NewFileURI string `json:"new_file_uri,omitempty"` - Error string `json:"error,omitempty"` - BitDepth int `json:"bit_depth,omitempty"` - SampleRate int `json:"sample_rate,omitempty"` -} - -type PostProcessInput struct { - Path string `json:"path,omitempty"` - URI string `json:"uri,omitempty"` - Name string `json:"name,omitempty"` - MimeType string `json:"mime_type,omitempty"` - Size int64 `json:"size,omitempty"` - IsSAF bool `json:"is_saf,omitempty"` -} - -func postProcessInputMap(input PostProcessInput) map[string]any { - result := make(map[string]any, 6) - if input.Path != "" { - result["path"] = input.Path - } - if input.URI != "" { - result["uri"] = input.URI - } - if input.Name != "" { - result["name"] = input.Name - } - if input.MimeType != "" { - result["mime_type"] = input.MimeType - } - if input.Size != 0 { - result["size"] = input.Size - } - if input.IsSAF { - result["is_saf"] = true - } - return result -} - -const PostProcessTimeout = 2 * time.Minute - -// PostProcessV2 probes extension.postProcessV2 first, then falls back to the -// legacy extension.postProcess contract. -func (p *extensionProviderWrapper) PostProcessV2(input PostProcessInput, metadata map[string]any, hookID string) (*PostProcessResult, error) { - if !p.extension.Manifest.HasPostProcessing() { - return nil, fmt.Errorf("extension '%s' does not support post-processing", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - - filePath := input.Path - - perfName := "postProcessV2" - inputMap := postProcessInputMap(input) - invoke := func(vm *goja.Runtime) (goja.Value, error) { - if hasExtensionMethod(vm, "postProcessV2") { - return invokeExtensionMethod(vm, "postProcessV2", inputMap, metadata, hookID) - } - return invokeExtensionMethod(vm, "postProcess", filePath, metadata, hookID) - } - - result, err := callExtension(p, extCallOpts{ - perfName: perfName, - invoke: invoke, - timeout: PostProcessTimeout, - timeoutMessage: "postProcess timeout: extension took too long to complete", - rawError: true, - }, func(perf *extensionCallPerf, value goja.Value) (*PostProcessResult, error) { - if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { - return &PostProcessResult{Success: false, Error: "postProcess returned null"}, nil - } - parseStartedAt := time.Now() - postResult := parseExtensionPostProcessValue(p.vm, value) - perf.recordParse(time.Since(parseStartedAt)) - perf.setItems(1) - return &postResult, nil - }) - if err != nil { - return &PostProcessResult{Success: false, Error: err.Error()}, nil - } - return result, nil -} - -type ExtLyricsResult struct { - Lines []ExtLyricsLine `json:"lines"` - SyncType string `json:"syncType"` - Instrumental bool `json:"instrumental"` - PlainLyrics string `json:"plainLyrics"` - Provider string `json:"provider"` -} - -type ExtLyricsLine struct { - StartTimeMs int64 `json:"startTimeMs"` - Words string `json:"words"` - EndTimeMs int64 `json:"endTimeMs"` -} - -func (p *extensionProviderWrapper) FetchLyrics(trackName, artistName, albumName string, durationSec float64) (*LyricsResponse, error) { - return p.FetchLyricsContext(context.Background(), trackName, artistName, albumName, durationSec) -} - -func (p *extensionProviderWrapper) FetchLyricsContext(ctx context.Context, trackName, artistName, albumName string, durationSec float64) (*LyricsResponse, error) { - if !p.extension.Manifest.IsLyricsProvider() { - return nil, fmt.Errorf("extension '%s' is not a lyrics provider", p.extension.ID) - } - if !p.extension.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", p.extension.ID) - } - - return callExtension(p, extCallOpts{ - perfName: "fetchLyrics", - invoke: extensionMethodInvocation("fetchLyrics", trackName, artistName, albumName, durationSec), - timeout: DefaultJSTimeout, - context: ctx, - }, func(perf *extensionCallPerf, result goja.Value) (*LyricsResponse, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return nil, fmt.Errorf("fetchLyrics returned null") - } - parseStartedAt := time.Now() - extResult, err := parseExtensionLyricsValue(p.vm, result) - perf.recordParse(time.Since(parseStartedAt)) - if err != nil { - return nil, fmt.Errorf("failed to parse lyrics result: %w", err) - } - perf.setItems(len(extResult.Lines)) - - response := &LyricsResponse{ - SyncType: extResult.SyncType, - Instrumental: extResult.Instrumental, - PlainLyrics: extResult.PlainLyrics, - Provider: extResult.Provider, - Source: "Extension: " + p.extension.ID, - } - - if response.Provider == "" { - response.Provider = p.extension.Manifest.DisplayName - } - - for _, line := range extResult.Lines { - response.Lines = append(response.Lines, LyricsLine(line)) - } - - if len(response.Lines) == 0 && response.PlainLyrics != "" && !response.Instrumental { - response.SyncType = "UNSYNCED" - for _, line := range strings.Split(response.PlainLyrics, "\n") { - if strings.TrimSpace(line) != "" { - response.Lines = append(response.Lines, LyricsLine{ - StartTimeMs: 0, - Words: line, - EndTimeMs: 0, - }) - } - } - } - - return response, nil - }) -} diff --git a/go_backend/extension_providers.go b/go_backend/extension_providers.go deleted file mode 100644 index 52ece19e..00000000 --- a/go_backend/extension_providers.go +++ /dev/null @@ -1,375 +0,0 @@ -package gobackend - -import ( - "errors" - "fmt" - "path/filepath" - "sort" - "strings" -) - -func (m *extensionManager) GetMetadataProviders() []*extensionProviderWrapper { - m.mu.RLock() - defer m.mu.RUnlock() - - var providers []*extensionProviderWrapper - for _, ext := range m.extensions { - if ext.Enabled && ext.Manifest.IsMetadataProvider() && ext.Error == "" { - providers = append(providers, newExtensionProviderWrapper(ext)) - } - } - return providers -} - -func (m *extensionManager) GetDownloadProviders() []*extensionProviderWrapper { - m.mu.RLock() - defer m.mu.RUnlock() - - var providers []*extensionProviderWrapper - for _, ext := range m.extensions { - if ext.Enabled && ext.Manifest.IsDownloadProvider() && ext.Error == "" { - providers = append(providers, newExtensionProviderWrapper(ext)) - } - } - return providers -} - -func metadataTrackDedupKey(track ExtTrackMetadata) string { - if isrc := strings.TrimSpace(track.ISRC); isrc != "" { - return "isrc:" + strings.ToUpper(isrc) - } - if spotifyID := strings.TrimSpace(track.SpotifyID); spotifyID != "" { - return "spotify:" + spotifyID - } - if providerID := strings.TrimSpace(track.ProviderID); providerID != "" && strings.TrimSpace(track.ID) != "" { - return providerID + ":" + strings.TrimSpace(track.ID) - } - return strings.TrimSpace(track.Name) + "|" + strings.TrimSpace(track.Artists) -} - -func (m *extensionManager) SearchTracksWithMetadataProviders(query string, limit int, includeExtensions bool) ([]ExtTrackMetadata, error) { - return m.SearchTracksWithMetadataProvidersForItemID(query, limit, includeExtensions, "") -} - -// SearchTracksWithMetadataProvider searches one explicitly selected metadata -// provider. Unlike the priority-based search, this never falls through to a -// different extension, so callers can reliably attribute the returned fields -// to the provider selected by the user. -func (m *extensionManager) SearchTracksWithMetadataProvider(providerID, query string, limit int) ([]ExtTrackMetadata, error) { - providerID = strings.TrimSpace(providerID) - if providerID == "" { - return nil, fmt.Errorf("metadata provider ID is required") - } - if limit <= 0 { - limit = 20 - } - - ext, err := m.GetExtension(providerID) - if err != nil { - return nil, err - } - if ext == nil || ext.Manifest == nil || !ext.Manifest.IsMetadataProvider() { - return nil, fmt.Errorf("extension '%s' is not a metadata provider", providerID) - } - if !ext.Enabled { - return nil, fmt.Errorf("extension '%s' is disabled", providerID) - } - if ext.Error != "" { - return nil, fmt.Errorf("extension '%s' is unavailable: %s", providerID, ext.Error) - } - - result, err := newExtensionProviderWrapper(ext).SearchTracks(query, limit) - if err != nil { - return nil, err - } - if result == nil || len(result.Tracks) <= limit { - if result == nil { - return []ExtTrackMetadata{}, nil - } - return result.Tracks, nil - } - return result.Tracks[:limit], nil -} - -func (m *extensionManager) SearchTracksWithMetadataProvidersForItemID(query string, limit int, includeExtensions bool, itemID string) ([]ExtTrackMetadata, error) { - priority := GetMetadataProviderPriority() - if limit <= 0 { - limit = 20 - } - - extensionProviders := make(map[string]*extensionProviderWrapper) - if includeExtensions { - for _, provider := range m.GetMetadataProviders() { - extensionProviders[provider.extension.ID] = provider - } - } - - orderedProviderIDs := make([]string, 0, len(priority)+len(extensionProviders)) - seenProviderIDs := make(map[string]struct{}, len(priority)+len(extensionProviders)) - for _, providerID := range priority { - providerID = strings.TrimSpace(providerID) - if providerID == "" { - continue - } - orderedProviderIDs = append(orderedProviderIDs, providerID) - seenProviderIDs[providerID] = struct{}{} - } - if includeExtensions { - remainingIDs := make([]string, 0, len(extensionProviders)) - for providerID := range extensionProviders { - if _, exists := seenProviderIDs[providerID]; exists { - continue - } - remainingIDs = append(remainingIDs, providerID) - } - sort.Strings(remainingIDs) - orderedProviderIDs = append(orderedProviderIDs, remainingIDs...) - } - - tracks := make([]ExtTrackMetadata, 0, limit) - seenTracks := make(map[string]struct{}) - var verificationErr error - for _, providerID := range orderedProviderIDs { - if isDownloadCancelled(itemID) { - return nil, ErrDownloadCancelled - } - - if !includeExtensions { - continue - } - provider := extensionProviders[providerID] - if provider == nil { - continue - } - result, err := provider.SearchTracksForItemID(query, limit, itemID) - providerTracks := []ExtTrackMetadata(nil) - if result != nil { - providerTracks = result.Tracks - } - - if err != nil { - if errors.Is(err, ErrDownloadCancelled) { - return nil, ErrDownloadCancelled - } - if verificationErr == nil && strings.EqualFold(classifyDownloadErrorType(err.Error()), "verification_required") { - verificationErr = fmt.Errorf( - "verification_required: extension '%s' needs verification: %w", - providerID, - err, - ) - } - GoLog("[MetadataSearch] Search error from %s: %v\n", providerID, err) - continue - } - - for _, track := range providerTracks { - key := metadataTrackDedupKey(track) - if key == "" { - continue - } - if _, exists := seenTracks[key]; exists { - continue - } - seenTracks[key] = struct{}{} - tracks = append(tracks, track) - if len(tracks) >= limit { - return tracks, nil - } - } - } - if len(tracks) == 0 && verificationErr != nil { - return nil, verificationErr - } - - return tracks, nil -} - -// FindURLHandler returns the enabled handler matching the URL. When several -// extensions match (e.g. two Spotify handlers), the user's metadata provider -// priority breaks the tie deterministically instead of Go's random map -// iteration order. -func (m *extensionManager) FindURLHandler(url string) *extensionProviderWrapper { - m.mu.RLock() - matches := make([]*loadedExtension, 0, 2) - for _, ext := range m.extensions { - if ext.Enabled && ext.Manifest.MatchesURL(url) && ext.Error == "" { - matches = append(matches, ext) - } - } - m.mu.RUnlock() - - if len(matches) == 0 { - return nil - } - if len(matches) > 1 { - rank := map[string]int{} - for i, id := range GetMetadataProviderPriority() { - rank[strings.ToLower(strings.TrimSpace(id))] = i - } - sort.SliceStable(matches, func(i, j int) bool { - ri, oki := rank[strings.ToLower(matches[i].ID)] - rj, okj := rank[strings.ToLower(matches[j].ID)] - switch { - case oki && okj: - return ri < rj - case oki: - return true - case okj: - return false - default: - return matches[i].ID < matches[j].ID - } - }) - } - return newExtensionProviderWrapper(matches[0]) -} - -type ExtURLHandleResultWithExtID struct { - Result *ExtURLHandleResult - ExtensionID string -} - -func (m *extensionManager) HandleURLWithExtension(url string) (*ExtURLHandleResultWithExtID, error) { - handler := m.FindURLHandler(url) - if handler == nil { - return nil, fmt.Errorf("no extension found to handle URL: %s", url) - } - - result, err := handler.HandleURL(url) - if err != nil { - return &ExtURLHandleResultWithExtID{ - Result: nil, - ExtensionID: handler.extension.ID, - }, err - } - - return &ExtURLHandleResultWithExtID{ - Result: result, - ExtensionID: handler.extension.ID, - }, nil -} - -func (m *extensionManager) GetPostProcessingProviders() []*extensionProviderWrapper { - m.mu.RLock() - defer m.mu.RUnlock() - - var providers []*extensionProviderWrapper - for _, ext := range m.extensions { - if ext.Enabled && ext.Manifest.HasPostProcessing() && ext.Error == "" { - providers = append(providers, newExtensionProviderWrapper(ext)) - } - } - return providers -} - -func (m *extensionManager) RunPostProcessingV2(input PostProcessInput, metadata map[string]any) (*PostProcessResult, error) { - providers := m.GetPostProcessingProviders() - if len(providers) == 0 { - return &PostProcessResult{Success: true, NewFilePath: input.Path, NewFileURI: input.URI}, nil - } - - logTag := "[PostProcessV2]" - - currentInput := input - for _, provider := range providers { - hooks := provider.extension.Manifest.GetPostProcessingHooks() - for _, hook := range hooks { - if !hook.DefaultEnabled { - continue - } - - ext := strings.ToLower(filepath.Ext(currentInput.Path)) - if ext == "" && currentInput.Name != "" { - ext = strings.ToLower(filepath.Ext(currentInput.Name)) - } - if len(hook.SupportedFormats) > 0 && ext != "" { - supported := false - for _, format := range hook.SupportedFormats { - if "."+format == ext || format == ext[1:] { - supported = true - break - } - } - if !supported { - continue - } - } - - GoLog("%s Running hook %s from %s on %s\n", logTag, hook.ID, provider.extension.ID, currentInput.Path) - - result, err := provider.PostProcessV2(currentInput, metadata, hook.ID) - if err != nil { - GoLog("%s Hook %s failed: %v\n", logTag, hook.ID, err) - continue - } - if result.Success { - if err := validatePostProcessResult(provider.extension, currentInput, result); err != nil { - GoLog("%s Hook %s returned an unsafe result: %v\n", logTag, hook.ID, err) - continue - } - } - - if result.Success && result.NewFilePath != "" { - currentInput.Path = result.NewFilePath - if currentInput.Name == "" { - currentInput.Name = filepath.Base(result.NewFilePath) - } - } - if result.Success && result.NewFileURI != "" { - currentInput.URI = result.NewFileURI - } - } - } - - return &PostProcessResult{Success: true, NewFilePath: currentInput.Path, NewFileURI: currentInput.URI}, nil -} - -func validatePostProcessResult(ext *loadedExtension, input PostProcessInput, result *PostProcessResult) error { - if ext == nil || ext.Manifest == nil || result == nil { - return fmt.Errorf("invalid post-processing result") - } - if result.NewFileURI != "" && result.NewFileURI != input.URI { - return fmt.Errorf("an extension cannot replace the destination URI") - } - if result.NewFilePath == "" || filepath.Clean(result.NewFilePath) == filepath.Clean(input.Path) { - return nil - } - if !ext.Manifest.Permissions.File { - return fmt.Errorf("file permission is required to replace the processed file") - } - if !filepath.IsAbs(result.NewFilePath) { - return fmt.Errorf("replacement file path must be absolute") - } - if input.Path != "" && isPathWithinBase(filepath.Dir(input.Path), result.NewFilePath) { - return nil - } - if ext.DataDir != "" && isPathWithinBase(ext.DataDir, result.NewFilePath) { - return nil - } - allowedDownloadDirsMu.RLock() - defer allowedDownloadDirsMu.RUnlock() - for _, dir := range allowedDownloadDirs { - if isPathWithinBase(dir, result.NewFilePath) { - return nil - } - } - return fmt.Errorf("replacement file path is outside allowed directories") -} - -func (m *extensionManager) GetLyricsProviders() []*extensionProviderWrapper { - m.mu.RLock() - defer m.mu.RUnlock() - - var providers []*extensionProviderWrapper - for _, ext := range m.extensions { - if ext.Enabled && ext.Manifest.IsLyricsProvider() && ext.Error == "" { - providers = append(providers, newExtensionProviderWrapper(ext)) - } - } - - sort.Slice(providers, func(i, j int) bool { - return providers[i].extension.ID < providers[j].extension.ID - }) - - return providers -} diff --git a/go_backend/extension_providers_test.go b/go_backend/extension_providers_test.go deleted file mode 100644 index 32569909..00000000 --- a/go_backend/extension_providers_test.go +++ /dev/null @@ -1,890 +0,0 @@ -package gobackend - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "net" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "reflect" - "strings" - "sync" - "testing" - "time" - - "github.com/dop251/goja" -) - -func TestSetMetadataProviderPriorityStripsRetiredBuiltIns(t *testing.T) { - original := GetMetadataProviderPriority() - defer SetMetadataProviderPriority(original) - - SetMetadataProviderPriority([]string{"qobuz"}) - got := GetMetadataProviderPriority() - if len(got) != 0 { - t.Fatalf("expected retired built-in qobuz to be stripped, got %v", got) - } -} - -func TestSetExtensionFallbackProviderIDsDedupesExtensions(t *testing.T) { - original := GetExtensionFallbackProviderIDs() - defer SetExtensionFallbackProviderIDs(original) - - SetExtensionFallbackProviderIDs([]string{"ext-a", "ext-a", " ext-b "}) - - got := GetExtensionFallbackProviderIDs() - want := []string{"ext-a", "ext-b"} - if len(got) != len(want) { - t.Fatalf("unexpected fallback provider length: got %v want %v", got, want) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("unexpected fallback provider at %d: got %v want %v", i, got, want) - } - } -} - -func TestIsExtensionFallbackAllowedDefaultsToAllExtensions(t *testing.T) { - original := GetExtensionFallbackProviderIDs() - defer SetExtensionFallbackProviderIDs(original) - - SetExtensionFallbackProviderIDs(nil) - - if !isExtensionFallbackAllowed("custom-ext") { - t.Fatal("expected custom extension to be allowed when no fallback allowlist is configured") - } -} - -func TestIsExtensionFallbackAllowedRespectsAllowlist(t *testing.T) { - original := GetExtensionFallbackProviderIDs() - defer SetExtensionFallbackProviderIDs(original) - - SetExtensionFallbackProviderIDs([]string{"allowed-ext"}) - - if !isExtensionFallbackAllowed("allowed-ext") { - t.Fatal("expected explicitly allowed extension to be permitted") - } - if isExtensionFallbackAllowed("blocked-ext") { - t.Fatal("expected extension outside allowlist to be blocked") - } - if isExtensionFallbackAllowed("deezer") { - t.Fatal("expected retired Deezer downloader to respect extension fallback allowlist") - } -} - -func TestSetProviderPriorityRemovesRetiredDeezerDownloader(t *testing.T) { - original := GetProviderPriority() - defer SetProviderPriority(original) - - SetProviderPriority([]string{"deezer", "qobuz", "custom-ext"}) - - got := GetProviderPriority() - want := []string{"custom-ext"} - if len(got) != len(want) { - t.Fatalf("unexpected priority length: got %v want %v", got, want) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("unexpected priority at %d: got %v want %v", i, got, want) - } - } -} - -func TestSetProviderPriorityKeepsExtensionNamedLikeRetiredDownloader(t *testing.T) { - original := GetProviderPriority() - defer SetProviderPriority(original) - - manager := getExtensionManager() - ext := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - ext.ID = "deezer" - ext.Manifest.Name = "deezer" - - manager.mu.Lock() - previous, hadPrevious := manager.extensions[ext.ID] - manager.extensions[ext.ID] = ext - manager.mu.Unlock() - defer func() { - manager.mu.Lock() - if hadPrevious { - manager.extensions[ext.ID] = previous - } else { - delete(manager.extensions, ext.ID) - } - manager.mu.Unlock() - }() - - SetProviderPriority([]string{"deezer", "custom-ext"}) - - got := GetProviderPriority() - want := []string{"deezer", "custom-ext"} - if len(got) != len(want) { - t.Fatalf("unexpected priority length: got %v want %v", got, want) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("unexpected priority at %d: got %v want %v", i, got, want) - } - } -} - -func TestPrioritizeFallbackProvidersByHealthPrefersOnlineAndSkipsOffline(t *testing.T) { - manager := getExtensionManager() - unavailable := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - unavailable.ID = "unavailable-provider" - unavailable.Manifest.Name = "unavailable-provider" - unavailable.Manifest.ServiceHealth = []ExtensionHealthCheck{{ - ID: "main", - URL: "://bad", - Required: true, - }} - - plain := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - plain.ID = "plain" - plain.Manifest.Name = "plain" - - available := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - available.ID = "available-provider" - available.Manifest.Name = "available-provider" - available.Manifest.ServiceHealth = []ExtensionHealthCheck{{ - ID: "main", - URL: "https://example.test/health", - }} - - manager.mu.Lock() - previousUnavailable, hadUnavailable := manager.extensions[unavailable.ID] - previousPlain, hadPlain := manager.extensions[plain.ID] - previousAvailable, hadAvailable := manager.extensions[available.ID] - manager.extensions[unavailable.ID] = unavailable - manager.extensions[plain.ID] = plain - manager.extensions[available.ID] = available - manager.mu.Unlock() - defer func() { - manager.mu.Lock() - if hadUnavailable { - manager.extensions[unavailable.ID] = previousUnavailable - } else { - delete(manager.extensions, unavailable.ID) - } - if hadPlain { - manager.extensions[plain.ID] = previousPlain - } else { - delete(manager.extensions, plain.ID) - } - if hadAvailable { - manager.extensions[available.ID] = previousAvailable - } else { - delete(manager.extensions, available.ID) - } - manager.mu.Unlock() - - extensionHealthCacheMu.Lock() - delete(extensionHealthCache, unavailable.ID) - delete(extensionHealthCache, available.ID) - extensionHealthCacheMu.Unlock() - }() - - extensionHealthCacheMu.Lock() - extensionHealthCache[unavailable.ID] = cachedExtensionHealthResult{ - result: ExtensionHealthResult{ - ExtensionID: unavailable.ID, - Status: "offline", - CheckedAt: time.Now().UTC().Format(time.RFC3339), - }, - expiresAt: time.Now().Add(time.Minute), - } - extensionHealthCache[available.ID] = cachedExtensionHealthResult{ - result: ExtensionHealthResult{ - ExtensionID: available.ID, - Status: "online", - CheckedAt: time.Now().UTC().Format(time.RFC3339), - }, - expiresAt: time.Now().Add(time.Minute), - } - extensionHealthCacheMu.Unlock() - - got := prioritizeFallbackProvidersByHealth( - []string{"unavailable-provider", "plain", "available-provider"}, - manager, - "", - ) - want := []string{"available-provider", "plain"} - if len(got) != len(want) { - t.Fatalf("unexpected provider order length: got %v want %v", got, want) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("unexpected provider order at %d: got %v want %v", i, got, want) - } - } -} - -func TestNormalizeDownloadDecryptionInfoPromotesLegacyKey(t *testing.T) { - normalized := normalizeDownloadDecryptionInfo(nil, " 001122 ") - if normalized == nil { - t.Fatal("expected legacy decryption key to produce normalized descriptor") - } - if normalized.Strategy != genericFFmpegMOVDecryptionStrategy { - t.Fatalf("strategy = %q", normalized.Strategy) - } - if normalized.Key != "001122" { - t.Fatalf("key = %q", normalized.Key) - } - if normalized.InputFormat != "mov" { - t.Fatalf("input format = %q", normalized.InputFormat) - } -} - -func TestNormalizeDownloadDecryptionInfoCanonicalizesMovAliases(t *testing.T) { - normalized := normalizeDownloadDecryptionInfo(&DownloadDecryptionInfo{ - Strategy: "mp4_decryption_key", - Key: "abcd", - InputFormat: "", - }, "") - if normalized == nil { - t.Fatal("expected descriptor to remain available") - } - if normalized.Strategy != genericFFmpegMOVDecryptionStrategy { - t.Fatalf("strategy = %q", normalized.Strategy) - } - if normalized.InputFormat != "mov" { - t.Fatalf("input format = %q", normalized.InputFormat) - } -} - -func TestExtensionDownloadUsesIsolatedRuntimeForConcurrentCalls(t *testing.T) { - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(500 * time.Millisecond) - _, _ = w.Write([]byte("ok")) - })) - defer server.Close() - setPrivateIPCache("download.test", false, time.Minute) - - originalTransport := sharedTransport - testTransport := &http.Transport{ - DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) - }, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - sharedTransport = testTransport - defer func() { - testTransport.CloseIdleConnections() - sharedTransport = originalTransport - }() - - extDir := t.TempDir() - if err := os.WriteFile(filepath.Join(extDir, "index.js"), []byte(` - registerExtension({ - download: function(trackID, quality, outputPath, onProgress) { - var result = file.download('https://download.test/' + trackID, outputPath, { - onProgress: function(written, total) { - if (onProgress) onProgress(50); - } - }); - if (!result || !result.success) { - return { - success: false, - error_message: result && result.error ? result.error : 'download failed', - error_type: 'download_error' - }; - } - if (onProgress) onProgress(100); - return { success: true, file_path: result.path }; - } - }); - `), 0600); err != nil { - t.Fatalf("write extension index: %v", err) - } - - outputDir := t.TempDir() - SetAllowedDownloadDirs([]string{outputDir}) - defer SetAllowedDownloadDirs(nil) - - ext := &loadedExtension{ - ID: "concurrent-download", - Manifest: &ExtensionManifest{ - Name: "concurrent-download", - Description: "Concurrent download test", - Version: "1.0.0", - Types: []ExtensionType{ExtensionTypeDownloadProvider}, - Permissions: ExtensionPermissions{ - Network: []string{"download.test"}, - File: true, - }, - }, - Enabled: true, - SourceDir: extDir, - DataDir: t.TempDir(), - } - provider := newExtensionProviderWrapper(ext) - - start := time.Now() - var wg sync.WaitGroup - errs := make(chan error, 2) - for i := 0; i < 2; i++ { - i := i - wg.Add(1) - go func() { - defer wg.Done() - result, err := provider.Download( - fmt.Sprintf("track-%d", i), - "LOSSLESS", - filepath.Join(outputDir, fmt.Sprintf("track-%d.flac", i)), - "", - nil, - ) - if err != nil { - errs <- err - return - } - if result == nil || !result.Success { - errs <- fmt.Errorf("download failed: %#v", result) - } - }() - } - wg.Wait() - close(errs) - for err := range errs { - if err != nil { - t.Fatal(err) - } - } - - if elapsed := time.Since(start); elapsed >= 850*time.Millisecond { - t.Fatalf("expected same-extension downloads to overlap, elapsed %s", elapsed) - } -} - -func TestBuildOutputPathAddsExplicitOutputDirToAllowedDirs(t *testing.T) { - SetAllowedDownloadDirs(nil) - - outputDir := t.TempDir() - outputPath := buildOutputPath(DownloadRequest{ - TrackName: "Song", - ArtistName: "Artist", - OutputDir: outputDir, - OutputExt: ".flac", - FilenameFormat: "", - }) - - if !isPathInAllowedDirs(outputPath) { - t.Fatalf("expected output path %q to be allowed", outputPath) - } -} - -func TestBuildOutputPathForExtensionAddsExplicitOutputPathDirToAllowedDirs(t *testing.T) { - SetAllowedDownloadDirs(nil) - - outputDir := t.TempDir() - outputPath := filepath.Join(outputDir, "custom.flac") - ext := &loadedExtension{DataDir: t.TempDir()} - - resolved := buildOutputPathForExtension(DownloadRequest{ - OutputPath: outputPath, - }, ext) - - if resolved != outputPath { - t.Fatalf("resolved output path = %q", resolved) - } - if !isPathInAllowedDirs(outputPath) { - t.Fatalf("expected output path %q to be allowed", outputPath) - } -} - -func TestBuildOutputPathForExtensionUsesTempDirForFDOutput(t *testing.T) { - SetAllowedDownloadDirs(nil) - - ext := &loadedExtension{DataDir: t.TempDir()} - resolved := buildOutputPathForExtension(DownloadRequest{ - TrackName: "Song", - ArtistName: "Artist", - OutputDir: filepath.Join("Artist", "Album"), - OutputFD: 123, - OutputExt: ".flac", - }, ext) - - expectedBase := filepath.Join(ext.DataDir, "downloads") - if !isPathWithinBase(expectedBase, resolved) { - t.Fatalf("expected SAF extension output under %q, got %q", expectedBase, resolved) - } - if !isPathInAllowedDirs(resolved) { - t.Fatalf("expected resolved output path %q to be allowed", resolved) - } -} - -func TestBuildOutputPathSanitizesTemplateFilename(t *testing.T) { - SetAllowedDownloadDirs(nil) - - outputDir := t.TempDir() - outputPath := buildOutputPath(DownloadRequest{ - TrackName: `Gehra Hua (From "Dhurandhar")`, - ArtistName: "Artist", - OutputDir: outputDir, - OutputExt: ".flac", - FilenameFormat: "{artist} - {title}", - }) - - base := filepath.Base(outputPath) - if strings.ContainsAny(base, `<>:"/\|?*`) { - t.Fatalf("output filename still contains illegal characters: %q", base) - } - if strings.Contains(base, `"`) { - t.Fatalf("output filename still contains straight double quote: %q", base) - } -} - -func TestBuildOutputPathForExtensionSanitizesTemplateFilename(t *testing.T) { - SetAllowedDownloadDirs(nil) - - ext := &loadedExtension{DataDir: t.TempDir()} - resolved := buildOutputPathForExtension(DownloadRequest{ - TrackName: `Gehra Hua (From "Dhurandhar")`, - ArtistName: "Artist", - OutputFD: 123, - OutputExt: ".flac", - FilenameFormat: "{artist} - {title}", - }, ext) - - base := filepath.Base(resolved) - if strings.ContainsAny(base, `<>:"/\|?*`) { - t.Fatalf("extension output filename still contains illegal characters: %q", base) - } -} - -func TestShouldStopProviderFallback(t *testing.T) { - if shouldStopProviderFallback(nil) { - t.Fatal("nil availability should not stop fallback") - } - if shouldStopProviderFallback(&ExtAvailabilityResult{Available: false}) { - t.Fatal("availability without skip_fallback should not stop fallback") - } - if !shouldStopProviderFallback(&ExtAvailabilityResult{Available: false, SkipFallback: true}) { - t.Fatal("skip_fallback availability should stop fallback") - } -} - -func TestMoveProviderToFrontPreservesExplicitSelection(t *testing.T) { - priority := []string{"provider-a", "provider-b", "provider-c"} - got := moveProviderToFront(priority, "PROVIDER-B") - want := []string{"provider-b", "provider-a", "provider-c"} - - if !reflect.DeepEqual(got, want) { - t.Fatalf("moveProviderToFront() = %#v, want %#v", got, want) - } - if !reflect.DeepEqual(priority, []string{"provider-a", "provider-b", "provider-c"}) { - t.Fatalf("moveProviderToFront mutated input: %#v", priority) - } -} - -func TestDiscardRejectedExtensionOutputStaysInsideRequestedDirectory(t *testing.T) { - outputDir := t.TempDir() - requestedPath := filepath.Join(outputDir, "Artist - Song.flac") - rejectedPath := filepath.Join(outputDir, "Artist - Song.m4a") - outsidePath := filepath.Join(t.TempDir(), "keep.flac") - if err := os.WriteFile(rejectedPath, []byte("wrong audio"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(outsidePath, []byte("keep"), 0o600); err != nil { - t.Fatal(err) - } - - discardRejectedExtensionOutput(&ExtDownloadResult{FilePath: rejectedPath}, requestedPath) - if _, err := os.Stat(rejectedPath); !os.IsNotExist(err) { - t.Fatalf("rejected output was not removed: %v", err) - } - - discardRejectedExtensionOutput(&ExtDownloadResult{FilePath: outsidePath}, requestedPath) - if _, err := os.Stat(outsidePath); err != nil { - t.Fatalf("output outside requested directory was removed: %v", err) - } -} - -func TestDiscardRejectedExtensionOutputPreservesExistingLibraryHit(t *testing.T) { - outputDir := t.TempDir() - path := filepath.Join(outputDir, "existing.flac") - if err := os.WriteFile(path, []byte("existing audio"), 0o600); err != nil { - t.Fatal(err) - } - - discardRejectedExtensionOutput(&ExtDownloadResult{ - FilePath: path, - AlreadyExists: true, - }, filepath.Join(outputDir, "requested.flac")) - if _, err := os.Stat(path); err != nil { - t.Fatalf("existing library file was removed: %v", err) - } -} - -func TestBuildExtensionFallbackStoppedResponsePrefersAvailabilityReason(t *testing.T) { - resp := buildExtensionFallbackStoppedResponse("direct-provider", &ExtAvailabilityResult{ - Reason: "direct provider track ID", - SkipFallback: true, - }, errors.New("ignored")) - - if resp.Service != "direct-provider" { - t.Fatalf("service = %q", resp.Service) - } - if resp.Error != "Fallback stopped by direct-provider: direct provider track ID" { - t.Fatalf("unexpected error message: %q", resp.Error) - } - if resp.ErrorType != "extension_error" { - t.Fatalf("error type = %q", resp.ErrorType) - } -} - -func TestBuildExtensionFallbackStoppedResponseFallsBackToError(t *testing.T) { - resp := buildExtensionFallbackStoppedResponse("direct-provider", &ExtAvailabilityResult{ - SkipFallback: true, - }, errors.New("lookup failed")) - - if resp.Error != "Fallback stopped by direct-provider: lookup failed" { - t.Fatalf("unexpected error message: %q", resp.Error) - } -} - -func TestShouldAbortCancelledFallbackWithCancelledError(t *testing.T) { - if !shouldAbortCancelledFallback("", ErrDownloadCancelled) { - t.Fatal("expected cancelled error to abort fallback") - } -} - -func TestShouldAbortCancelledFallbackWithCancelledItemState(t *testing.T) { - const itemID = "cancelled-item" - initDownloadCancel(itemID) - defer clearDownloadCancel(itemID) - - cancelDownload(itemID) - - if !shouldAbortCancelledFallback(itemID, errors.New("generic failure")) { - t.Fatal("expected cancelled item state to abort fallback even for generic errors") - } -} - -func TestCanEmbedGenreLabelRequiresExistingAbsoluteLocalFile(t *testing.T) { - tempFile := filepath.Join(t.TempDir(), "track.flac") - if err := os.WriteFile(tempFile, []byte("fLaC"), 0644); err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - tempM4A := filepath.Join(t.TempDir(), "track.m4a") - if err := os.WriteFile(tempM4A, []byte("not-flac"), 0644); err != nil { - t.Fatalf("failed to create temp m4a file: %v", err) - } - - if canEmbedGenreLabel("relative.flac") { - t.Fatal("expected relative path to be rejected") - } - if canEmbedGenreLabel("content://example") { - t.Fatal("expected content URI to be rejected") - } - if canEmbedGenreLabel(filepath.Join(t.TempDir(), "missing.flac")) { - t.Fatal("expected missing file to be rejected") - } - if canEmbedGenreLabel(tempM4A) { - t.Fatalf("expected non-FLAC file %q to be rejected", tempM4A) - } - if !canEmbedGenreLabel(tempFile) { - t.Fatalf("expected existing absolute file %q to be accepted", tempFile) - } -} - -func TestSearchTracksWithMetadataProvidersIgnoresRetiredBuiltIns(t *testing.T) { - originalPriority := GetMetadataProviderPriority() - defer func() { - SetMetadataProviderPriority(originalPriority) - }() - - SetMetadataProviderPriority([]string{"qobuz"}) - - manager := getExtensionManager() - tracks, err := manager.SearchTracksWithMetadataProviders("query", 3, false) - if err != nil { - t.Fatalf("SearchTracksWithMetadataProviders returned error: %v", err) - } - if len(tracks) != 0 { - t.Fatalf("expected no tracks from retired built-in provider, got %+v", tracks) - } -} - -func TestParseExtensionSearchResultAcceptsObjectAndArrayShapes(t *testing.T) { - vm := goja.New() - value, err := vm.RunString(`({ - tracks: [{ - id: "track-1", - name: "Song", - artists: "Artist", - album_name: "Album", - duration_ms: 123000, - cover_url: "https://img.test/cover.jpg", - external_links: { provider: "https://provider.example/track/1" }, - audio_quality: "LOSSLESS" - }], - total: 9 - })`) - if err != nil { - t.Fatalf("build object search result: %v", err) - } - - result, err := parseExtensionSearchResult(vm, value) - if err != nil { - t.Fatalf("parse object search result: %v", err) - } - if result.Total != 9 || len(result.Tracks) != 1 { - t.Fatalf("unexpected object result: %+v", result) - } - track := result.Tracks[0] - if track.ID != "track-1" || - track.AlbumName != "Album" || - track.DurationMS != 123000 || - track.CoverURL != "https://img.test/cover.jpg" || - track.ExternalLinks["provider"] != "https://provider.example/track/1" || - track.AudioQuality != "LOSSLESS" { - t.Fatalf("unexpected parsed track: %+v", track) - } - - arrayValue, err := vm.RunString(`[ - {id: "track-2", name: "Other Song", artists: "Other Artist", albumName: "Other Album", durationMs: 456000} - ]`) - if err != nil { - t.Fatalf("build array search result: %v", err) - } - - arrayResult, err := parseExtensionSearchResult(vm, arrayValue) - if err != nil { - t.Fatalf("parse array search result: %v", err) - } - if arrayResult.Total != 1 || - len(arrayResult.Tracks) != 1 || - arrayResult.Tracks[0].AlbumName != "Other Album" || - arrayResult.Tracks[0].DurationMS != 456000 { - t.Fatalf("unexpected array result: %+v", arrayResult) - } -} - -func TestParseExtensionMetadataAndDownloadResults(t *testing.T) { - vm := goja.New() - value, err := vm.RunString(`({ - id: "album-1", - name: "Album", - artists: "Artist", - artistId: "artist-1", - coverUrl: "https://img.test/album.jpg", - releaseDate: "2024-02-03", - totalTracks: 2, - albumType: "album", - tracks: [ - {id: "track-1", name: "Song 1", artists: "Artist", durationMs: 180000}, - {id: "track-2", name: "Song 2", artists: "Artist", duration_ms: 181000} - ] - })`) - if err != nil { - t.Fatalf("build album value: %v", err) - } - - album, err := parseExtensionAlbumValue(vm, value) - if err != nil { - t.Fatalf("parse album: %v", err) - } - if album.ID != "album-1" || - album.ArtistID != "artist-1" || - album.CoverURL != "https://img.test/album.jpg" || - album.TotalTracks != 2 || - len(album.Tracks) != 2 || - album.Tracks[0].DurationMS != 180000 || - album.Tracks[1].DurationMS != 181000 { - t.Fatalf("unexpected album: %+v", album) - } - - artistValue, err := vm.RunString(`({ - id: "artist-1", - name: "Artist", - imageUrl: "https://img.test/artist.jpg", - headerImage: "https://img.test/header.jpg", - listeners: 1234, - albums: [{id: "album-1", name: "Album", tracks: [{id: "track-1", name: "Song"}]}], - releases: [{id: "single-1", name: "Single"}], - tracks: [{id: "top-1", name: "Top Song"}] - })`) - if err != nil { - t.Fatalf("build artist value: %v", err) - } - - artist, err := parseExtensionArtistValue(vm, artistValue) - if err != nil { - t.Fatalf("parse artist: %v", err) - } - if artist.ID != "artist-1" || - artist.ImageURL != "https://img.test/artist.jpg" || - artist.HeaderImage != "https://img.test/header.jpg" || - artist.Listeners != 1234 || - len(artist.Albums) != 1 || - len(artist.Albums[0].Tracks) != 1 || - len(artist.Releases) != 1 || - len(artist.TopTracks) != 1 { - t.Fatalf("unexpected artist: %+v", artist) - } - - downloadValue, err := vm.RunString(`({ - success: true, - filePath: "/tmp/song.flac", - alreadyExists: true, - bitDepth: 24, - sampleRate: 96000, - durationMs: 181000, - title: "Song", - albumArtist: "Album Artist", - lyricsLrc: "[00:00.00]Line", - decryptionKey: "001122", - decryption: { - strategy: "mp4_decryption_key", - key: "001122", - inputFormat: "m4a", - options: { map: "0:a" } - } - })`) - if err != nil { - t.Fatalf("build download value: %v", err) - } - - download := parseExtensionDownloadResultValue(vm, downloadValue) - if !download.Success || - download.FilePath != "/tmp/song.flac" || - !download.AlreadyExists || - download.BitDepth != 24 || - download.SampleRate != 96000 || - download.DurationMS != 181000 || - download.AlbumArtist != "Album Artist" || - download.LyricsLRC != "[00:00.00]Line" || - download.Decryption == nil || - download.Decryption.InputFormat != "m4a" || - download.Decryption.Options["map"] != "0:a" { - t.Fatalf("unexpected download result: %+v", download) - } - - availabilityValue, err := vm.RunString(`({ available: true, trackId: "track-1", skipFallback: true, reason: "direct" })`) - if err != nil { - t.Fatalf("build availability value: %v", err) - } - availability := parseExtensionAvailabilityValue(vm, availabilityValue) - if !availability.Available || availability.TrackID != "track-1" || !availability.SkipFallback || availability.Reason != "direct" { - t.Fatalf("unexpected availability: %+v", availability) - } -} - -func TestParseExtensionURLHandleResult(t *testing.T) { - vm := goja.New() - value, err := vm.RunString(`({ - type: "album", - name: "Shared Album", - coverUrl: "https://img.test/shared.jpg", - track: { id: "track-1", name: "Song" }, - tracks: [{ id: "track-2", name: "Song 2" }], - album: { id: "album-1", name: "Album", tracks: [{ id: "track-3", name: "Song 3" }] }, - artist: { id: "artist-1", name: "Artist", topTracks: [{ id: "track-4", name: "Song 4" }] } - })`) - if err != nil { - t.Fatalf("build URL handle value: %v", err) - } - - result, err := parseExtensionURLHandleValue(vm, value) - if err != nil { - t.Fatalf("parse URL handle: %v", err) - } - if result.Type != "album" || - result.CoverURL != "https://img.test/shared.jpg" || - result.Track == nil || - result.Track.ID != "track-1" || - len(result.Tracks) != 1 || - result.Album == nil || - len(result.Album.Tracks) != 1 || - result.Artist == nil || - len(result.Artist.TopTracks) != 1 { - t.Fatalf("unexpected URL handle result: %+v", result) - } -} - -func TestParseExtensionAuxiliaryResults(t *testing.T) { - vm := goja.New() - - postValue, err := vm.RunString(`({ success: true, newFilePath: "/tmp/new.flac", newFileUri: "content://new", bitDepth: 24, sampleRate: 96000 })`) - if err != nil { - t.Fatalf("build post-process value: %v", err) - } - post := parseExtensionPostProcessValue(vm, postValue) - if !post.Success || post.NewFilePath != "/tmp/new.flac" || post.NewFileURI != "content://new" || post.BitDepth != 24 || post.SampleRate != 96000 { - t.Fatalf("unexpected post-process result: %+v", post) - } - - lyricsValue, err := vm.RunString(`({ - syncType: "LINE_SYNCED", - instrumental: false, - plainLyrics: "Line", - provider: "Lyrics Provider", - lines: [{ startTimeMs: 1000, words: "Line", endTimeMs: 2000 }] - })`) - if err != nil { - t.Fatalf("build lyrics value: %v", err) - } - lyrics, err := parseExtensionLyricsValue(vm, lyricsValue) - if err != nil { - t.Fatalf("parse lyrics: %v", err) - } - if lyrics.SyncType != "LINE_SYNCED" || - lyrics.PlainLyrics != "Line" || - lyrics.Provider != "Lyrics Provider" || - len(lyrics.Lines) != 1 || - lyrics.Lines[0].StartTimeMs != 1000 || - lyrics.Lines[0].EndTimeMs != 2000 { - t.Fatalf("unexpected lyrics result: %+v", lyrics) - } -} - -func TestMatchesURLHostAnchored(t *testing.T) { - manifest := &ExtensionManifest{ - URLHandler: &URLHandlerConfig{ - Enabled: true, - Patterns: []string{"catalog.example", "short.example", "catalog:"}, - }, - } - - for _, urlStr := range []string{ - "https://open.catalog.example/track/abc", - "https://catalog.example/track/abc", - "HTTPS://OPEN.CATALOG.EXAMPLE/track/ABC", - "https://short.example/xyz", - "catalog:track:abc123", - } { - if !manifest.MatchesURL(urlStr) { - t.Fatalf("expected match for %q", urlStr) - } - } - - for _, urlStr := range []string{ - // The old substring matching accepted all of these. - "https://evil.example/?next=https://catalog.example/track/abc", - "https://notcatalog.example/track/abc", - "https://catalog.example.evil.example/track/abc", - "https://example.com/catalog.example", - "not a url at all", - } { - if manifest.MatchesURL(urlStr) { - t.Fatalf("expected no match for %q", urlStr) - } - } - - withPath := &ExtensionManifest{ - URLHandler: &URLHandlerConfig{ - Enabled: true, - Patterns: []string{"video.example/watch"}, - }, - } - if !withPath.MatchesURL("https://www.video.example/watch?v=abc") { - t.Fatal("expected host+path prefix to match") - } - if withPath.MatchesURL("https://www.video.example/playlist?list=abc") { - t.Fatal("expected different path to not match") - } -} diff --git a/go_backend/extension_repo.go b/go_backend/extension_repo.go deleted file mode 100644 index 06665732..00000000 --- a/go_backend/extension_repo.go +++ /dev/null @@ -1,770 +0,0 @@ -package gobackend - -import ( - "crypto/sha256" - "crypto/subtle" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "golang.org/x/sync/singleflight" -) - -const ( - CategoryMetadata = "metadata" - CategoryDownload = "download" - CategoryUtility = "utility" - CategoryLyrics = "lyrics" - CategoryIntegration = "integration" -) - -type repoExtension struct { - ID string `json:"id"` - Name string `json:"name"` - DisplayName string `json:"display_name,omitempty"` - Version string `json:"version"` - Description string `json:"description"` - DownloadURL string `json:"download_url,omitempty"` - IconURL string `json:"icon_url,omitempty"` - Category string `json:"category"` - Tags []string `json:"tags,omitempty"` - Downloads int `json:"downloads"` - UpdatedAt string `json:"updated_at"` - MinAppVersion string `json:"min_app_version,omitempty"` - SHA256 string `json:"sha256,omitempty"` - ChecksumSHA256 string `json:"checksum_sha256,omitempty"` - DisplayNameAlt string `json:"displayName,omitempty"` - DownloadURLAlt string `json:"downloadUrl,omitempty"` - IconURLAlt string `json:"iconUrl,omitempty"` - MinAppVersionAlt string `json:"minAppVersion,omitempty"` - ChecksumAlt string `json:"checksumSha256,omitempty"` -} - -func (e *repoExtension) getDisplayName() string { - if e.DisplayName != "" { - return e.DisplayName - } - if e.DisplayNameAlt != "" { - return e.DisplayNameAlt - } - return e.Name -} - -func (e *repoExtension) getDownloadURL() string { - if e.DownloadURL != "" { - return e.DownloadURL - } - return e.DownloadURLAlt -} - -func (e *repoExtension) getIconURL() string { - if e.IconURL != "" { - return e.IconURL - } - return e.IconURLAlt -} - -func (e *repoExtension) getMinAppVersion() string { - if e.MinAppVersion != "" { - return e.MinAppVersion - } - return e.MinAppVersionAlt -} - -func (e *repoExtension) getRawSHA256() string { - return firstNonEmptyTrimmed(e.SHA256, e.ChecksumSHA256, e.ChecksumAlt) -} - -func (e *repoExtension) getSHA256() string { - return normalizeSHA256(e.getRawSHA256()) -} - -type repoRegistry struct { - Version int `json:"version"` - UpdatedAt string `json:"updated_at"` - Extensions []repoExtension `json:"extensions"` -} - -type repoExtensionResponse struct { - ID string `json:"id"` - Name string `json:"name"` - DisplayName string `json:"display_name"` - Version string `json:"version"` - Description string `json:"description"` - DownloadURL string `json:"download_url"` - IconURL string `json:"icon_url,omitempty"` - Category string `json:"category"` - Tags []string `json:"tags,omitempty"` - Downloads int `json:"downloads"` - UpdatedAt string `json:"updated_at"` - MinAppVersion string `json:"min_app_version,omitempty"` - SHA256 string `json:"sha256,omitempty"` - IsInstalled bool `json:"is_installed"` - InstalledVersion string `json:"installed_version,omitempty"` - HasUpdate bool `json:"has_update"` -} - -func (e *repoExtension) toResponse() repoExtensionResponse { - resp := repoExtensionResponse{ - ID: e.ID, - Name: e.Name, - DisplayName: e.getDisplayName(), - Version: e.Version, - Description: e.Description, - DownloadURL: e.getDownloadURL(), - IconURL: e.getIconURL(), - Category: e.Category, - Downloads: e.Downloads, - UpdatedAt: e.UpdatedAt, - MinAppVersion: e.getMinAppVersion(), - SHA256: e.getSHA256(), - } - - if len(e.Tags) > 0 { - resp.Tags = append([]string(nil), e.Tags...) - } - - return resp -} - -type extensionRepo struct { - registryURL string - cacheDir string - cache *repoRegistry - cacheMu sync.RWMutex - cacheTime time.Time - cacheTTL time.Duration - etag string - lastModified string - fetchGroup singleflight.Group -} - -var ( - globalExtensionRepo *extensionRepo - extensionRepoMu sync.Mutex -) - -const ( - cacheTTL = 30 * time.Minute - cacheFileName = "store_cache.json" - maxRegistryBodyBytes = 4 << 20 - registryRequestTimout = 30 * time.Second -) - -func initExtensionRepo(cacheDir string) *extensionRepo { - extensionRepoMu.Lock() - defer extensionRepoMu.Unlock() - - if globalExtensionRepo == nil { - globalExtensionRepo = &extensionRepo{ - registryURL: "", - cacheDir: cacheDir, - cacheTTL: cacheTTL, - } - globalExtensionRepo.loadDiskCache() - } - return globalExtensionRepo -} - -func (s *extensionRepo) setRegistryURL(registryURL string) { - s.cacheMu.Lock() - defer s.cacheMu.Unlock() - - if s.registryURL == registryURL { - return - } - - s.registryURL = registryURL - s.cache = nil - s.cacheTime = time.Time{} - s.etag = "" - s.lastModified = "" - - if s.cacheDir != "" { - cachePath := filepath.Join(s.cacheDir, cacheFileName) - os.Remove(cachePath) - } - - LogInfo("ExtensionRepo", "Registry URL updated to: %s", registryURL) -} - -func (s *extensionRepo) getRegistryURL() string { - s.cacheMu.RLock() - defer s.cacheMu.RUnlock() - return s.registryURL -} - -func getExtensionRepo() *extensionRepo { - extensionRepoMu.Lock() - defer extensionRepoMu.Unlock() - return globalExtensionRepo -} - -func (s *extensionRepo) loadDiskCache() { - if s.cacheDir == "" { - return - } - - cachePath := filepath.Join(s.cacheDir, cacheFileName) - data, err := os.ReadFile(cachePath) - if err != nil { - return - } - - var cacheData struct { - RegistryURL string `json:"registry_url"` - Registry repoRegistry `json:"registry"` - CacheTime int64 `json:"cache_time"` - ETag string `json:"etag,omitempty"` - LastModified string `json:"last_modified,omitempty"` - } - - if err := json.Unmarshal(data, &cacheData); err != nil { - return - } - - s.cache = &cacheData.Registry - s.cacheTime = time.Unix(cacheData.CacheTime, 0) - s.etag = cacheData.ETag - s.lastModified = cacheData.LastModified - if s.registryURL == "" { - // Restore the URL that produced this cache so a later setRegistryURL - // with the same URL keeps the cache instead of wiping it. - s.registryURL = cacheData.RegistryURL - } - LogDebug("ExtensionRepo", "Loaded %d extensions from disk cache", len(s.cache.Extensions)) -} - -func (s *extensionRepo) saveDiskCache() { - s.cacheMu.RLock() - cacheDir := s.cacheDir - registryURL := s.registryURL - registry := s.cache - cacheTime := s.cacheTime - etag := s.etag - lastModified := s.lastModified - s.cacheMu.RUnlock() - - if cacheDir == "" || registry == nil { - return - } - - cacheData := struct { - RegistryURL string `json:"registry_url"` - Registry repoRegistry `json:"registry"` - CacheTime int64 `json:"cache_time"` - ETag string `json:"etag,omitempty"` - LastModified string `json:"last_modified,omitempty"` - }{ - RegistryURL: registryURL, - Registry: *registry, - CacheTime: cacheTime.Unix(), - ETag: etag, - LastModified: lastModified, - } - - data, err := json.Marshal(cacheData) - if err != nil { - return - } - - if err := os.MkdirAll(cacheDir, 0755); err != nil { - return - } - cachePath := filepath.Join(cacheDir, cacheFileName) - tempPath := cachePath + ".tmp" - if err := os.WriteFile(tempPath, data, 0600); err != nil { - return - } - if err := os.Rename(tempPath, cachePath); err != nil { - _ = os.Remove(tempPath) - } -} - -func (s *extensionRepo) fetchRegistry(forceRefresh bool) (*repoRegistry, error) { - s.cacheMu.RLock() - registryURL := s.registryURL - cached := s.cache - cacheTime := s.cacheTime - s.cacheMu.RUnlock() - - if registryURL == "" { - return nil, fmt.Errorf("no registry URL configured. Please add a repository URL first") - } - - if !forceRefresh && cached != nil && time.Since(cacheTime) < s.cacheTTL { - LogDebug("ExtensionRepo", "Using cached registry (%d extensions)", len(cached.Extensions)) - return cached, nil - } - - if err := requireHTTPSURL(registryURL, "registry"); err != nil { - return nil, err - } - - value, err, _ := s.fetchGroup.Do(registryURL, func() (any, error) { - return s.fetchRegistryUncoalesced(registryURL, forceRefresh) - }) - if err != nil { - return nil, err - } - registry, _ := value.(*repoRegistry) - if registry == nil { - return nil, fmt.Errorf("registry request returned no data") - } - return registry, nil -} - -func (s *extensionRepo) fetchRegistryUncoalesced(registryURL string, forceRefresh bool) (*repoRegistry, error) { - s.cacheMu.RLock() - if s.registryURL != registryURL { - s.cacheMu.RUnlock() - return nil, fmt.Errorf("registry URL changed while refreshing") - } - cached := s.cache - cacheTime := s.cacheTime - etag := s.etag - lastModified := s.lastModified - s.cacheMu.RUnlock() - - if !forceRefresh && cached != nil && time.Since(cacheTime) < s.cacheTTL { - return cached, nil - } - - LogInfo("ExtensionRepo", "Fetching registry from %s", registryURL) - - client := NewHTTPClientWithTimeout(registryRequestTimout) - req, err := http.NewRequest(http.MethodGet, registryURL, nil) - if err != nil { - if cached != nil { - LogWarn("ExtensionRepo", "Failed to build registry request, using cached registry: %v", err) - return cached, nil - } - return nil, fmt.Errorf("failed to build registry request: %w", err) - } - if etag != "" { - req.Header.Set("If-None-Match", etag) - } - if lastModified != "" { - req.Header.Set("If-Modified-Since", lastModified) - } - resp, err := client.Do(req) - if err != nil { - if cached != nil { - LogWarn("ExtensionRepo", "Network error, using cached registry: %v", err) - return cached, nil - } - return nil, fmt.Errorf("failed to fetch registry: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusNotModified && cached != nil { - s.cacheMu.Lock() - if s.registryURL == registryURL && s.cache == cached { - s.cacheTime = time.Now() - } - s.cacheMu.Unlock() - s.saveDiskCache() - return cached, nil - } - - if resp.StatusCode != http.StatusOK { - if cached != nil { - LogWarn("ExtensionRepo", "HTTP %d, using cached registry", resp.StatusCode) - return cached, nil - } - return nil, fmt.Errorf("registry returned HTTP %d", resp.StatusCode) - } - - body, err := io.ReadAll(io.LimitReader(resp.Body, maxRegistryBodyBytes+1)) - if err != nil { - return nil, fmt.Errorf("failed to read registry: %w", err) - } - if len(body) > maxRegistryBodyBytes { - if cached != nil { - LogWarn("ExtensionRepo", "Registry response exceeded %d bytes, using cached registry", maxRegistryBodyBytes) - return cached, nil - } - return nil, fmt.Errorf("registry response exceeds %d bytes", maxRegistryBodyBytes) - } - - registry, err := parseRegistryBody(body) - if err != nil { - if cached != nil { - LogWarn("ExtensionRepo", "Failed to parse registry, using cached registry: %v", err) - return cached, nil - } - return nil, err - } - - s.cacheMu.Lock() - if s.registryURL != registryURL { - s.cacheMu.Unlock() - return registry, nil - } - s.cache = registry - s.cacheTime = time.Now() - s.etag = strings.TrimSpace(resp.Header.Get("ETag")) - s.lastModified = strings.TrimSpace(resp.Header.Get("Last-Modified")) - s.cacheMu.Unlock() - s.saveDiskCache() - - LogInfo("ExtensionRepo", "Fetched %d extensions from registry", len(registry.Extensions)) - return registry, nil -} - -func parseRegistryBody(body []byte) (*repoRegistry, error) { - var registry repoRegistry - if err := json.Unmarshal(body, ®istry); err != nil { - if strings.HasPrefix(strings.TrimSpace(string(body)), "<") { - return nil, fmt.Errorf("registry URL returned a web page instead of JSON. Make sure the URL points to a registry.json file or a GitHub repository that contains one") - } - return nil, fmt.Errorf("failed to parse registry: %w", err) - } - validExtensions := make([]repoExtension, 0, len(registry.Extensions)) - for index := range registry.Extensions { - ext := ®istry.Extensions[index] - rawChecksum := ext.getRawSHA256() - if rawChecksum != "" && normalizeSHA256(rawChecksum) == "" { - LogWarn( - "ExtensionRepo", - "Skipping registry extension %q at index %d: invalid SHA-256 checksum", - ext.ID, - index, - ) - continue - } - validExtensions = append(validExtensions, *ext) - } - registry.Extensions = validExtensions - return ®istry, nil -} - -func (s *extensionRepo) getExtensionsWithStatus(forceRefresh bool) ([]repoExtensionResponse, error) { - registry, err := s.fetchRegistry(forceRefresh) - if err != nil { - return nil, err - } - - manager := getExtensionManager() - installed := make(map[string]string) // id -> version - - if manager != nil { - for _, ext := range manager.GetAllExtensions() { - installed[ext.ID] = ext.Manifest.Version - } - } - - LogDebug("ExtensionRepo", "Building store response for %d registry extensions (%d installed)", len(registry.Extensions), len(installed)) - - result := make([]repoExtensionResponse, 0, len(registry.Extensions)) - for i := range registry.Extensions { - ext := ®istry.Extensions[i] - resp := ext.toResponse() - if installedVersion, ok := installed[ext.ID]; ok { - resp.IsInstalled = true - resp.InstalledVersion = installedVersion - resp.HasUpdate = compareVersions(ext.Version, installedVersion) > 0 - } - - result = append(result, resp) - } - - LogDebug("ExtensionRepo", "Built store response payload for %d extensions", len(result)) - return result, nil -} - -func (s *extensionRepo) findExtension(extensionID string) (*repoExtension, error) { - registry, err := s.fetchRegistry(false) - if err != nil { - return nil, err - } - - for _, e := range registry.Extensions { - if e.ID == extensionID { - ext := e - return &ext, nil - } - } - - return nil, fmt.Errorf("extension %s not found in repo", extensionID) -} - -func (s *extensionRepo) downloadExtension(extensionID string, destPath string) error { - ext, err := s.findExtension(extensionID) - if err != nil { - return err - } - - if err := requireHTTPSURL(ext.getDownloadURL(), "extension download"); err != nil { - return err - } - - LogInfo("ExtensionRepo", "Downloading %s from %s", ext.getDisplayName(), ext.getDownloadURL()) - - client := NewHTTPClientWithTimeout(5 * time.Minute) - req, err := http.NewRequest(http.MethodGet, ext.getDownloadURL(), nil) - if err != nil { - return fmt.Errorf("failed to build download request: %w", err) - } - req.Header.Set("Cache-Control", "no-cache") - req.Header.Set("Pragma", "no-cache") - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("failed to download: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download returned HTTP %d", resp.StatusCode) - } - - if err := writeVerifiedExtensionPackage( - resp.Body, - destPath, - ext.getRawSHA256(), - ); err != nil { - return err - } - - LogInfo("ExtensionRepo", "Downloaded %s to %s", ext.getDisplayName(), destPath) - return nil -} - -const maxExtensionPackageBytes int64 = 64 * 1024 * 1024 - -func normalizeSHA256(value string) string { - normalized := strings.ToLower(strings.TrimSpace(value)) - normalized = strings.TrimPrefix(normalized, "sha256:") - if len(normalized) != sha256.Size*2 { - return "" - } - for _, char := range normalized { - if (char < '0' || char > '9') && (char < 'a' || char > 'f') { - return "" - } - } - return normalized -} - -func writeVerifiedExtensionPackage( - reader io.Reader, - destPath string, - expectedSHA256 string, -) error { - if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { - return fmt.Errorf("failed to prepare extension download directory: %w", err) - } - - tempFile, err := os.CreateTemp( - filepath.Dir(destPath), - "."+filepath.Base(destPath)+".download-*", - ) - if err != nil { - return fmt.Errorf("failed to create extension download: %w", err) - } - tempPath := tempFile.Name() - committed := false - defer func() { - _ = tempFile.Close() - if !committed { - _ = os.Remove(tempPath) - } - }() - - hasher := sha256.New() - limited := &io.LimitedReader{R: reader, N: maxExtensionPackageBytes + 1} - written, copyErr := io.Copy(io.MultiWriter(tempFile, hasher), limited) - if copyErr != nil { - return fmt.Errorf("failed to write extension package: %w", copyErr) - } - if written > maxExtensionPackageBytes { - return fmt.Errorf( - "extension package exceeds the %d MiB size limit", - maxExtensionPackageBytes/(1024*1024), - ) - } - if err := tempFile.Sync(); err != nil { - return fmt.Errorf("failed to flush extension package: %w", err) - } - if err := tempFile.Close(); err != nil { - return fmt.Errorf("failed to close extension package: %w", err) - } - - expected := normalizeSHA256(expectedSHA256) - if strings.TrimSpace(expectedSHA256) != "" && expected == "" { - return fmt.Errorf("registry contains an invalid extension SHA-256 checksum") - } - if expected != "" { - actual := fmt.Sprintf("%x", hasher.Sum(nil)) - if subtle.ConstantTimeCompare([]byte(actual), []byte(expected)) != 1 { - return fmt.Errorf( - "extension package integrity check failed: SHA-256 mismatch", - ) - } - } else { - LogWarn( - "ExtensionRepo", - "Registry entry has no SHA-256 checksum; package integrity cannot be verified", - ) - } - - if err := os.Remove(destPath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to replace cached extension package: %w", err) - } - if err := os.Rename(tempPath, destPath); err != nil { - return fmt.Errorf("failed to publish extension package: %w", err) - } - committed = true - return nil -} - -func resolveRegistryURL(input string) (string, error) { - input = strings.TrimSpace(input) - if input == "" { - return "", fmt.Errorf("registry URL is empty") - } - - if strings.Contains(input, "raw.githubusercontent.com") { - return input, nil - } - - const ghPrefix = "https://github.com/" - if !strings.HasPrefix(input, ghPrefix) { - const ghPrefixHTTP = "http://github.com/" - if strings.HasPrefix(input, ghPrefixHTTP) { - input = "https://github.com/" + input[len(ghPrefixHTTP):] - } else { - return input, nil - } - } - - path := input[len(ghPrefix):] - parts := strings.SplitN(path, "/", 3) // owner, repo, [rest] - if len(parts) < 2 || parts[0] == "" || parts[1] == "" { - return "", fmt.Errorf("invalid GitHub URL: expected github.com/<owner>/<repo>") - } - owner := parts[0] - repo := strings.TrimSuffix(parts[1], ".git") - - branch := resolveGitHubDefaultBranch(owner, repo) - - resolved := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/registry.json", owner, repo, branch) - LogInfo("ExtensionRepo", "Resolved %s → %s (branch: %s)", input, resolved, branch) - return resolved, nil -} - -func resolveGitHubDefaultBranch(owner, repo string) string { - apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s", owner, repo) - client := NewHTTPClientWithTimeout(10 * time.Second) - - resp, err := client.Get(apiURL) - if err != nil { - LogWarn("ExtensionRepo", "GitHub API request failed for %s/%s: %v – falling back to main", owner, repo, err) - return "main" - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - LogWarn("ExtensionRepo", "GitHub API returned %d for %s/%s – falling back to main", resp.StatusCode, owner, repo) - return "main" - } - - var info struct { - DefaultBranch string `json:"default_branch"` - } - if err := json.NewDecoder(resp.Body).Decode(&info); err != nil || info.DefaultBranch == "" { - LogWarn("ExtensionRepo", "Could not parse default_branch for %s/%s – falling back to main", owner, repo) - return "main" - } - - return info.DefaultBranch -} - -func requireHTTPSURL(rawURL string, context string) error { - if rawURL == "" { - return fmt.Errorf("%s URL is empty", context) - } - parsed, err := url.Parse(rawURL) - if err != nil || parsed.Host == "" { - return fmt.Errorf("%s URL is invalid: %s", context, rawURL) - } - if parsed.Scheme != "https" { - return fmt.Errorf("%s URL must use https: %s", context, rawURL) - } - return nil -} - -func (s *extensionRepo) getCategories() []string { - return []string{ - CategoryMetadata, - CategoryDownload, - CategoryUtility, - CategoryLyrics, - CategoryIntegration, - } -} - -func (s *extensionRepo) searchExtensions(query string, category string) ([]repoExtensionResponse, error) { - extensions, err := s.getExtensionsWithStatus(false) - if err != nil { - return nil, err - } - - if query == "" && category == "" { - return extensions, nil - } - - result := make([]repoExtensionResponse, 0, len(extensions)) - queryLower := strings.ToLower(query) - - for _, ext := range extensions { - if category != "" && ext.Category != category { - continue - } - - if query != "" { - if !strings.Contains(strings.ToLower(ext.Name), queryLower) && - !strings.Contains(strings.ToLower(ext.DisplayName), queryLower) && - !strings.Contains(strings.ToLower(ext.Description), queryLower) { - found := false - for _, tag := range ext.Tags { - if strings.Contains(strings.ToLower(tag), queryLower) { - found = true - break - } - } - if !found { - continue - } - } - } - - result = append(result, ext) - } - - return result, nil -} - -func (s *extensionRepo) clearCache() { - s.cacheMu.Lock() - defer s.cacheMu.Unlock() - - s.cache = nil - s.cacheTime = time.Time{} - s.etag = "" - s.lastModified = "" - - if s.cacheDir != "" { - cachePath := filepath.Join(s.cacheDir, cacheFileName) - os.Remove(cachePath) - } - - LogInfo("ExtensionRepo", "Cache cleared") -} diff --git a/go_backend/extension_resolution_budget.go b/go_backend/extension_resolution_budget.go deleted file mode 100644 index 67a1cb45..00000000 --- a/go_backend/extension_resolution_budget.go +++ /dev/null @@ -1,216 +0,0 @@ -package gobackend - -import ( - "context" - "io" - "net/http" - "sync" - "time" - - "github.com/dop251/goja" -) - -const extensionResolutionTimeout = 60 * time.Second - -// resolutionBudget counts total resolver time across URL refreshes and retries. -// Only native transfers with received bytes and bounded native conversion work -// pause it; progress/status callbacks cannot reset the allowance. -type resolutionBudget struct { - ctx context.Context - cancel context.CancelCauseFunc - mu sync.Mutex - remaining time.Duration - started time.Time - timer *time.Timer - generation uint64 - pauses int - charges int - stopped bool -} - -func newResolutionBudget(parent context.Context, allowance time.Duration) *resolutionBudget { - ctx, cancel := context.WithCancelCause(parent) - b := &resolutionBudget{ctx: ctx, cancel: cancel, remaining: allowance} - b.mu.Lock() - b.armLocked() - b.mu.Unlock() - return b -} - -func (b *resolutionBudget) armLocked() { - b.started = time.Now() - b.generation++ - generation := b.generation - b.timer = time.AfterFunc(b.remaining, func() { - b.mu.Lock() - defer b.mu.Unlock() - if b.stopped || (b.pauses > 0 && b.charges == 0) || b.generation != generation { - return - } - b.remaining = 0 - b.cancel(context.DeadlineExceeded) - }) -} - -func (b *resolutionBudget) pause() func() { - b.mu.Lock() - if b.stopped || b.ctx.Err() != nil { - b.mu.Unlock() - return func() {} - } - if b.pauses == 0 && b.charges == 0 { - b.timer.Stop() - b.generation++ - b.remaining -= time.Since(b.started) - if b.remaining <= 0 { - b.remaining = 0 - b.cancel(context.DeadlineExceeded) - b.mu.Unlock() - return func() {} - } - } - b.pauses++ - b.mu.Unlock() - var once sync.Once - return func() { - once.Do(func() { - b.mu.Lock() - defer b.mu.Unlock() - b.pauses-- - if b.pauses == 0 && b.charges == 0 && !b.stopped && b.ctx.Err() == nil { - b.armLocked() - } - }) - } -} - -// Resolver work must keep spending the allowance even while parallel segment -// workers are paused in native network reads. -func (b *resolutionBudget) charge() func() { - b.mu.Lock() - if b.stopped || b.ctx.Err() != nil { - b.mu.Unlock() - return func() {} - } - if b.charges == 0 && b.pauses > 0 { - b.armLocked() - } - b.charges++ - b.mu.Unlock() - var once sync.Once - return func() { - once.Do(func() { - b.mu.Lock() - defer b.mu.Unlock() - b.charges-- - if b.charges == 0 && b.pauses > 0 && !b.stopped && b.ctx.Err() == nil { - b.timer.Stop() - b.generation++ - b.remaining -= time.Since(b.started) - if b.remaining <= 0 { - b.remaining = 0 - b.cancel(context.DeadlineExceeded) - } - } - }) - } -} - -func (b *resolutionBudget) remainingTime() time.Duration { - b.mu.Lock() - defer b.mu.Unlock() - remaining := b.remaining - if (b.pauses == 0 || b.charges > 0) && !b.stopped { - remaining -= time.Since(b.started) - } - if remaining < 0 || b.ctx.Err() != nil { - return 0 - } - return remaining -} - -func (b *resolutionBudget) stop() { - b.mu.Lock() - defer b.mu.Unlock() - b.stopped = true - b.generation++ - b.timer.Stop() - b.cancel(context.Canceled) -} - -func (r *extensionRuntime) currentResolutionBudget() *resolutionBudget { - r.resolutionMu.RLock() - defer r.resolutionMu.RUnlock() - return r.resolutionBudget -} - -func (r *extensionRuntime) beginResolutionBudget(ctx context.Context, allowance time.Duration) (context.Context, func()) { - b := newResolutionBudget(ctx, allowance) - r.resolutionMu.Lock() - r.resolutionBudget = b - r.resolutionMu.Unlock() - return b.ctx, func() { - b.stop() - r.resolutionMu.Lock() - if r.resolutionBudget == b { - r.resolutionBudget = nil - } - r.resolutionMu.Unlock() - } -} - -func (r *extensionRuntime) getResolutionRemainingMs(goja.FunctionCall) goja.Value { - if b := r.currentResolutionBudget(); b != nil { - return r.vm.ToValue(b.remainingTime().Milliseconds()) - } - return r.vm.ToValue(extensionResolutionTimeout.Milliseconds()) -} - -// A successful segment/range establishes the current native transfer. Later -// parts need no new resolution, so their headers and first byte are transfer -// time too. Initial requests and retries still spend the resolver allowance. -func (r *extensionRuntime) doResolutionTransfer(client *http.Client, req *http.Request, continuation bool) (*http.Response, error) { - b := r.currentResolutionBudget() - resp, err := func() (*http.Response, error) { - if b != nil { - if continuation { - defer b.pause()() - } else { - defer b.charge()() - } - } - return client.Do(req) - }() - if b != nil && resp != nil && resp.Body != nil && (resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusPartialContent) { - resp.Body = &resolutionTransferBody{ReadCloser: resp.Body, budget: b, receivedBytes: continuation} - } - return resp, err -} - -type resolutionTransferBody struct { - io.ReadCloser - budget *resolutionBudget - receivedBytes bool // accessed only by the body's reader -} - -func (b *resolutionTransferBody) Read(p []byte) (int, error) { - // Pause only the native read, so JS progress callbacks (which can invoke - // more resolvers) and retry waits continue spending the same allowance. - if b.receivedBytes { - defer b.budget.pause()() - } else { - defer b.budget.charge()() - } - n, err := b.ReadCloser.Read(p) - if n > 0 { - b.receivedBytes = true - } - return n, err -} - -func (r *extensionRuntime) waitResolutionRetry(ctx context.Context, delay time.Duration) error { - if b := r.currentResolutionBudget(); b != nil { - defer b.charge()() - } - return waitTransferRetry(ctx, delay) -} diff --git a/go_backend/extension_resolution_budget_test.go b/go_backend/extension_resolution_budget_test.go deleted file mode 100644 index 751e5275..00000000 --- a/go_backend/extension_resolution_budget_test.go +++ /dev/null @@ -1,440 +0,0 @@ -package gobackend - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "sync" - "testing" - "time" - - "github.com/dop251/goja" -) - -func runResolutionScript(t *testing.T, r *extensionRuntime, parent context.Context, allowance time.Duration, script string) (goja.Value, error) { - t.Helper() - ctx, finish := r.beginResolutionBudget(parent, allowance) - defer finish() - return RunWithTimeoutContextAndRecover(ctx, r.vm, script, 3*time.Second) -} - -func TestResolutionBudgetInterruptsBlockedOperationsAsTimeout(t *testing.T) { - for _, operation := range []string{"http", "sleep", "signed-session", "busy-script"} { - t.Run(operation, func(t *testing.T) { - r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - <-req.Context().Done() - return nil, req.Context().Err() - }) - r.vm.Set("blockedHTTP", func() goja.Value { return r.doExtensionHTTP("GET", "https://cdn.example.com/api", nil, false, nil) }) - r.vm.Set("sleep", r.sleep) - r.vm.Set("signedSessionWait", func() goja.Value { - ctx, cancel := r.signedSessionExchangeContext() - defer cancel() - <-ctx.Done() - return r.vm.ToValue(false) - }) - script := map[string]string{ - "http": "blockedHTTP(); true", - "sleep": "sleep(300000); true", - "signed-session": "signedSessionWait(); true", - "busy-script": "while (true) {}", - }[operation] - started := time.Now() - _, err := runResolutionScript(t, r, context.Background(), 50*time.Millisecond, script) - if !IsTimeoutError(err) || errors.Is(err, ErrExtensionRequestCancelled) || IsRuntimeUnsafeError(err) { - t.Fatalf("expected safe timeout, got %v", err) - } - if time.Since(started) > time.Second { - t.Fatal("blocked operation did not stop promptly") - } - // The interrupt was cleared and the operation context was detached. - value, err := runResolutionScript(t, r, context.Background(), time.Second, "42") - if err != nil || value.ToInteger() != 42 { - t.Fatalf("reuse: %v, %v", value, err) - } - }) - } -} - -type resolutionSlowBody struct { - ctx context.Context - reads int -} - -func (b *resolutionSlowBody) Read(p []byte) (int, error) { - if b.reads == 3 { - return 0, io.EOF - } - if b.reads > 0 { - select { - case <-time.After(90 * time.Millisecond): - case <-b.ctx.Done(): - return 0, b.ctx.Err() - } - } - p[0] = 'a' - b.reads++ - return 1, nil -} -func (*resolutionSlowBody) Close() error { return nil } - -type resolutionDelayedBody struct { - io.ReadCloser - ctx context.Context - delay time.Duration -} - -func (b *resolutionDelayedBody) Read(p []byte) (int, error) { - if err := sleepRetry(b.ctx, b.delay); err != nil { - return 0, err - } - b.delay = 0 - return b.ReadCloser.Read(p) -} - -func TestResolutionBudgetAllowsResolvedSegments(t *testing.T) { - for _, maxParallel := range []int{1, 3} { - for _, phase := range []string{"headers", "first-byte"} { - t.Run(fmt.Sprintf("%s/parallel-%d", phase, maxParallel), func(t *testing.T) { - r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - body := &resolutionDelayedBody{ReadCloser: io.NopCloser(strings.NewReader("audio")), ctx: req.Context()} - if phase == "headers" { - if err := sleepRetry(req.Context(), 90*time.Millisecond); err != nil { - return nil, err - } - } else { - body.delay = 90 * time.Millisecond - } - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body, ContentLength: 5, Request: req}, nil - }) - r.vm.Set("segments", r.fileDownloadSegments) - script := fmt.Sprintf(` -var urls = []; -for (var i = 0; i < 8; i++) urls.push("https://cdn.example.com/segment/" + i); -segments(urls, "audio.flac", {maxParallel: %d, maxAttempts: 1}); -`, maxParallel) - started := time.Now() - value, err := runResolutionScript(t, r, context.Background(), 150*time.Millisecond, script) - if err != nil { - t.Fatal(err) - } - if result := value.Export().(map[string]any); result["success"] != true { - t.Fatalf("resolved segments failed: %#v", result) - } - if time.Since(started) < 250*time.Millisecond { - t.Fatal("transfer did not exceed resolution allowance") - } - data, err := os.ReadFile(filepath.Join(r.dataDir, "audio.flac")) - if err != nil || string(data) != strings.Repeat("audio", 8) { - t.Fatalf("output: %q, %v", data, err) - } - }) - } - } -} - -func TestResolutionBudgetAllowsResolvedChunks(t *testing.T) { - for _, phase := range []string{"headers", "first-byte"} { - t.Run(phase, func(t *testing.T) { - r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - var start, end int - if _, err := fmt.Sscanf(req.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil { - return nil, err - } - body := &resolutionDelayedBody{ReadCloser: io.NopCloser(strings.NewReader(strings.Repeat("a", end-start+1))), ctx: req.Context()} - if end-start > 1 { // The two-byte capability probe is immediate. - if phase == "headers" { - if err := sleepRetry(req.Context(), 90*time.Millisecond); err != nil { - return nil, err - } - } else { - body.delay = 90 * time.Millisecond - } - } - header := make(http.Header) - header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/40", start, end)) - return &http.Response{StatusCode: 206, Header: header, Body: body, ContentLength: int64(end - start + 1), Request: req}, nil - }) - r.vm.Set("download", r.fileDownload) - value, err := runResolutionScript(t, r, context.Background(), 150*time.Millisecond, `download("https://cdn.example.com/audio", "audio.flac", {chunked: 5, maxAttempts: 1})`) - if err != nil { - t.Fatal(err) - } - if result := value.Export().(map[string]any); result["success"] != true { - t.Fatalf("resolved chunks failed: %#v", result) - } - data, err := os.ReadFile(filepath.Join(r.dataDir, "audio.flac")) - if err != nil || string(data) != strings.Repeat("a", 40) { - t.Fatalf("output: %q, %v", data, err) - } - }) - } -} - -func TestResolutionBudgetChargesSegmentCallbacksAndRetryWaits(t *testing.T) { - for _, mode := range []string{"callback", "retry-wait", "new-transfer"} { - t.Run(mode, func(t *testing.T) { - waiting := make(chan struct{}) - r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - if strings.HasSuffix(req.URL.Path, "/blocked") { - close(waiting) - <-req.Context().Done() - return nil, req.Context().Err() - } - header := make(http.Header) - status := http.StatusOK - if strings.HasSuffix(req.URL.Path, "/retry") { - header.Set("Retry-After", "1") - status = http.StatusServiceUnavailable - } - return &http.Response{StatusCode: status, Header: header, Body: io.NopCloser(strings.NewReader("audio")), ContentLength: 5, Request: req}, nil - }) - r.vm.Set("segments", r.fileDownloadSegments) - r.vm.Set("sleep", r.sleep) - r.vm.Set("waitForNextRequest", func() { - select { - case <-waiting: - case <-time.After(time.Second): - t.Error("next segment request did not start") - } - }) - script := map[string]string{ - "callback": `segments(["https://cdn.example.com/first", "https://cdn.example.com/blocked"], "audio.flac", { -maxParallel: 1, maxAttempts: 1, onProgress: function() { waitForNextRequest(); sleep(1000); } -})`, - "retry-wait": `segments(["https://cdn.example.com/first", "https://cdn.example.com/second", "https://cdn.example.com/blocked", "https://cdn.example.com/retry"], "audio.flac", {maxParallel: 2, maxAttempts: 2})`, - "new-transfer": `segments(["https://cdn.example.com/first"], "first.flac", {maxAttempts: 1}); -segments(["https://cdn.example.com/blocked"], "audio.flac", {maxAttempts: 1});`, - }[mode] - started := time.Now() - _, err := runResolutionScript(t, r, context.Background(), 150*time.Millisecond, script) - if !IsTimeoutError(err) || time.Since(started) > 500*time.Millisecond { - t.Fatalf("resolver work escaped allowance: elapsed=%s err=%v", time.Since(started), err) - } - }) - } -} - -func TestResolutionBudgetContinuationPreservesStallCancellation(t *testing.T) { - r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: &resolutionDelayedBody{ - ReadCloser: io.NopCloser(strings.NewReader("audio")), ctx: req.Context(), delay: time.Second, - }, Request: req}, nil - }) - ctx, finish := r.beginResolutionBudget(context.Background(), 30*time.Millisecond) - defer finish() - req, err := http.NewRequestWithContext(ctx, "GET", "https://cdn.example.com/audio", nil) - if err != nil { - t.Fatal(err) - } - req, watchdog := bindStallWatchdog(req, 90*time.Millisecond) - defer watchdog.stop() - resp, err := r.doResolutionTransfer(r.httpClient, req, true) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - _, err = io.ReadAll(resp.Body) - if !errors.Is(err, context.Canceled) || !watchdog.stalled.Load() || ctx.Err() != nil { - t.Fatalf("stall cancellation lost: read=%v stalled=%v resolution=%v", err, watchdog.stalled.Load(), ctx.Err()) - } -} - -func TestResolutionBudgetAllowsActiveNativeTransfers(t *testing.T) { - for _, kind := range []string{"plain", "chunked", "segments"} { - t.Run(kind, func(t *testing.T) { - r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - header := make(http.Header) - header.Set("Content-Length", "3") - status := http.StatusOK - if req.Header.Get("Range") != "" { - status = http.StatusPartialContent - header.Set("Content-Range", "bytes 0-2/3") - } - var body io.ReadCloser = &resolutionSlowBody{ctx: req.Context()} - if req.Method == "HEAD" { - body = io.NopCloser(strings.NewReader("")) - } - return &http.Response{StatusCode: status, Header: header, Body: body, ContentLength: 3, Request: req}, nil - }) - r.vm.Set("download", r.fileDownload) - r.vm.Set("segments", r.fileDownloadSegments) - script := `download("https://cdn.example.com/audio", "audio.flac")` - if kind == "chunked" { - script = `download("https://cdn.example.com/audio", "audio.flac", {chunked:true})` - } - if kind == "segments" { - script = `segments(["https://cdn.example.com/audio"], "audio.flac")` - } - started := time.Now() - value, err := runResolutionScript(t, r, context.Background(), 70*time.Millisecond, script) - if err != nil { - t.Fatal(err) - } - if result := value.Export().(map[string]any); result["success"] != true { - t.Fatalf("transfer failed: %#v", result) - } - if time.Since(started) < 180*time.Millisecond { - t.Fatal("transfer did not exceed resolution allowance") - } - data, err := os.ReadFile(filepath.Join(r.dataDir, "audio.flac")) - if err != nil || string(data) != "aaa" { - t.Fatalf("output: %q, %v", data, err) - } - }) - } -} - -func TestResolutionBudgetDoesNotResetAcrossTransfersAndRefresh(t *testing.T) { - r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: &resolutionSlowBody{ctx: req.Context()}, ContentLength: 3, Request: req}, nil - }) - r.vm.Set("download", r.fileDownload) - r.vm.Set("sleep", r.sleep) - _, err := runResolutionScript(t, r, context.Background(), 120*time.Millisecond, ` - sleep(70); - var result = download("https://cdn.example.com/audio", "audio.flac"); - if (!result.success) throw new Error("transfer failed"); - sleep(70); // refresh must spend the remaining allowance, not a new 120ms - true; - `) - if !IsTimeoutError(err) { - t.Fatalf("expected cumulative timeout, got %v", err) - } -} - -func TestResolutionBudgetIncludesTransferFirstByteAndProgressCallbacks(t *testing.T) { - for _, mode := range []string{"first-byte", "callback"} { - t.Run(mode, func(t *testing.T) { - r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - body := &resolutionSlowBody{ctx: req.Context()} - if mode == "first-byte" { - body.reads = 1 - } - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body, ContentLength: 3, Request: req}, nil - }) - r.vm.Set("download", r.fileDownload) - r.vm.Set("sleep", r.sleep) - _, err := runResolutionScript(t, r, context.Background(), 60*time.Millisecond, `download("https://cdn.example.com/audio", "audio.flac", {onProgress:function() { sleep(1000); }})`) - if !IsTimeoutError(err) { - t.Fatalf("expected timeout, got %v", err) - } - }) - } -} - -func TestResolutionBudgetPreservesUserCancellationDuringTransfer(t *testing.T) { - parent, cancel := context.WithCancel(context.Background()) - defer cancel() - r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - time.AfterFunc(40*time.Millisecond, cancel) - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: &resolutionSlowBody{ctx: req.Context()}, ContentLength: 3, Request: req}, nil - }) - r.vm.Set("download", r.fileDownload) - _, err := runResolutionScript(t, r, parent, time.Second, `download("https://cdn.example.com/audio","audio.flac")`) - if !errors.Is(err, ErrExtensionRequestCancelled) || IsTimeoutError(err) { - t.Fatalf("expected cancellation, got %v", err) - } -} - -func TestResolutionBudgetConcurrentPausesAndStop(t *testing.T) { - b := newResolutionBudget(context.Background(), time.Second) - var workers sync.WaitGroup - for i := 0; i < 20; i++ { - workers.Add(1) - go func() { - defer workers.Done() - for j := 0; j < 100; j++ { - resume := b.pause() - b.remainingTime() - resume() - resume() - } - }() - } - workers.Wait() - b.stop() - if !errors.Is(context.Cause(b.ctx), context.Canceled) { - t.Fatalf("unexpected cause: %v", context.Cause(b.ctx)) - } -} - -func TestResolutionBudgetPoolClearsOperationContext(t *testing.T) { - ext := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - provider := newExtensionProviderWrapper(ext) - for i := 0; i < 2; i++ { - result, err := provider.Download("track-1", "LOSSLESS", filepath.Join(t.TempDir(), "audio.flac"), "", nil) - if err != nil || !result.Success { - t.Fatalf("download: %#v, %v", result, err) - } - ext.isolatedPoolMu.Lock() - if len(ext.isolatedPool) != 1 { - ext.isolatedPoolMu.Unlock() - t.Fatal("healthy runtime was not pooled") - } - r := ext.isolatedPool[0].runtime - ext.isolatedPoolMu.Unlock() - if r.currentResolutionBudget() != nil || r.activeOperationContext(context.Background()).Err() != nil { - t.Fatal("pooled runtime retained expired resolution context") - } - } -} - -func TestResolutionBudgetFFmpegWaitExcludesConversionAndHonorsCancellation(t *testing.T) { - for _, cancelled := range []bool{false, true} { - t.Run(map[bool]string{false: "complete", true: "cancel"}[cancelled], func(t *testing.T) { - r := newFileDownloadTestRuntime(t, nil) - r.extensionID = "resolution-ffmpeg-test" - r.vm.Set("convert", r.ffmpegConvert) - parent, cancel := context.WithCancel(context.Background()) - defer cancel() - responderDone := make(chan struct{}) - go func() { - defer close(responderDone) - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { - ffmpegCommandsMu.RLock() - id := "" - for key, command := range ffmpegCommands { - if command.ExtensionID == r.extensionID { - id = key - break - } - } - ffmpegCommandsMu.RUnlock() - if id != "" { - time.Sleep(120 * time.Millisecond) - if cancelled { - cancel() - } else { - SetFFmpegCommandResult(id, true, "converted", "") - } - return - } - time.Sleep(time.Millisecond) - } - }() - value, err := runResolutionScript(t, r, parent, 60*time.Millisecond, `convert("input.flac","output.flac",{codec:"flac"})`) - <-responderDone - if cancelled { - if !errors.Is(err, ErrExtensionRequestCancelled) { - t.Fatalf("expected cancel: %v", err) - } - } else if err != nil || value.Export().(map[string]any)["success"] != true { - t.Fatalf("conversion: %v, %v", value, err) - } - ffmpegCommandsMu.RLock() - defer ffmpegCommandsMu.RUnlock() - for _, command := range ffmpegCommands { - if command.ExtensionID == r.extensionID { - t.Fatal("FFmpeg command leaked") - } - } - }) - } -} diff --git a/go_backend/extension_runtime.go b/go_backend/extension_runtime.go deleted file mode 100644 index 78e0a219..00000000 --- a/go_backend/extension_runtime.go +++ /dev/null @@ -1,816 +0,0 @@ -package gobackend - -import ( - "context" - "crypto/rand" - "encoding/base64" - "fmt" - "net" - "net/http" - "net/http/cookiejar" - "net/url" - "slices" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/dop251/goja" -) - -// allowPrivateNetworkAccess, when enabled, disables the SSRF guard that blocks -// requests resolving to private/local/loopback addresses. This is opt-in and -// intended for users who route the app's traffic through a local proxy or -// custom DNS (e.g. a local mirror of api.zarz.moe). Disabled by default. -var allowPrivateNetworkAccess atomic.Bool - -// SetAllowPrivateNetwork toggles whether extensions and built-in network code -// are permitted to reach private/local network targets. Exposed to the Flutter -// layer via the platform bridge. -func SetAllowPrivateNetwork(allowed bool) { - allowPrivateNetworkAccess.Store(allowed) - if allowed { - GoLog("[HTTP] Private/local network access ENABLED (SSRF guard relaxed)\n") - } else { - GoLog("[HTTP] Private/local network access disabled (default)\n") - } -} - -// IsPrivateNetworkAllowed reports the current state of the private-network guard. -func IsPrivateNetworkAllowed() bool { - return allowPrivateNetworkAccess.Load() -} - -const DefaultJSTimeout = 30 * time.Second - -var ( - extensionAuthState = make(map[string]*ExtensionAuthState) - extensionAuthStateMu sync.RWMutex -) - -type ExtensionAuthState struct { - PendingAuthURL string - AuthCode string - AccessToken string - RefreshToken string - ExpiresAt time.Time - IsAuthenticated bool - PKCEVerifier string - PKCEChallenge string -} - -type PendingAuthRequest struct { - ExtensionID string - AuthURL string - CallbackURL string - State string - CreatedAt time.Time -} - -// Challenge URLs are short-lived; serving one past this age sends the user -// to an already-expired verification page. -const pendingAuthRequestTTL = 3 * time.Minute - -var ( - pendingAuthRequests = make(map[string]*PendingAuthRequest) - pendingAuthStates = make(map[string]string) - pendingAuthRequestsMu sync.RWMutex -) - -func newExtensionCallbackState() (string, error) { - random := make([]byte, 32) - if _, err := rand.Read(random); err != nil { - return "", fmt.Errorf("generate callback state: %w", err) - } - return base64.RawURLEncoding.EncodeToString(random), nil -} - -func registerPendingAuthRequest(request *PendingAuthRequest) error { - if request == nil || strings.TrimSpace(request.ExtensionID) == "" { - return fmt.Errorf("extension id is required") - } - if request.State == "" { - state, err := newExtensionCallbackState() - if err != nil { - return err - } - request.State = state - } - if request.CreatedAt.IsZero() { - request.CreatedAt = time.Now() - } - - pendingAuthRequestsMu.Lock() - if owner := pendingAuthStates[request.State]; owner != "" && owner != request.ExtensionID { - ownerRequest := pendingAuthRequests[owner] - sameChallenge := ownerRequest != nil && - ownerRequest.State == request.State && - ownerRequest.AuthURL == request.AuthURL && - ownerRequest.CallbackURL == request.CallbackURL && - ownerRequest.CreatedAt.Equal(request.CreatedAt) - if !sameChallenge { - pendingAuthRequestsMu.Unlock() - return fmt.Errorf("callback state is already registered") - } - } - if previous := pendingAuthRequests[request.ExtensionID]; previous != nil && previous.State != request.State { - removePendingAuthRequestLocked(request.ExtensionID) - } - pendingAuthRequests[request.ExtensionID] = request - if pendingAuthStates[request.State] == "" { - pendingAuthStates[request.State] = request.ExtensionID - } - pendingAuthRequestsMu.Unlock() - return nil -} - -func removePendingAuthRequestLocked(extensionID string) { - request := pendingAuthRequests[extensionID] - delete(pendingAuthRequests, extensionID) - if request == nil || pendingAuthStates[request.State] != extensionID { - return - } - delete(pendingAuthStates, request.State) - for candidateID, candidate := range pendingAuthRequests { - if candidate != nil && candidate.State == request.State && - time.Since(candidate.CreatedAt) < pendingAuthRequestTTL { - pendingAuthStates[request.State] = candidateID - return - } - } -} - -func removePendingAuthStateLocked(state string) { - delete(pendingAuthStates, state) - for extensionID, request := range pendingAuthRequests { - if request != nil && request.State == state { - delete(pendingAuthRequests, extensionID) - } - } -} - -func resolveExtensionCallbackStateLocked(state string) (string, error) { - extensionID := pendingAuthStates[state] - request := pendingAuthRequests[extensionID] - if extensionID == "" || request == nil || request.State != state || - time.Since(request.CreatedAt) >= pendingAuthRequestTTL { - removePendingAuthStateLocked(state) - return "", fmt.Errorf("callback state is invalid, expired, or already used") - } - return extensionID, nil -} - -// ResolveExtensionCallbackState validates a callback nonce without consuming -// it. Callback handlers use this before an exchange so a transient exchange -// failure can still be retried with the same short-lived challenge. -func ResolveExtensionCallbackState(state string) (string, error) { - state = strings.TrimSpace(state) - if state == "" { - return "", fmt.Errorf("callback state is required") - } - - pendingAuthRequestsMu.Lock() - extensionID, err := resolveExtensionCallbackStateLocked(state) - pendingAuthRequestsMu.Unlock() - return extensionID, err -} - -func ConsumeExtensionCallbackState(state string) (string, error) { - state = strings.TrimSpace(state) - if state == "" { - return "", fmt.Errorf("callback state is required") - } - - pendingAuthRequestsMu.Lock() - extensionID, err := resolveExtensionCallbackStateLocked(state) - if err != nil { - pendingAuthRequestsMu.Unlock() - return "", err - } - removePendingAuthStateLocked(state) - pendingAuthRequestsMu.Unlock() - return extensionID, nil -} - -func GetPendingAuthRequest(extensionID string) *PendingAuthRequest { - pendingAuthRequestsMu.RLock() - defer pendingAuthRequestsMu.RUnlock() - return pendingAuthRequests[extensionID] -} - -func ClearPendingAuthRequest(extensionID string) { - pendingAuthRequestsMu.Lock() - defer pendingAuthRequestsMu.Unlock() - removePendingAuthRequestLocked(extensionID) -} - -func SetExtensionAuthCode(extensionID string, authCode string) { - extensionAuthStateMu.Lock() - defer extensionAuthStateMu.Unlock() - - state, exists := extensionAuthState[extensionID] - if !exists { - state = &ExtensionAuthState{} - extensionAuthState[extensionID] = state - } - state.AuthCode = authCode -} - -func SetExtensionTokens(extensionID string, accessToken, refreshToken string, expiresAt time.Time) { - extensionAuthStateMu.Lock() - defer extensionAuthStateMu.Unlock() - - state, exists := extensionAuthState[extensionID] - if !exists { - state = &ExtensionAuthState{} - extensionAuthState[extensionID] = state - } - state.AccessToken = accessToken - state.RefreshToken = refreshToken - state.ExpiresAt = expiresAt - state.IsAuthenticated = accessToken != "" -} - -type extensionRuntime struct { - extensionID string - manifest *ExtensionManifest - settings map[string]any - httpClient *http.Client - downloadClient *http.Client - cookieJar http.CookieJar - dataDir string - vm *goja.Runtime - - activeDownloadMu sync.RWMutex - activeDownloadItemID string - - resolutionMu sync.RWMutex - resolutionBudget *resolutionBudget - - activeRequestMu sync.RWMutex - activeRequestID string - - storageMu sync.RWMutex - storageCache map[string]any - storageClosed bool - - credentialsMu sync.RWMutex - credentialsCache map[string]any - - // Set when a signed-session call inside the current script invocation - // required verification. The provider wrapper consumes it after the - // script returns, so verification surfaces even when the extension - // script swallowed the needsVerification response (issue: fallback - // skipped provider B's challenge and failed outright). - verificationMu sync.Mutex - verificationRequiredURL string -} - -func (r *extensionRuntime) noteVerificationRequired(authURL string) { - r.verificationMu.Lock() - if authURL == "" { - authURL = "pending" - } - r.verificationRequiredURL = authURL - r.verificationMu.Unlock() -} - -// consumeVerificationRequired returns the noted auth URL (or "pending") and -// clears the flag; "" means no verification was requested since the last -// consume. -func (r *extensionRuntime) consumeVerificationRequired() string { - r.verificationMu.Lock() - url := r.verificationRequiredURL - r.verificationRequiredURL = "" - r.verificationMu.Unlock() - return url -} - -type privateIPCacheEntry struct { - isPrivate bool - expiresAt time.Time -} - -const ( - privateIPCacheTTL = 5 * time.Minute - privateIPErrorCacheTTL = 30 * time.Second - maxPrivateIPCacheSize = 1024 -) - -var ( - privateIPCache = make(map[string]privateIPCacheEntry) - privateIPCacheMu sync.RWMutex -) - -func clearPrivateIPCache() { - privateIPCacheMu.Lock() - privateIPCache = make(map[string]privateIPCacheEntry) - privateIPCacheMu.Unlock() -} - -func newExtensionRuntime(ext *loadedExtension) *extensionRuntime { - jar, _ := newSimpleCookieJar() - - runtime := &extensionRuntime{ - extensionID: ext.ID, - manifest: ext.Manifest, - settings: make(map[string]any), - cookieJar: jar, - dataDir: ext.DataDir, - vm: ext.VM, - } - - runtime.httpClient = newExtensionHTTPClient(ext, jar, extensionHTTPTimeout(ext, 30*time.Second), true) - runtime.downloadClient = newExtensionHTTPClient(ext, jar, DownloadTimeout, false) - - return runtime -} - -func extensionHTTPTimeout(ext *loadedExtension, fallback time.Duration) time.Duration { - if ext == nil || ext.Manifest == nil || ext.Manifest.Capabilities == nil { - return fallback - } - - raw, ok := ext.Manifest.Capabilities["networkTimeoutSeconds"] - if !ok { - return fallback - } - - seconds := parseExtensionTimeoutSeconds(raw) - if seconds <= 0 { - return fallback - } - - if seconds < 5 { - seconds = 5 - } - if seconds > 300 { - seconds = 300 - } - - return time.Duration(seconds) * time.Second -} - -func parseExtensionTimeoutSeconds(raw any) int { - switch v := raw.(type) { - case int: - return v - case int32: - return int(v) - case int64: - return int(v) - case float32: - return int(v) - case float64: - return int(v) - case string: - parsed, err := strconv.Atoi(strings.TrimSpace(v)) - if err != nil { - return 0 - } - return parsed - default: - return 0 - } -} - -func (r *extensionRuntime) setActiveDownloadItemID(itemID string) { - r.activeDownloadMu.Lock() - defer r.activeDownloadMu.Unlock() - r.activeDownloadItemID = strings.TrimSpace(itemID) -} - -func (r *extensionRuntime) clearActiveDownloadItemID() { - r.activeDownloadMu.Lock() - defer r.activeDownloadMu.Unlock() - r.activeDownloadItemID = "" -} - -func (r *extensionRuntime) getActiveDownloadItemID() string { - r.activeDownloadMu.RLock() - defer r.activeDownloadMu.RUnlock() - return r.activeDownloadItemID -} - -func (r *extensionRuntime) setActiveRequestID(requestID string) { - r.activeRequestMu.Lock() - defer r.activeRequestMu.Unlock() - r.activeRequestID = strings.TrimSpace(requestID) -} - -func (r *extensionRuntime) clearActiveRequestID() { - r.activeRequestMu.Lock() - defer r.activeRequestMu.Unlock() - r.activeRequestID = "" -} - -func (r *extensionRuntime) getActiveRequestID() string { - r.activeRequestMu.RLock() - defer r.activeRequestMu.RUnlock() - return r.activeRequestID -} - -func (r *extensionRuntime) bindDownloadCancelContext(req *http.Request) *http.Request { - if req == nil { - return nil - } - return req.WithContext(r.activeOperationContext(req.Context())) -} - -// activeOperationContext is stable for the full extension operation. An -// http.Client with a finite Timeout derives a per-request child context and -// cancels it when that response body closes, so that request context must not -// be reused for provider retry delays between requests. -func (r *extensionRuntime) activeOperationContext(fallback context.Context) context.Context { - if budget := r.currentResolutionBudget(); budget != nil { - return budget.ctx - } - itemID := r.getActiveDownloadItemID() - if itemID == "" { - requestID := r.getActiveRequestID() - if requestID == "" { - if fallback != nil { - return fallback - } - return context.Background() - } - return extensionRequestCancelContext(requestID) - } - - return downloadCancelContext(itemID) -} - -// downloadStallTimeout is how long a download may go without receiving a single -// byte before the stall watchdog aborts it. A dead radio mid-transfer otherwise -// blocks on Body.Read until the 24h client timeout with no error and no retry. -const downloadStallTimeout = 60 * time.Second - -// stallWatchdog cancels an in-flight download when no data arrives within -// timeout. It wraps the request context in a child cancel so firing it does NOT -// set the user-cancel flag (isDownloadCancelled stays false) — a stall is a -// distinct, retryable condition. Call reset() after every successful Read and -// stop() when the transfer ends. -type stallWatchdog struct { - cancel context.CancelFunc - timer *time.Timer - timeout time.Duration - stalled atomic.Bool -} - -func bindStallWatchdog(req *http.Request, timeout time.Duration) (*http.Request, *stallWatchdog) { - ctx, cancel := context.WithCancel(req.Context()) - w := &stallWatchdog{cancel: cancel, timeout: timeout} - w.timer = time.AfterFunc(timeout, func() { - w.stalled.Store(true) - cancel() - }) - return req.WithContext(ctx), w -} - -func (w *stallWatchdog) reset() { w.timer.Reset(w.timeout) } - -// stop halts the timer and releases the child context so a completed download -// leaks neither a pending timer nor a live cancel func. -func (w *stallWatchdog) stop() { - w.timer.Stop() - w.cancel() -} - -func newExtensionHTTPClient(ext *loadedExtension, jar http.CookieJar, timeout time.Duration, compressResponses bool) *http.Client { - // Extension sandbox enforces HTTPS-only domains. Do not apply global - // allow_http scheme downgrade here, because some extension APIs (e.g. - // spotify-web) will redirect http -> https and can end up in 301 loops. - // API calls can use response compression for faster metadata/search loads, - // while media downloads keep identity transfer semantics for progress/streaming. - transport := sharedTransport - if compressResponses { - transport = extensionAPITransport - } - client := &http.Client{ - Transport: transport, - Timeout: timeout, - Jar: jar, - } - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - if req.URL.Scheme != "https" && - !(req.URL.Scheme == "http" && ext.Manifest.Permissions.AllowHTTP) { - GoLog("[Extension:%s] Redirect blocked: non-https scheme '%s'\n", ext.ID, req.URL.Scheme) - return fmt.Errorf("redirect blocked: only https is allowed") - } - - domain := req.URL.Hostname() - if domain == "" { - GoLog("[Extension:%s] Redirect blocked: missing hostname\n", ext.ID) - return fmt.Errorf("redirect blocked: hostname is required") - } - if !ext.Manifest.IsDomainAllowed(domain) { - GoLog("[Extension:%s] Redirect blocked: domain '%s' not in allowed list\n", ext.ID, domain) - return &RedirectBlockedError{Domain: domain} - } - // The transport resolves and pins every redirect target before dialing. - // Reject literals/local aliases here without doing a second, uncancellable - // DNS lookup on the redirect path. - if isPrivateIPLiteralOrLocal(domain) { - GoLog("[Extension:%s] Redirect blocked: private IP '%s'\n", ext.ID, domain) - return &RedirectBlockedError{Domain: domain, IsPrivate: true} - } - if len(via) >= 10 { - return http.ErrUseLastResponse - } - return nil - } - return client -} - -type RedirectBlockedError struct { - Domain string - IsPrivate bool -} - -func (e *RedirectBlockedError) Error() string { - if e.IsPrivate { - return "redirect blocked: private/local network access denied" - } - return "redirect blocked: domain '" + e.Domain + "' not in allowed list" -} - -func isPrivateIP(host string) bool { - // Opt-in escape hatch: when the user has enabled private/local network - // access, treat every host as public so local proxies / custom DNS work. - if allowPrivateNetworkAccess.Load() { - return false - } - - hostLower := strings.ToLower(strings.TrimSpace(host)) - if hostLower == "" { - return false - } - if isPrivateIPLiteralOrLocal(hostLower) { - return true - } - - if cached, ok := getPrivateIPCache(hostLower); ok { - return cached - } - - ips, err := net.LookupIP(hostLower) - if err != nil { - // Defer the final decision to dialWithDoHFallback. It resolves and filters - // every concrete address (including DoH answers) before opening a socket. - setPrivateIPCache(hostLower, false, privateIPErrorCacheTTL) - return false - } - - isPrivate := slices.ContainsFunc(ips, isPrivateIPAddr) - - setPrivateIPCache(hostLower, isPrivate, privateIPCacheTTL) - return isPrivate -} - -// isPrivateIPLiteralOrLocal performs the validation that does not require DNS. -// Extension HTTP requests use this before dispatch; dialWithDoHFallback remains -// the authoritative hostname check because it filters and pins the exact DNS -// answers used by the socket, closing the rebinding window without a duplicate -// lookup. -func isPrivateIPLiteralOrLocal(host string) bool { - if allowPrivateNetworkAccess.Load() { - return false - } - host = strings.ToLower(strings.TrimSpace(host)) - if host == "" { - return false - } - if host == "localhost" || strings.HasSuffix(host, ".local") { - return true - } - if ip := net.ParseIP(host); ip != nil { - return isPrivateIPAddr(ip) - } - return false -} - -func getPrivateIPCache(host string) (bool, bool) { - now := time.Now() - - privateIPCacheMu.RLock() - entry, exists := privateIPCache[host] - privateIPCacheMu.RUnlock() - if !exists { - return false, false - } - - if now.Before(entry.expiresAt) { - return entry.isPrivate, true - } - - privateIPCacheMu.Lock() - delete(privateIPCache, host) - privateIPCacheMu.Unlock() - return false, false -} - -func setPrivateIPCache(host string, isPrivate bool, ttl time.Duration) { - expiresAt := time.Now().Add(ttl) - - privateIPCacheMu.Lock() - if len(privateIPCache) >= maxPrivateIPCacheSize { - now := time.Now() - for key, entry := range privateIPCache { - if now.After(entry.expiresAt) { - delete(privateIPCache, key) - } - } - if len(privateIPCache) >= maxPrivateIPCacheSize { - privateIPCache = make(map[string]privateIPCacheEntry) - } - } - privateIPCache[host] = privateIPCacheEntry{ - isPrivate: isPrivate, - expiresAt: expiresAt, - } - privateIPCacheMu.Unlock() -} - -func isPrivateIPAddr(ip net.IP) bool { - if ip == nil { - return false - } - if ip.IsLoopback() || - ip.IsPrivate() || - ip.IsLinkLocalUnicast() || - ip.IsLinkLocalMulticast() || - ip.IsMulticast() || - ip.IsUnspecified() { - return true - } - if !ip.IsGlobalUnicast() { - return true - } - return false -} - -type simpleCookieJar struct { - mu sync.RWMutex - jar *cookiejar.Jar -} - -func newSimpleCookieJar() (*simpleCookieJar, error) { - jar, err := cookiejar.New(nil) - if err != nil { - return nil, err - } - return &simpleCookieJar{jar: jar}, nil -} - -func (j *simpleCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) { - j.mu.RLock() - defer j.mu.RUnlock() - j.jar.SetCookies(u, cookies) -} - -func (j *simpleCookieJar) Cookies(u *url.URL) []*http.Cookie { - j.mu.RLock() - defer j.mu.RUnlock() - return j.jar.Cookies(u) - -} - -func (j *simpleCookieJar) Clear() { - jar, _ := cookiejar.New(nil) - j.mu.Lock() - j.jar = jar - j.mu.Unlock() -} - -func (r *extensionRuntime) SetSettings(settings map[string]any) { - r.settings = settings -} - -func (r *extensionRuntime) RegisterAPIs(vm *goja.Runtime) { - r.vm = vm - - httpObj := vm.NewObject() - httpObj.Set("get", r.httpGet) - httpObj.Set("post", r.httpPost) - httpObj.Set("put", r.httpPut) - httpObj.Set("delete", r.httpDelete) - httpObj.Set("patch", r.httpPatch) - httpObj.Set("request", r.httpRequest) - httpObj.Set("clearCookies", r.httpClearCookies) - vm.Set("http", httpObj) - - if r.manifest != nil && r.manifest.Permissions.Storage { - storageObj := vm.NewObject() - storageObj.Set("get", r.storageGet) - storageObj.Set("set", r.storageSet) - storageObj.Set("remove", r.storageRemove) - vm.Set("storage", storageObj) - - credentialsObj := vm.NewObject() - credentialsObj.Set("store", r.credentialsStore) - credentialsObj.Set("get", r.credentialsGet) - credentialsObj.Set("remove", r.credentialsRemove) - credentialsObj.Set("has", r.credentialsHas) - vm.Set("credentials", credentialsObj) - } - - if r.manifest != nil && r.manifest.Permissions.Storage { - authObj := vm.NewObject() - authObj.Set("openAuthUrl", r.authOpenUrl) - authObj.Set("getAuthCode", r.authGetCode) - authObj.Set("setAuthCode", r.authSetCode) - authObj.Set("clearAuth", r.authClear) - authObj.Set("isAuthenticated", r.authIsAuthenticated) - authObj.Set("getTokens", r.authGetTokens) - authObj.Set("generatePKCE", r.authGeneratePKCE) - authObj.Set("getPKCE", r.authGetPKCE) - authObj.Set("startOAuthWithPKCE", r.authStartOAuthWithPKCE) - authObj.Set("exchangeCodeWithPKCE", r.authExchangeCodeWithPKCE) - vm.Set("auth", authObj) - - if r.manifest.SignedSession != nil { - sessionObj := vm.NewObject() - sessionObj.Set("signedFetch", r.signedSessionFetch) - sessionObj.Set("completeGrant", r.signedSessionCompleteGrant) - sessionObj.Set("status", r.signedSessionStatus) - sessionObj.Set("clear", r.signedSessionClear) - vm.Set("session", sessionObj) - } - } - - if r.manifest != nil && r.manifest.Permissions.File { - fileObj := vm.NewObject() - fileObj.Set("download", r.fileDownload) - fileObj.Set("downloadSegments", r.fileDownloadSegments) - fileObj.Set("exists", r.fileExists) - fileObj.Set("delete", r.fileDelete) - fileObj.Set("read", r.fileRead) - fileObj.Set("readBytes", r.fileReadBytes) - fileObj.Set("write", r.fileWrite) - fileObj.Set("writeBytes", r.fileWriteBytes) - fileObj.Set("copy", r.fileCopy) - fileObj.Set("move", r.fileMove) - fileObj.Set("getSize", r.fileGetSize) - fileObj.Set("transformPatternedBlocks", r.fileTransformPatternedBlocks) - vm.Set("file", fileObj) - - ffmpegObj := vm.NewObject() - if r.manifest.HasCapability("rawFfmpeg") { - ffmpegObj.Set("execute", r.ffmpegExecute) - } - ffmpegObj.Set("getInfo", r.ffmpegGetInfo) - ffmpegObj.Set("convert", r.ffmpegConvert) - vm.Set("ffmpeg", ffmpegObj) - } - - matchingObj := vm.NewObject() - matchingObj.Set("compareStrings", r.matchingCompareStrings) - matchingObj.Set("compareDuration", r.matchingCompareDuration) - matchingObj.Set("normalizeString", r.matchingNormalizeString) - vm.Set("matching", matchingObj) - - utilsObj := vm.NewObject() - utilsObj.Set("base64Encode", r.base64Encode) - utilsObj.Set("base64Decode", r.base64Decode) - utilsObj.Set("md5", r.md5Hash) - utilsObj.Set("sha256", r.sha256Hash) - utilsObj.Set("hmacSHA256", r.hmacSHA256) - utilsObj.Set("hmacSHA256Base64", r.hmacSHA256Base64) - utilsObj.Set("hmacSHA1", r.hmacSHA1) - utilsObj.Set("parseJSON", r.parseJSON) - utilsObj.Set("stringifyJSON", r.stringifyJSON) - utilsObj.Set("encrypt", r.cryptoEncrypt) - utilsObj.Set("decrypt", r.cryptoDecrypt) - utilsObj.Set("encryptBlockCipher", r.encryptBlockCipher) - utilsObj.Set("decryptBlockCipher", r.decryptBlockCipher) - utilsObj.Set("decryptCTRSegments", r.decryptCTRSegments) - utilsObj.Set("generateKey", r.cryptoGenerateKey) - utilsObj.Set("randomUserAgent", r.randomUserAgent) - utilsObj.Set("appVersion", r.appVersion) - utilsObj.Set("appUserAgent", r.appUserAgent) - utilsObj.Set("sleep", r.sleep) - utilsObj.Set("getResolutionRemainingMs", r.getResolutionRemainingMs) - utilsObj.Set("isDownloadCancelled", r.isDownloadCancelled) - utilsObj.Set("isRequestCancelled", r.isRequestCancelled) - utilsObj.Set("setDownloadStatus", r.setDownloadStatus) - vm.Set("utils", utilsObj) - - logObj := vm.NewObject() - logObj.Set("debug", r.logDebug) - logObj.Set("info", r.logInfo) - logObj.Set("warn", r.logWarn) - logObj.Set("error", r.logError) - vm.Set("log", logObj) - - vm.Set("fetch", r.fetchPolyfill) - - vm.Set("atob", r.atobPolyfill) - vm.Set("btoa", r.btoaPolyfill) - - r.registerTextEncoderDecoder(vm) - - r.registerURLClass(vm) - - r.registerJSONGlobal(vm) -} diff --git a/go_backend/extension_runtime_auth.go b/go_backend/extension_runtime_auth.go deleted file mode 100644 index 72cbb772..00000000 --- a/go_backend/extension_runtime_auth.go +++ /dev/null @@ -1,525 +0,0 @@ -package gobackend - -import ( - "crypto/rand" - "crypto/sha256" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/dop251/goja" -) - -func validateExtensionAuthURL(urlStr string) error { - parsed, err := url.Parse(urlStr) - if err != nil { - return fmt.Errorf("invalid auth URL: %w", err) - } - - if parsed.Scheme != "https" { - return fmt.Errorf("invalid auth URL: only https is allowed") - } - - host := parsed.Hostname() - if host == "" { - return fmt.Errorf("invalid auth URL: hostname is required") - } - - if parsed.User != nil { - return fmt.Errorf("invalid auth URL: embedded credentials are not allowed") - } - - if isPrivateIP(host) { - return fmt.Errorf("invalid auth URL: private/local network is not allowed") - } - - return nil -} - -func summarizeURLForLog(urlStr string) string { - parsed, err := url.Parse(urlStr) - if err != nil { - return urlStr - } - if parsed.Host == "" { - return parsed.Scheme + "://" - } - return fmt.Sprintf("%s://%s%s", parsed.Scheme, parsed.Host, parsed.Path) -} - -func setOAuthState(urlStr, state string) (string, error) { - parsed, err := url.Parse(urlStr) - if err != nil { - return "", err - } - query := parsed.Query() - query.Set("state", state) - parsed.RawQuery = query.Encode() - return parsed.String(), nil -} - -func (r *extensionRuntime) authOpenUrl(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.jsError("auth URL is required") - } - - authURL := call.Arguments[0].String() - callbackURL := "" - if len(call.Arguments) > 1 && !goja.IsUndefined(call.Arguments[1]) { - callbackURL = call.Arguments[1].String() - } - - if err := validateExtensionAuthURL(authURL); err != nil { - return r.jsError("%s", err.Error()) - } - callbackState, err := newExtensionCallbackState() - if err != nil { - return r.jsError("%s", err.Error()) - } - authURL, err = setOAuthState(authURL, callbackState) - if err != nil { - return r.jsError("invalid auth URL: %v", err) - } - if err := registerPendingAuthRequest(&PendingAuthRequest{ - ExtensionID: r.extensionID, - AuthURL: authURL, - CallbackURL: callbackURL, - State: callbackState, - CreatedAt: time.Now(), - }); err != nil { - return r.jsError("%s", err.Error()) - } - - extensionAuthStateMu.Lock() - state, exists := extensionAuthState[r.extensionID] - if !exists { - state = &ExtensionAuthState{} - extensionAuthState[r.extensionID] = state - } - state.PendingAuthURL = authURL - state.AuthCode = "" - extensionAuthStateMu.Unlock() - - GoLog("[Extension:%s] Auth URL requested: %s\n", r.extensionID, summarizeURLForLog(authURL)) - - return r.jsSuccess(map[string]any{ - "message": "Auth URL will be opened by the app", - }) -} - -func (r *extensionRuntime) authGetCode(call goja.FunctionCall) goja.Value { - extensionAuthStateMu.RLock() - defer extensionAuthStateMu.RUnlock() - - state, exists := extensionAuthState[r.extensionID] - if !exists || state.AuthCode == "" { - return goja.Undefined() - } - - return r.vm.ToValue(state.AuthCode) -} - -func (r *extensionRuntime) authSetCode(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue(false) - } - - arg := call.Arguments[0].Export() - - extensionAuthStateMu.Lock() - defer extensionAuthStateMu.Unlock() - - state, exists := extensionAuthState[r.extensionID] - if !exists { - state = &ExtensionAuthState{} - extensionAuthState[r.extensionID] = state - } - - switch v := arg.(type) { - case string: - state.AuthCode = v - case map[string]any: - if code, ok := v["code"].(string); ok { - state.AuthCode = code - } - if accessToken, ok := v["access_token"].(string); ok { - state.AccessToken = accessToken - state.IsAuthenticated = true - } - if refreshToken, ok := v["refresh_token"].(string); ok { - state.RefreshToken = refreshToken - } - if expiresIn, ok := v["expires_in"].(float64); ok { - state.ExpiresAt = time.Now().Add(time.Duration(expiresIn) * time.Second) - } - } - - return r.vm.ToValue(true) -} - -func (r *extensionRuntime) authClear(call goja.FunctionCall) goja.Value { - extensionAuthStateMu.Lock() - delete(extensionAuthState, r.extensionID) - extensionAuthStateMu.Unlock() - - ClearPendingAuthRequest(r.extensionID) - - GoLog("[Extension:%s] Auth state cleared\n", r.extensionID) - return r.vm.ToValue(true) -} - -func (r *extensionRuntime) authIsAuthenticated(call goja.FunctionCall) goja.Value { - extensionAuthStateMu.RLock() - defer extensionAuthStateMu.RUnlock() - - state, exists := extensionAuthState[r.extensionID] - if !exists { - return r.vm.ToValue(false) - } - - if state.IsAuthenticated && !state.ExpiresAt.IsZero() && time.Now().After(state.ExpiresAt) { - return r.vm.ToValue(false) - } - - return r.vm.ToValue(state.IsAuthenticated) -} - -func (r *extensionRuntime) authGetTokens(call goja.FunctionCall) goja.Value { - extensionAuthStateMu.RLock() - defer extensionAuthStateMu.RUnlock() - - state, exists := extensionAuthState[r.extensionID] - if !exists { - return r.vm.ToValue(map[string]any{}) - } - - result := map[string]any{ - "access_token": state.AccessToken, - "refresh_token": state.RefreshToken, - "is_authenticated": state.IsAuthenticated, - } - - if !state.ExpiresAt.IsZero() { - result["expires_at"] = state.ExpiresAt.Unix() - result["is_expired"] = time.Now().After(state.ExpiresAt) - } - - return r.vm.ToValue(result) -} - -func generatePKCEVerifier(length int) (string, error) { - if length < 43 { - length = 43 - } - if length > 128 { - length = 128 - } - - bytes := make([]byte, length) - if _, err := rand.Read(bytes); err != nil { - return "", err - } - - verifier := base64.RawURLEncoding.EncodeToString(bytes) - - if len(verifier) > length { - verifier = verifier[:length] - } - - return verifier, nil -} - -func generatePKCEChallenge(verifier string) string { - hash := sha256.Sum256([]byte(verifier)) - return base64.RawURLEncoding.EncodeToString(hash[:]) -} - -func (r *extensionRuntime) authGeneratePKCE(call goja.FunctionCall) goja.Value { - length := 64 - if len(call.Arguments) > 0 && !goja.IsUndefined(call.Arguments[0]) { - if l, ok := call.Arguments[0].Export().(float64); ok && l >= 43 && l <= 128 { - length = int(l) - } - } - - verifier, err := generatePKCEVerifier(length) - if err != nil { - GoLog("[Extension:%s] PKCE generation error: %v\n", r.extensionID, err) - return r.vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - - challenge := generatePKCEChallenge(verifier) - - extensionAuthStateMu.Lock() - state, exists := extensionAuthState[r.extensionID] - if !exists { - state = &ExtensionAuthState{} - extensionAuthState[r.extensionID] = state - } - state.PKCEVerifier = verifier - state.PKCEChallenge = challenge - extensionAuthStateMu.Unlock() - - GoLog("[Extension:%s] PKCE generated (verifier length: %d)\n", r.extensionID, len(verifier)) - - return r.vm.ToValue(map[string]any{ - "verifier": verifier, - "challenge": challenge, - "method": "S256", - }) -} - -func (r *extensionRuntime) authGetPKCE(call goja.FunctionCall) goja.Value { - extensionAuthStateMu.RLock() - defer extensionAuthStateMu.RUnlock() - - state, exists := extensionAuthState[r.extensionID] - if !exists || state.PKCEVerifier == "" { - return r.vm.ToValue(map[string]any{}) - } - - return r.vm.ToValue(map[string]any{ - "verifier": state.PKCEVerifier, - "challenge": state.PKCEChallenge, - "method": "S256", - }) -} - -func (r *extensionRuntime) authStartOAuthWithPKCE(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.jsError("config object is required") - } - - configObj := call.Arguments[0].Export() - config, ok := configObj.(map[string]any) - if !ok { - return r.jsError("config must be an object") - } - - authURL, _ := config["authUrl"].(string) - clientID, _ := config["clientId"].(string) - redirectURI, _ := config["redirectUri"].(string) - - if authURL == "" || clientID == "" || redirectURI == "" { - return r.jsError("authUrl, clientId, and redirectUri are required") - } - if err := validateExtensionAuthURL(authURL); err != nil { - return r.jsError("%s", err.Error()) - } - - scope, _ := config["scope"].(string) - extraParams, _ := config["extraParams"].(map[string]any) - - verifier, err := generatePKCEVerifier(64) - if err != nil { - return r.jsError("failed to generate PKCE: %v", err) - } - challenge := generatePKCEChallenge(verifier) - - extensionAuthStateMu.Lock() - state, exists := extensionAuthState[r.extensionID] - if !exists { - state = &ExtensionAuthState{} - extensionAuthState[r.extensionID] = state - } - state.PKCEVerifier = verifier - state.PKCEChallenge = challenge - state.AuthCode = "" - extensionAuthStateMu.Unlock() - - parsedURL, err := url.Parse(authURL) - if err != nil { - return r.jsError("invalid authUrl: %v", err) - } - - query := parsedURL.Query() - query.Set("client_id", clientID) - query.Set("redirect_uri", redirectURI) - query.Set("response_type", "code") - query.Set("code_challenge", challenge) - query.Set("code_challenge_method", "S256") - - if scope != "" { - query.Set("scope", scope) - } - - for k, v := range extraParams { - query.Set(k, fmt.Sprintf("%v", v)) - } - callbackState, err := newExtensionCallbackState() - if err != nil { - return r.jsError("failed to generate OAuth state: %v", err) - } - // Host-generated state always wins over extension-supplied extraParams. - query.Set("state", callbackState) - - parsedURL.RawQuery = query.Encode() - fullAuthURL := parsedURL.String() - - if err := registerPendingAuthRequest(&PendingAuthRequest{ - ExtensionID: r.extensionID, - AuthURL: fullAuthURL, - CallbackURL: redirectURI, - State: callbackState, - CreatedAt: time.Now(), - }); err != nil { - return r.jsError("failed to register OAuth callback: %v", err) - } - - GoLog("[Extension:%s] PKCE OAuth started: %s\n", r.extensionID, summarizeURLForLog(fullAuthURL)) - - return r.jsSuccess(map[string]any{ - "authUrl": fullAuthURL, - "pkce": map[string]any{ - "verifier": verifier, - "challenge": challenge, - "method": "S256", - }, - }) -} - -func (r *extensionRuntime) authExchangeCodeWithPKCE(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.jsError("config object is required") - } - - configObj := call.Arguments[0].Export() - config, ok := configObj.(map[string]any) - if !ok { - return r.jsError("config must be an object") - } - - tokenURL, _ := config["tokenUrl"].(string) - clientID, _ := config["clientId"].(string) - redirectURI, _ := config["redirectUri"].(string) - code, _ := config["code"].(string) - - if tokenURL == "" || clientID == "" || code == "" { - return r.jsError("tokenUrl, clientId, and code are required") - } - - extensionAuthStateMu.RLock() - state, exists := extensionAuthState[r.extensionID] - var verifier string - if exists { - verifier = state.PKCEVerifier - } - extensionAuthStateMu.RUnlock() - - if verifier == "" { - return r.jsError("no PKCE verifier found - call generatePKCE or startOAuthWithPKCE first") - } - - if err := r.validateDomain(tokenURL); err != nil { - return r.jsError("%s", err.Error()) - } - - formData := url.Values{} - formData.Set("grant_type", "authorization_code") - formData.Set("client_id", clientID) - formData.Set("code", code) - formData.Set("code_verifier", verifier) - if redirectURI != "" { - formData.Set("redirect_uri", redirectURI) - } - - if extraParams, ok := config["extraParams"].(map[string]any); ok { - for k, v := range extraParams { - formData.Set(k, fmt.Sprintf("%v", v)) - } - } - - req, err := http.NewRequest("POST", tokenURL, strings.NewReader(formData.Encode())) - if err != nil { - return r.jsError("%s", err.Error()) - } - req = r.bindDownloadCancelContext(req) - - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("User-Agent", appUserAgent()) - - resp, err := r.httpClient.Do(req) - if err != nil { - return r.jsError("%s", err.Error()) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return r.jsError("%s", err.Error()) - } - bodyPreview := sanitizeSensitiveLogText(string(body)) - if len(bodyPreview) > 1000 { - bodyPreview = bodyPreview[:1000] + "...[truncated]" - } - - var tokenResp map[string]any - if err := json.Unmarshal(body, &tokenResp); err != nil { - return r.vm.ToValue(map[string]any{ - "success": false, - "error": fmt.Sprintf("failed to parse token response: %v", err), - "body": bodyPreview, - }) - } - - if errMsg, ok := tokenResp["error"].(string); ok { - errDesc, _ := tokenResp["error_description"].(string) - return r.vm.ToValue(map[string]any{ - "success": false, - "error": errMsg, - "error_description": errDesc, - }) - } - - accessToken, _ := tokenResp["access_token"].(string) - refreshToken, _ := tokenResp["refresh_token"].(string) - expiresIn, _ := tokenResp["expires_in"].(float64) - - if accessToken == "" { - return r.vm.ToValue(map[string]any{ - "success": false, - "error": "no access_token in response", - "body": bodyPreview, - }) - } - - extensionAuthStateMu.Lock() - state, exists = extensionAuthState[r.extensionID] - if !exists { - state = &ExtensionAuthState{} - extensionAuthState[r.extensionID] = state - } - state.AccessToken = accessToken - state.RefreshToken = refreshToken - state.IsAuthenticated = true - if expiresIn > 0 { - state.ExpiresAt = time.Now().Add(time.Duration(expiresIn) * time.Second) - } - state.PKCEVerifier = "" - state.PKCEChallenge = "" - extensionAuthStateMu.Unlock() - - GoLog("[Extension:%s] PKCE token exchange successful\n", r.extensionID) - - result := map[string]any{ - "success": true, - "access_token": accessToken, - "refresh_token": refreshToken, - "token_type": tokenResp["token_type"], - } - if expiresIn > 0 { - result["expires_in"] = expiresIn - } - if scope, ok := tokenResp["scope"].(string); ok { - result["scope"] = scope - } - - return r.vm.ToValue(result) -} diff --git a/go_backend/extension_runtime_binary.go b/go_backend/extension_runtime_binary.go deleted file mode 100644 index 908be6dd..00000000 --- a/go_backend/extension_runtime_binary.go +++ /dev/null @@ -1,503 +0,0 @@ -package gobackend - -import ( - "crypto/aes" - "crypto/cipher" - "encoding/base64" - "encoding/hex" - "fmt" - "strings" - - "github.com/dop251/goja" - //lint:ignore SA1019 Blowfish is required for legacy extension crypto compatibility. - "golang.org/x/crypto/blowfish" -) - -type runtimeBlockCipherOptions struct { - Algorithm string - Mode string - Key []byte - IV []byte - InputEncoding string - OutputEncoding string - Padding string -} - -func parseRuntimeOptionsArgument(call goja.FunctionCall, index int) map[string]any { - if len(call.Arguments) <= index { - return nil - } - - value := call.Arguments[index] - if goja.IsUndefined(value) || goja.IsNull(value) { - return nil - } - - exported := value.Export() - if options, ok := exported.(map[string]any); ok { - return options - } - return nil -} - -func runtimeOptionString(options map[string]any, key, defaultValue string) string { - if options == nil { - return defaultValue - } - raw, ok := options[key] - if !ok || raw == nil { - return defaultValue - } - switch value := raw.(type) { - case string: - if trimmed := strings.TrimSpace(value); trimmed != "" { - return trimmed - } - case []byte: - if len(value) > 0 { - return string(value) - } - } - return defaultValue -} - -func runtimeOptionBool(options map[string]any, key string, defaultValue bool) bool { - if options == nil { - return defaultValue - } - raw, ok := options[key] - if !ok || raw == nil { - return defaultValue - } - switch value := raw.(type) { - case bool: - return value - case int: - return value != 0 - case int64: - return value != 0 - case float64: - return value != 0 - case string: - switch strings.ToLower(strings.TrimSpace(value)) { - case "1", "true", "yes", "on": - return true - case "0", "false", "no", "off": - return false - } - } - return defaultValue -} - -func runtimeOptionInt64(options map[string]any, key string, defaultValue int64) int64 { - if options == nil { - return defaultValue - } - raw, ok := options[key] - if !ok || raw == nil { - return defaultValue - } - switch value := raw.(type) { - case int: - return int64(value) - case int32: - return int64(value) - case int64: - return value - case float32: - return int64(value) - case float64: - return int64(value) - case string: - value = strings.TrimSpace(value) - if value == "" { - return defaultValue - } - var parsed int64 - if _, err := fmt.Sscanf(value, "%d", &parsed); err == nil { - return parsed - } - } - return defaultValue -} - -func runtimeOptionHasKey(options map[string]any, key string) bool { - if options == nil { - return false - } - _, exists := options[key] - return exists -} - -func decodeRuntimeBytesString(input, encoding string) ([]byte, error) { - switch strings.ToLower(strings.TrimSpace(encoding)) { - case "", "utf8", "utf-8", "text": - return []byte(input), nil - case "base64": - decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(input)) - if err != nil { - return nil, fmt.Errorf("invalid base64 data: %w", err) - } - return decoded, nil - case "hex": - decoded, err := hex.DecodeString(strings.TrimSpace(input)) - if err != nil { - return nil, fmt.Errorf("invalid hex data: %w", err) - } - return decoded, nil - default: - return nil, fmt.Errorf("unsupported byte encoding: %s", encoding) - } -} - -func decodeRuntimeBytesValue(raw any, encoding string) ([]byte, error) { - switch value := raw.(type) { - case string: - return decodeRuntimeBytesString(value, encoding) - case []byte: - cloned := make([]byte, len(value)) - copy(cloned, value) - return cloned, nil - case goja.ArrayBuffer: - src := value.Bytes() - cloned := make([]byte, len(src)) - copy(cloned, src) - return cloned, nil - case []any: - decoded := make([]byte, len(value)) - for i, item := range value { - switch num := item.(type) { - case int: - decoded[i] = byte(num) - case int64: - decoded[i] = byte(num) - case float64: - decoded[i] = byte(int(num)) - default: - return nil, fmt.Errorf("unsupported byte array item at index %d", i) - } - } - return decoded, nil - default: - return nil, fmt.Errorf("unsupported byte payload type") - } -} - -func encodeRuntimeBytes(data []byte, encoding string) (string, error) { - switch strings.ToLower(strings.TrimSpace(encoding)) { - case "", "base64": - return base64.StdEncoding.EncodeToString(data), nil - case "hex": - return hex.EncodeToString(data), nil - case "utf8", "utf-8", "text": - return string(data), nil - default: - return "", fmt.Errorf("unsupported byte encoding: %s", encoding) - } -} - -func parseRuntimeBlockCipherOptions(options map[string]any) (*runtimeBlockCipherOptions, error) { - parsed := &runtimeBlockCipherOptions{ - Algorithm: strings.ToLower(runtimeOptionString(options, "algorithm", "")), - Mode: strings.ToLower(runtimeOptionString(options, "mode", "cbc")), - InputEncoding: strings.ToLower(runtimeOptionString(options, "inputEncoding", "base64")), - OutputEncoding: strings.ToLower(runtimeOptionString(options, "outputEncoding", "base64")), - Padding: strings.ToLower(runtimeOptionString(options, "padding", "none")), - } - if parsed.Algorithm == "" { - return nil, fmt.Errorf("algorithm is required") - } - if parsed.Mode == "" { - return nil, fmt.Errorf("mode is required") - } - - key, err := decodeRuntimeBytesString(runtimeOptionString(options, "key", ""), runtimeOptionString(options, "keyEncoding", "utf8")) - if err != nil { - return nil, fmt.Errorf("invalid key: %w", err) - } - if len(key) == 0 { - return nil, fmt.Errorf("key is required") - } - parsed.Key = key - - iv, err := decodeRuntimeBytesString(runtimeOptionString(options, "iv", ""), runtimeOptionString(options, "ivEncoding", "utf8")) - if err != nil { - return nil, fmt.Errorf("invalid iv: %w", err) - } - parsed.IV = iv - return parsed, nil -} - -func newRuntimeBlockCipher(options *runtimeBlockCipherOptions) (cipher.Block, error) { - switch options.Algorithm { - case "blowfish": - return blowfish.NewCipher(options.Key) - case "aes": - return aes.NewCipher(options.Key) - default: - return nil, fmt.Errorf("unsupported block cipher algorithm: %s", options.Algorithm) - } -} - -func applyPKCS7Padding(data []byte, blockSize int) []byte { - padding := blockSize - (len(data) % blockSize) - if padding == 0 { - padding = blockSize - } - out := make([]byte, len(data)+padding) - copy(out, data) - for i := len(data); i < len(out); i++ { - out[i] = byte(padding) - } - return out -} - -func removePKCS7Padding(data []byte, blockSize int) ([]byte, error) { - if len(data) == 0 || len(data)%blockSize != 0 { - return nil, fmt.Errorf("invalid padded payload length") - } - padding := int(data[len(data)-1]) - if padding <= 0 || padding > blockSize || padding > len(data) { - return nil, fmt.Errorf("invalid PKCS7 padding") - } - for i := len(data) - padding; i < len(data); i++ { - if int(data[i]) != padding { - return nil, fmt.Errorf("invalid PKCS7 padding") - } - } - return data[:len(data)-padding], nil -} - -func (r *extensionRuntime) transformBlockCipher(call goja.FunctionCall, decrypt bool) goja.Value { - if len(call.Arguments) < 2 { - return r.jsError("data and options are required") - } - - options := parseRuntimeOptionsArgument(call, 1) - parsedOptions, err := parseRuntimeBlockCipherOptions(options) - if err != nil { - return r.jsError("%s", err.Error()) - } - switch parsedOptions.Mode { - case "cbc", "ctr": - default: - return r.jsError("unsupported block cipher mode: %s", parsedOptions.Mode) - } - - inputData, err := decodeRuntimeBytesValue(call.Arguments[0].Export(), parsedOptions.InputEncoding) - if err != nil { - return r.jsError("%s", err.Error()) - } - - block, err := newRuntimeBlockCipher(parsedOptions) - if err != nil { - return r.jsError("%s", err.Error()) - } - - if len(parsedOptions.IV) != block.BlockSize() { - ivLabel := "iv" - if parsedOptions.Mode == "ctr" { - ivLabel = "iv (counter)" - } - return r.jsError("%s must be %d bytes for %s", ivLabel, block.BlockSize(), parsedOptions.Algorithm) - } - - var output []byte - if parsedOptions.Mode == "ctr" { - // CTR is a stream mode: encryption and decryption are identical, - // require no padding, and accept arbitrary input lengths. - output = make([]byte, len(inputData)) - cipher.NewCTR(block, parsedOptions.IV).XORKeyStream(output, inputData) - } else { - data := inputData - if !decrypt && parsedOptions.Padding == "pkcs7" { - data = applyPKCS7Padding(data, block.BlockSize()) - } - if len(data)%block.BlockSize() != 0 { - return r.jsError("input length must be a multiple of %d bytes", block.BlockSize()) - } - - output = make([]byte, len(data)) - if decrypt { - cipher.NewCBCDecrypter(block, parsedOptions.IV).CryptBlocks(output, data) - if parsedOptions.Padding == "pkcs7" { - output, err = removePKCS7Padding(output, block.BlockSize()) - if err != nil { - return r.jsError("%s", err.Error()) - } - } - } else { - cipher.NewCBCEncrypter(block, parsedOptions.IV).CryptBlocks(output, data) - } - } - - encoded, err := encodeRuntimeBytes(output, parsedOptions.OutputEncoding) - if err != nil { - return r.jsError("%s", err.Error()) - } - - return r.jsSuccess(map[string]any{ - "data": encoded, - "block_size": block.BlockSize(), - }) -} - -func (r *extensionRuntime) encryptBlockCipher(call goja.FunctionCall) goja.Value { - return r.transformBlockCipher(call, false) -} - -func (r *extensionRuntime) decryptBlockCipher(call goja.FunctionCall) goja.Value { - return r.transformBlockCipher(call, true) -} - -// decryptCTRSegments decrypts many independently-IV'd AES-CTR segments inside a -// single buffer in one host call. This exists to avoid thousands of JS->Go -// bridge crossings when an extension decrypts per-sample CENC media (each -// sample has its own IV/counter and cannot be merged into one stream). -// -// It is a generic primitive: any extension can use it for "one buffer, many -// CTR segments" workloads, not just Apple CENC. -// -// For best performance, pass the buffer as an ArrayBuffer/Uint8Array and set -// outputEncoding:"bytes" to get an ArrayBuffer back. This avoids base64 -// encode/decode of the (potentially multi-MB) payload entirely, which is the -// dominant cost under the goja interpreter. -// -// JS signature: -// -// utils.decryptCTRSegments(data, { -// algorithm: "aes", // optional, default "aes" -// key: "<hex>", keyEncoding: "hex", -// segments: [ { offset: <int>, size: <int>, iv: "<base64>" }, ... ], -// ivEncoding: "base64", // encoding of each segment.iv, default base64 -// inputEncoding: "bytes", // "bytes" for ArrayBuffer/Uint8Array, else base64/hex -// outputEncoding: "bytes" // "bytes" -> ArrayBuffer; else base64/hex string -// }) -// -// Returns { success, data, segments_processed } or { success:false, error }. -func (r *extensionRuntime) decryptCTRSegments(call goja.FunctionCall) goja.Value { - fail := func(msg string) goja.Value { - return r.jsError("%s", msg) - } - - if len(call.Arguments) < 2 { - return fail("data and options are required") - } - - options := parseRuntimeOptionsArgument(call, 1) - if options == nil { - return fail("options object is required") - } - - algorithm := strings.ToLower(runtimeOptionString(options, "algorithm", "aes")) - inputEncoding := strings.ToLower(runtimeOptionString(options, "inputEncoding", "base64")) - outputEncoding := strings.ToLower(runtimeOptionString(options, "outputEncoding", "base64")) - ivEncoding := strings.ToLower(runtimeOptionString(options, "ivEncoding", "base64")) - - key, err := decodeRuntimeBytesString( - runtimeOptionString(options, "key", ""), - runtimeOptionString(options, "keyEncoding", "hex"), - ) - if err != nil { - return fail(fmt.Sprintf("invalid key: %v", err)) - } - if len(key) == 0 { - return fail("key is required") - } - - var block cipher.Block - switch algorithm { - case "aes": - block, err = aes.NewCipher(key) - case "blowfish": - block, err = blowfish.NewCipher(key) - default: - return fail("unsupported algorithm: " + algorithm) - } - if err != nil { - return fail(err.Error()) - } - blockSize := block.BlockSize() - - // Decode the payload. For "bytes" input we operate on the raw []byte - // (ArrayBuffer/Uint8Array) without any base64 round-trip. - var data []byte - if inputEncoding == "bytes" || inputEncoding == "raw" { - data, err = decodeRuntimeBytesValue(call.Arguments[0].Export(), "") - if err != nil { - return fail("invalid byte payload: " + err.Error()) - } - } else { - data, err = decodeRuntimeBytesValue(call.Arguments[0].Export(), inputEncoding) - if err != nil { - return fail(err.Error()) - } - } - - rawSegments, ok := options["segments"] - if !ok || rawSegments == nil { - return fail("segments array is required") - } - segments, ok := rawSegments.([]any) - if !ok { - return fail("segments must be an array") - } - - processed := 0 - for i, rawSeg := range segments { - seg, ok := rawSeg.(map[string]any) - if !ok { - return fail(fmt.Sprintf("segment %d is not an object", i)) - } - - offset := int(runtimeOptionInt64(seg, "offset", -1)) - size := int(runtimeOptionInt64(seg, "size", -1)) - if offset < 0 || size < 0 { - return fail(fmt.Sprintf("segment %d has invalid offset/size", i)) - } - if size == 0 { - continue - } - if offset+size > len(data) { - return fail(fmt.Sprintf("segment %d out of bounds (offset=%d size=%d len=%d)", i, offset, size, len(data))) - } - - iv, err := decodeRuntimeBytesString(runtimeOptionString(seg, "iv", ""), ivEncoding) - if err != nil { - return fail(fmt.Sprintf("segment %d has invalid iv: %v", i, err)) - } - if len(iv) != blockSize { - // Accept short IVs by left-aligning into a block-sized counter - // (CENC commonly uses 8-byte IVs for a 16-byte AES counter). - if len(iv) > blockSize { - return fail(fmt.Sprintf("segment %d iv longer than block size (%d > %d)", i, len(iv), blockSize)) - } - padded := make([]byte, blockSize) - copy(padded, iv) - iv = padded - } - - segData := data[offset : offset+size] - cipher.NewCTR(block, iv).XORKeyStream(segData, segData) - processed++ - } - - // Return raw bytes as an ArrayBuffer when requested (zero-copy-ish, no - // base64). Otherwise fall back to an encoded string. - if outputEncoding == "bytes" || outputEncoding == "raw" { - return r.jsSuccess(map[string]any{ - "data": r.vm.NewArrayBuffer(data), - "segments_processed": processed, - }) - } - - encoded, err := encodeRuntimeBytes(data, outputEncoding) - if err != nil { - return fail(err.Error()) - } - - return r.jsSuccess(map[string]any{ - "data": encoded, - "segments_processed": processed, - }) -} diff --git a/go_backend/extension_runtime_binary_test.go b/go_backend/extension_runtime_binary_test.go deleted file mode 100644 index fd0e47cd..00000000 --- a/go_backend/extension_runtime_binary_test.go +++ /dev/null @@ -1,583 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "testing" - - "github.com/dop251/goja" -) - -func newBinaryTestRuntime(t *testing.T, withFilePermission bool) *goja.Runtime { - t.Helper() - - ext := &loadedExtension{ - ID: "binary-test-ext", - Manifest: &ExtensionManifest{ - Name: "binary-test-ext", - Permissions: ExtensionPermissions{ - File: withFilePermission, - }, - }, - DataDir: t.TempDir(), - } - - runtime := newExtensionRuntime(ext) - vm := goja.New() - runtime.RegisterAPIs(vm) - return vm -} - -func decodeJSONResult[T any](t *testing.T, value goja.Value) T { - t.Helper() - - var decoded T - if err := json.Unmarshal([]byte(value.String()), &decoded); err != nil { - t.Fatalf("failed to decode JSON result: %v", err) - } - return decoded -} - -func TestExtensionRuntime_FileByteAPIs(t *testing.T) { - vm := newBinaryTestRuntime(t, true) - - result, err := vm.RunString(` - (function() { - var first = file.writeBytes("bytes.bin", "AAEC", {encoding: "base64", truncate: true}); - if (!first.success) throw new Error(first.error); - - var second = file.writeBytes("bytes.bin", "0304ff", {encoding: "hex", append: true}); - if (!second.success) throw new Error(second.error); - - var all = file.readBytes("bytes.bin", {encoding: "hex"}); - if (!all.success) throw new Error(all.error); - - var slice = file.readBytes("bytes.bin", {offset: 2, length: 2, encoding: "hex"}); - if (!slice.success) throw new Error(slice.error); - - var tail = file.readBytes("bytes.bin", {offset: 6, length: 4, encoding: "hex"}); - if (!tail.success) throw new Error(tail.error); - - return JSON.stringify({ - all: all.data, - slice: slice.data, - size: all.size, - sliceBytes: slice.bytes_read, - sliceEof: slice.eof, - tailBytes: tail.bytes_read, - tailEof: tail.eof - }); - })() - `) - if err != nil { - t.Fatalf("file byte APIs failed: %v", err) - } - - decoded := decodeJSONResult[struct { - All string `json:"all"` - Slice string `json:"slice"` - Size int64 `json:"size"` - SliceBytes int `json:"sliceBytes"` - SliceEof bool `json:"sliceEof"` - TailBytes int `json:"tailBytes"` - TailEof bool `json:"tailEof"` - }](t, result) - - if decoded.All != "0001020304ff" { - t.Fatalf("all = %q", decoded.All) - } - if decoded.Slice != "0203" { - t.Fatalf("slice = %q", decoded.Slice) - } - if decoded.Size != 6 { - t.Fatalf("size = %d", decoded.Size) - } - if decoded.SliceBytes != 2 { - t.Fatalf("slice bytes = %d", decoded.SliceBytes) - } - if decoded.SliceEof { - t.Fatal("slice should not be EOF") - } - if decoded.TailBytes != 0 || !decoded.TailEof { - t.Fatalf("tail read mismatch: bytes=%d eof=%v", decoded.TailBytes, decoded.TailEof) - } -} - -func TestExtensionRuntime_BlockCipherCBCSupportsBlowfish(t *testing.T) { - vm := newBinaryTestRuntime(t, false) - - result, err := vm.RunString(` - (function() { - var options = { - algorithm: "blowfish", - mode: "cbc", - key: "0123456789ABCDEFF0E1D2C3B4A59687", - keyEncoding: "hex", - iv: "0001020304050607", - ivEncoding: "hex", - inputEncoding: "hex", - outputEncoding: "hex", - padding: "none" - }; - var enc = utils.encryptBlockCipher("00112233445566778899aabbccddeeff", options); - if (!enc.success) throw new Error(enc.error); - var dec = utils.decryptBlockCipher(enc.data, options); - if (!dec.success) throw new Error(dec.error); - return JSON.stringify({enc: enc.data, dec: dec.data}); - })() - `) - if err != nil { - t.Fatalf("blowfish block cipher failed: %v", err) - } - - decoded := decodeJSONResult[struct { - Enc string `json:"enc"` - Dec string `json:"dec"` - }](t, result) - - if decoded.Dec != "00112233445566778899aabbccddeeff" { - t.Fatalf("dec = %q", decoded.Dec) - } - if decoded.Enc == decoded.Dec { - t.Fatal("expected ciphertext to differ from plaintext") - } -} - -func TestExtensionRuntime_BlockCipherCBCSupportsAES(t *testing.T) { - vm := newBinaryTestRuntime(t, false) - - result, err := vm.RunString(` - (function() { - var options = { - algorithm: "aes", - mode: "cbc", - key: "000102030405060708090a0b0c0d0e0f", - keyEncoding: "hex", - iv: "0f0e0d0c0b0a09080706050403020100", - ivEncoding: "hex", - inputEncoding: "utf8", - outputEncoding: "base64", - padding: "pkcs7" - }; - var enc = utils.encryptBlockCipher("hello generic cbc", options); - if (!enc.success) throw new Error(enc.error); - var dec = utils.decryptBlockCipher(enc.data, { - algorithm: "aes", - mode: "cbc", - key: options.key, - keyEncoding: options.keyEncoding, - iv: options.iv, - ivEncoding: options.ivEncoding, - inputEncoding: "base64", - outputEncoding: "utf8", - padding: "pkcs7" - }); - if (!dec.success) throw new Error(dec.error); - return dec.data; - })() - `) - if err != nil { - t.Fatalf("aes block cipher failed: %v", err) - } - - if result.String() != "hello generic cbc" { - t.Fatalf("unexpected decrypted value: %q", result.String()) - } -} - -func TestExtensionRuntime_FileTransformPatternedBlocks(t *testing.T) { - vm := newBinaryTestRuntime(t, true) - - result, err := vm.RunString(` - (function() { - var options = { - algorithm: "blowfish", - mode: "cbc", - key: "0123456789ABCDEFF0E1D2C3B4A59687", - keyEncoding: "hex", - iv: "0001020304050607", - ivEncoding: "hex", - inputEncoding: "hex", - outputEncoding: "hex", - padding: "none" - }; - var plainSegments = [ - "00112233445566778899aabbccddeeff", - "102132435465768798a9bacbdcedfe0f", - "2031425364758697a8b9cadbecfd0e1f", - "30415263748596a7b8c9daebfc0d1e2f" - ]; - var encrypted = ""; - for (var i = 0; i < plainSegments.length; i++) { - if (i % 3 === 0) { - var enc = utils.encryptBlockCipher(plainSegments[i], options); - if (!enc.success) throw new Error(enc.error); - encrypted += enc.data; - } else { - encrypted += plainSegments[i]; - } - } - var partialTail = "a1b2c3d4e5"; - encrypted += partialTail; - var write = file.writeBytes("encrypted.bin", encrypted, { - encoding: "hex", truncate: true - }); - if (!write.success) throw new Error(write.error); - - var callbacks = 0; - var transformed = file.transformPatternedBlocks( - "encrypted.bin", - "decrypted.bin", - { - operation: "decrypt", - algorithm: "blowfish", - mode: "cbc", - key: options.key, - keyEncoding: "hex", - iv: options.iv, - ivEncoding: "hex", - padding: "none", - segmentSize: 16, - transformEvery: 3, - transformOffset: 0, - bufferSize: 32 - }, - function(processed, total) { - if (processed > total) throw new Error("invalid progress"); - callbacks++; - } - ); - if (!transformed.success) throw new Error(transformed.error); - var output = file.readBytes("decrypted.bin", {encoding: "hex"}); - if (!output.success) throw new Error(output.error); - return JSON.stringify({ - output: output.data, - expected: plainSegments.join("") + partialTail, - processed: transformed.bytes_processed, - segments: transformed.segments_processed, - transformedSegments: transformed.segments_transformed, - callbacks: callbacks - }); - })() - `) - if err != nil { - t.Fatalf("patterned file transform failed: %v", err) - } - - decoded := decodeJSONResult[struct { - Output string `json:"output"` - Expected string `json:"expected"` - Processed int64 `json:"processed"` - Segments int64 `json:"segments"` - TransformedSegments int64 `json:"transformedSegments"` - Callbacks int `json:"callbacks"` - }](t, result) - if decoded.Output != decoded.Expected { - t.Fatalf("output = %q, want %q", decoded.Output, decoded.Expected) - } - if decoded.Processed != 69 || decoded.Segments != 5 || decoded.TransformedSegments != 2 { - t.Fatalf("unexpected transform stats: %+v", decoded) - } - if decoded.Callbacks != 3 { - t.Fatalf("callbacks = %d, want 3 buffered updates", decoded.Callbacks) - } -} - -func TestExtensionRuntime_BlockCipherCTRSupportsAES(t *testing.T) { - vm := newBinaryTestRuntime(t, false) - - // NIST SP 800-38A, F.5.1 CTR-AES128.Encrypt test vector. - // Key: 2b7e151628aed2a6abf7158809cf4f3c - // Counter: f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff - // Plaintext: 6bc1bee22e409f96e93d7e117393172a (block 1) - // Ciphertext: 874d6191b620e3261bef6864990db6ce (block 1) - result, err := vm.RunString(` - (function() { - var options = { - algorithm: "aes", - mode: "ctr", - key: "2b7e151628aed2a6abf7158809cf4f3c", - keyEncoding: "hex", - iv: "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff", - ivEncoding: "hex", - inputEncoding: "hex", - outputEncoding: "hex" - }; - var enc = utils.encryptBlockCipher("6bc1bee22e409f96e93d7e117393172a", options); - if (!enc.success) throw new Error(enc.error); - // CTR is symmetric: decrypt is the same transform as encrypt. - var dec = utils.decryptBlockCipher(enc.data, options); - if (!dec.success) throw new Error(dec.error); - return JSON.stringify({enc: enc.data, dec: dec.data}); - })() - `) - if err != nil { - t.Fatalf("aes ctr block cipher failed: %v", err) - } - - decoded := decodeJSONResult[struct { - Enc string `json:"enc"` - Dec string `json:"dec"` - }](t, result) - - if decoded.Enc != "874d6191b620e3261bef6864990db6ce" { - t.Fatalf("ctr ciphertext = %q, want NIST vector 874d6191b620e3261bef6864990db6ce", decoded.Enc) - } - if decoded.Dec != "6bc1bee22e409f96e93d7e117393172a" { - t.Fatalf("ctr round-trip dec = %q", decoded.Dec) - } -} - -func TestExtensionRuntime_BlockCipherCTRHandlesNonBlockLength(t *testing.T) { - vm := newBinaryTestRuntime(t, false) - - // CTR is a stream mode, so arbitrary (non-16-byte-aligned) input lengths - // must round-trip without any padding. - result, err := vm.RunString(` - (function() { - var options = { - algorithm: "aes", - mode: "ctr", - key: "000102030405060708090a0b0c0d0e0f", - keyEncoding: "hex", - iv: "0f0e0d0c0b0a09080706050403020100", - ivEncoding: "hex", - inputEncoding: "utf8", - outputEncoding: "base64" - }; - var enc = utils.encryptBlockCipher("stream ctr of odd length", options); - if (!enc.success) throw new Error(enc.error); - var dec = utils.decryptBlockCipher(enc.data, { - algorithm: "aes", - mode: "ctr", - key: options.key, - keyEncoding: options.keyEncoding, - iv: options.iv, - ivEncoding: options.ivEncoding, - inputEncoding: "base64", - outputEncoding: "utf8" - }); - if (!dec.success) throw new Error(dec.error); - return dec.data; - })() - `) - if err != nil { - t.Fatalf("aes ctr stream length failed: %v", err) - } - - if result.String() != "stream ctr of odd length" { - t.Fatalf("unexpected ctr decrypted value: %q", result.String()) - } -} - -func TestExtensionRuntime_BlockCipherCTRRejectsBadIV(t *testing.T) { - vm := newBinaryTestRuntime(t, false) - - result, err := vm.RunString(` - (function() { - var res = utils.encryptBlockCipher("00112233", { - algorithm: "aes", - mode: "ctr", - key: "000102030405060708090a0b0c0d0e0f", - keyEncoding: "hex", - iv: "0001", - ivEncoding: "hex", - inputEncoding: "hex", - outputEncoding: "hex" - }); - return JSON.stringify({success: res.success, error: res.error || ""}); - })() - `) - if err != nil { - t.Fatalf("aes ctr bad iv eval failed: %v", err) - } - - decoded := decodeJSONResult[struct { - Success bool `json:"success"` - Error string `json:"error"` - }](t, result) - - if decoded.Success { - t.Fatal("expected failure for undersized CTR iv") - } - if decoded.Error == "" { - t.Fatal("expected error message for undersized CTR iv") - } -} - -func TestExtensionRuntime_DecryptCTRSegmentsMatchesPerSegment(t *testing.T) { - vm := newBinaryTestRuntime(t, false) - - // Build a buffer of 3 segments encrypted with distinct 8-byte IVs (CENC - // style), then verify the batch primitive decrypts all of them in one call, - // matching what per-segment decryptBlockCipher would produce. - result, err := vm.RunString(` - (function() { - var keyHex = "000102030405060708090a0b0c0d0e0f"; - function b64(bytes){return utils.base64Encode(utils.toHex ? bytes : bytes);} - - // segment plaintexts (hex) and 8-byte IVs (hex) - var segs = [ - { pt: "11111111111111111111", iv: "0000000000000001" }, - { pt: "2222222222", iv: "0000000000000002" }, - { pt: "333333333333333333333333", iv: "00000000000000ff" } - ]; - - // Encrypt each segment individually using single-shot CTR with a - // 16-byte counter (8-byte iv left-aligned), producing ciphertext hex. - function ivToB64(ivHex){ - // pad 8-byte hex iv to 16 bytes then base64 - var full = ivHex + "00000000000000000000000000000000".slice(ivHex.length); - return utils.base64Encode(utils.hexToBytes ? utils.hexToBytes(full) : full); - } - - var cipherHex = ""; - var offsets = []; - var off = 0; - var ivB64s = []; - for (var i=0;i<segs.length;i++){ - var ivFullHex = (segs[i].iv + "00000000000000000000000000000000").slice(0,32); - var enc = utils.encryptBlockCipher(segs[i].pt, { - algorithm:"aes", mode:"ctr", key:keyHex, keyEncoding:"hex", - iv: ivFullHex, ivEncoding:"hex", - inputEncoding:"hex", outputEncoding:"hex" - }); - if(!enc.success) throw new Error("enc seg "+i+": "+enc.error); - cipherHex += enc.data; - var sz = segs[i].pt.length/2; - offsets.push({offset: off, size: sz, ivHex: ivFullHex}); - off += sz; - } - - // Now decrypt the whole concatenated buffer in ONE batch call. - var segments = offsets.map(function(o){ - return { offset:o.offset, size:o.size, iv:o.ivHex }; - }); - var batch = utils.decryptCTRSegments(cipherHex, { - algorithm:"aes", key:keyHex, keyEncoding:"hex", - segments: segments, ivEncoding:"hex", - inputEncoding:"hex", outputEncoding:"hex" - }); - if(!batch.success) throw new Error("batch: "+batch.error); - - var expected = ""; - for (var j=0;j<segs.length;j++) expected += segs[j].pt; - - return JSON.stringify({ - out: batch.data, - expected: expected, - processed: batch.segments_processed - }); - })() - `) - if err != nil { - t.Fatalf("batch CTR eval failed: %v", err) - } - - decoded := decodeJSONResult[struct { - Out string `json:"out"` - Expected string `json:"expected"` - Processed int `json:"processed"` - }](t, result) - - if decoded.Out != decoded.Expected { - t.Fatalf("batch decrypt mismatch:\n got=%s\nwant=%s", decoded.Out, decoded.Expected) - } - if decoded.Processed != 3 { - t.Fatalf("segments_processed = %d, want 3", decoded.Processed) - } -} - -func TestExtensionRuntime_DecryptCTRSegmentsRejectsOutOfBounds(t *testing.T) { - vm := newBinaryTestRuntime(t, false) - - result, err := vm.RunString(` - (function() { - var res = utils.decryptCTRSegments("00112233", { - algorithm:"aes", key:"000102030405060708090a0b0c0d0e0f", keyEncoding:"hex", - inputEncoding:"hex", outputEncoding:"hex", - ivEncoding:"hex", - segments: [ { offset: 0, size: 99, iv: "00000000000000000000000000000000" } ] - }); - return JSON.stringify({ success: res.success, error: res.error || "" }); - })() - `) - if err != nil { - t.Fatalf("oob eval failed: %v", err) - } - - decoded := decodeJSONResult[struct { - Success bool `json:"success"` - Error string `json:"error"` - }](t, result) - - if decoded.Success { - t.Fatal("expected out-of-bounds segment to fail") - } - if decoded.Error == "" { - t.Fatal("expected error message for out-of-bounds segment") - } -} - -func TestExtensionRuntime_DecryptCTRSegmentsRawBytes(t *testing.T) { - vm := newBinaryTestRuntime(t, false) - - // Verify the zero-base64 path: pass an ArrayBuffer in, request bytes out, - // and confirm round-trip correctness against single-shot CTR. - result, err := vm.RunString(` - (function() { - var keyHex = "000102030405060708090a0b0c0d0e0f"; - var ivFullHex = "0000000000000001" + "00000000000000000000000000000000".slice(16); - - // Plaintext as a Uint8Array of 20 bytes. - var pt = new Uint8Array(20); - for (var i = 0; i < pt.length; i++) pt[i] = (i * 7 + 3) & 0xff; - - // Encrypt single-shot to get ciphertext (hex output for clarity). - var ptHex = ""; - for (var j = 0; j < pt.length; j++) { var h = pt[j].toString(16); ptHex += (h.length === 1 ? "0" : "") + h; } - var enc = utils.encryptBlockCipher(ptHex, { - algorithm:"aes", mode:"ctr", key:keyHex, keyEncoding:"hex", - iv: ivFullHex, ivEncoding:"hex", inputEncoding:"hex", outputEncoding:"base64" - }); - if (!enc.success) throw new Error("enc: " + enc.error); - - // Decode ciphertext base64 into a Uint8Array to feed the raw path. - var cipherBytes = utils.base64Decode ? null : null; - // Build ArrayBuffer from base64 via Uint8Array manually: - var b64 = enc.data; - var bin = (typeof atob === "function") ? null : null; - - // Simpler: ask the host to give us bytes by decrypting nothing is hard, - // so just pass the base64 ciphertext through decryptCTRSegments using - // base64 input but bytes output, then re-run with bytes input. - var step1 = utils.decryptCTRSegments(b64, { - algorithm:"aes", key:keyHex, keyEncoding:"hex", - segments: [ { offset:0, size:20, iv: ivFullHex } ], - ivEncoding:"hex", inputEncoding:"base64", outputEncoding:"bytes" - }); - if (!step1.success) throw new Error("step1: " + step1.error); - if (typeof step1.data === "string") throw new Error("expected ArrayBuffer output, got string"); - - var outArr = new Uint8Array(step1.data); - var outHex = ""; - for (var k = 0; k < outArr.length; k++) { var hh = outArr[k].toString(16); outHex += (hh.length === 1 ? "0" : "") + hh; } - return JSON.stringify({ out: outHex, expected: ptHex, len: outArr.length }); - })() - `) - if err != nil { - t.Fatalf("raw-bytes eval failed: %v", err) - } - - decoded := decodeJSONResult[struct { - Out string `json:"out"` - Expected string `json:"expected"` - Len int `json:"len"` - }](t, result) - - if decoded.Out != decoded.Expected { - t.Fatalf("raw-bytes decrypt mismatch:\n got=%s\nwant=%s", decoded.Out, decoded.Expected) - } - if decoded.Len != 20 { - t.Fatalf("output length = %d, want 20", decoded.Len) - } -} diff --git a/go_backend/extension_runtime_chunked.go b/go_backend/extension_runtime_chunked.go deleted file mode 100644 index e83f86d1..00000000 --- a/go_backend/extension_runtime_chunked.go +++ /dev/null @@ -1,586 +0,0 @@ -package gobackend - -import ( - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/dop251/goja" -) - -func chunkedTransferTotal(resp *http.Response) int64 { - if resp == nil { - return 0 - } - if contentRange := resp.Header.Get("Content-Range"); contentRange != "" { - if slash := strings.LastIndex(contentRange, "/"); slash >= 0 { - var total int64 - if _, err := fmt.Sscanf(contentRange[slash+1:], "%d", &total); err == nil { - return total - } - } - } - if resp.StatusCode == http.StatusOK { - return resp.ContentLength - } - return 0 -} - -func (r *extensionRuntime) chunkedTransferCancelled(activeItemID string) bool { - return activeItemID != "" && isDownloadCancelled(activeItemID) -} - -// fileDownloadChunked downloads sequential byte ranges. The same transfer -// policy used by ordinary and segmented downloads controls retries and -// checkpoints here, so specialized CDN downloads do not lose the reliability -// guarantees of the generic file API. -func (r *extensionRuntime) fileDownloadChunked( - client *http.Client, - urlStr, fullPath string, - headers map[string]string, - userAgent string, - chunkSize int64, - onProgress goja.Callable, - trackItemBytes bool, - persistentCheckpoint bool, - policy DownloadTransferPolicy, -) goja.Value { - unlock := lockDownloadOutputPath(fullPath) - defer unlock() - - activeItemID := r.getActiveDownloadItemID() - if activeItemID != "" { - SetItemDownloading(activeItemID) - } - config := transferRetryConfig(policy) - var probeResp *http.Response - var probeFailure transferFailure - probeDelay := config.InitialDelay - probeAttempts := 0 - for attempt := 1; attempt <= policy.MaxAttempts; attempt++ { - probeAttempts = attempt - request, err := http.NewRequest("GET", urlStr, nil) - if err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "invalid_request", - Message: fmt.Sprintf("chunked probe request: %v", err), - Attempts: attempt, - }) - } - request = r.bindDownloadCancelContext(request) - retryContext := request.Context() - for key, value := range headers { - if !strings.EqualFold(key, "Range") { - request.Header.Set(key, value) - } - } - request.Header.Set("User-Agent", userAgent) - request.Header.Set("Range", "bytes=0-1") - request, watchdog := bindStallWatchdog(request, downloadStallTimeout) - response, requestErr := client.Do(request) - if requestErr != nil { - stalled := watchdog.stalled.Load() - watchdog.stop() - if r.chunkedTransferCancelled(activeItemID) { - return r.jsTransferError(transferFailure{ - ErrorType: "cancelled", - Message: "download cancelled", - Attempts: attempt, - }) - } - message := fmt.Sprintf("chunked probe failed: %v", requestErr) - if stalled { - message = fmt.Sprintf( - "chunked probe stalled for %ds", - int(downloadStallTimeout.Seconds()), - ) - } - probeFailure = transferFailure{ - ErrorType: "transient_network", - Message: message, - Attempts: attempt, - } - if attempt == policy.MaxAttempts { - return r.jsTransferError(probeFailure) - } - if waitTransferRetry(retryContext, probeDelay) != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "cancelled", - Message: "download cancelled", - Attempts: attempt, - }) - } - probeDelay = calculateNextDelay(probeDelay, config) - continue - } - watchdog.stop() - if response.StatusCode == http.StatusPartialContent || - response.StatusCode == http.StatusOK { - io.Copy(io.Discard, io.LimitReader(response.Body, 32*1024)) - response.Body.Close() - probeResp = response - break - } - - retryAfter := retryAfterSeconds(response) - io.Copy(io.Discard, io.LimitReader(response.Body, 32*1024)) - response.Body.Close() - probeFailure = transferFailure{ - ErrorType: transferErrorTypeForStatus(response.StatusCode, policy), - Message: fmt.Sprintf("chunked probe HTTP %d", response.StatusCode), - HTTPStatus: response.StatusCode, - RetryAfterSeconds: retryAfter, - Attempts: attempt, - } - if !retryableTransferStatus(response.StatusCode) || attempt == policy.MaxAttempts { - return r.jsTransferError(probeFailure) - } - delay := probeDelay - if retryAfter > 0 { - delay = time.Duration(retryAfter) * time.Second - } - if waitTransferRetry(retryContext, delay) != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "cancelled", - Message: "download cancelled", - Attempts: attempt, - }) - } - probeDelay = calculateNextDelay(probeDelay, config) - } - if probeResp == nil { - return r.jsTransferError(probeFailure) - } - - totalSize := chunkedTransferTotal(probeResp) - validator := transferResponseValidator(probeResp.Header) - fingerprint := transferURLFingerprint(urlStr) - stagedPath := stagedDownloadPath(fullPath) - checkpointPath := transferCheckpointPath(stagedPath) - keepPartial := persistentCheckpoint && validator != "" && fingerprint != "" - checkpoint, checkpointOK := loadTransferCheckpoint(checkpointPath, fingerprint) - if checkpointOK && (checkpoint.Validator != validator || - (checkpoint.Total > 0 && totalSize > 0 && checkpoint.Total != totalSize)) { - checkpointOK = false - } - if !keepPartial || !checkpointOK { - os.Remove(stagedPath) - os.Remove(checkpointPath) - checkpoint = transferCheckpoint{} - } - - output, err := os.OpenFile(stagedPath, os.O_CREATE|os.O_RDWR, 0600) - if err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to create chunked staged file: %v", err), - }) - } - promoted := false - defer func() { - output.Close() - if promoted { - os.Remove(checkpointPath) - } else if !keepPartial { - os.Remove(stagedPath) - os.Remove(checkpointPath) - } - }() - - var totalWritten int64 - if checkpointOK { - if info, statErr := output.Stat(); statErr == nil { - totalWritten = min(checkpoint.Bytes, info.Size()) - } - } - if err := output.Truncate(totalWritten); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to restore chunked partial: %v", err), - }) - } - if _, err := output.Seek(totalWritten, io.SeekStart); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to seek chunked partial: %v", err), - }) - } - - shouldTrackBytes := activeItemID != "" && trackItemBytes - itemProgressReporter := NewItemTransferProgressReporter(activeItemID, totalWritten, totalSize) - if shouldTrackBytes { - if totalSize > 0 { - SetItemProgress( - activeItemID, - float64(totalWritten)/float64(totalSize), - totalWritten, - totalSize, - ) - } else if totalWritten > 0 { - SetItemBytesReceived(activeItemID, totalWritten) - } - } - - lastProgressNotify := totalWritten - lastCheckpointBytes := totalWritten - lastCheckpointAt := time.Now() - attemptsUsed := probeAttempts - fullResponse := false - completedChunk := false - buffer := make([]byte, 64*1024) - for totalSize <= 0 || totalWritten < totalSize { - chunkStart := totalWritten - chunkEnd := chunkStart + chunkSize - 1 - if totalSize > 0 && chunkEnd >= totalSize { - chunkEnd = totalSize - 1 - } - retryDelay := config.InitialDelay - var chunkComplete bool - var lastFailure transferFailure - - for attempt := 1; attempt <= policy.MaxAttempts; attempt++ { - attemptsUsed++ - request, requestErr := http.NewRequest("GET", urlStr, nil) - if requestErr != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "invalid_request", - Message: fmt.Sprintf("chunked request at %d: %v", chunkStart, requestErr), - Attempts: attemptsUsed, - }) - } - request = r.bindDownloadCancelContext(request) - retryContext := request.Context() - for key, value := range headers { - if !strings.EqualFold(key, "Range") { - request.Header.Set(key, value) - } - } - request.Header.Set("User-Agent", userAgent) - request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", chunkStart, chunkEnd)) - if validator != "" { - request.Header.Set("If-Range", validator) - } - request, watchdog := bindStallWatchdog(request, downloadStallTimeout) - response, responseErr := r.doResolutionTransfer(client, request, attempt == 1 && completedChunk) - if responseErr != nil { - stalled := watchdog.stalled.Load() - watchdog.stop() - if r.chunkedTransferCancelled(activeItemID) { - lastFailure = transferFailure{ - ErrorType: "cancelled", - Message: "download cancelled", - Attempts: attemptsUsed, - } - return r.jsTransferError(lastFailure) - } - message := fmt.Sprintf("chunked request at %d failed: %v", chunkStart, responseErr) - if stalled { - message = fmt.Sprintf( - "chunked request at %d stalled for %ds", - chunkStart, - int(downloadStallTimeout.Seconds()), - ) - } - lastFailure = transferFailure{ - ErrorType: "transient_network", - Message: message, - Attempts: attemptsUsed, - } - if attempt == policy.MaxAttempts { - break - } - if waitTransferRetry(retryContext, retryDelay) != nil { - lastFailure.ErrorType = "cancelled" - lastFailure.Message = "download cancelled" - return r.jsTransferError(lastFailure) - } - retryDelay = calculateNextDelay(retryDelay, config) - continue - } - - if response.StatusCode != http.StatusPartialContent && - response.StatusCode != http.StatusOK { - retryAfter := retryAfterSeconds(response) - io.Copy(io.Discard, io.LimitReader(response.Body, 32*1024)) - response.Body.Close() - watchdog.stop() - lastFailure = transferFailure{ - ErrorType: transferErrorTypeForStatus(response.StatusCode, policy), - Message: fmt.Sprintf("chunked HTTP %d at offset %d", response.StatusCode, chunkStart), - HTTPStatus: response.StatusCode, - RetryAfterSeconds: retryAfter, - Attempts: attemptsUsed, - } - if !retryableTransferStatus(response.StatusCode) || attempt == policy.MaxAttempts { - break - } - delay := retryDelay - if retryAfter > 0 { - delay = time.Duration(retryAfter) * time.Second - } - if waitTransferRetry(retryContext, delay) != nil { - lastFailure.ErrorType = "cancelled" - lastFailure.Message = "download cancelled" - return r.jsTransferError(lastFailure) - } - retryDelay = calculateNextDelay(retryDelay, config) - continue - } - - if response.StatusCode == http.StatusPartialContent && - !strings.HasPrefix( - response.Header.Get("Content-Range"), - fmt.Sprintf("bytes %d-", chunkStart), - ) { - contentRange := response.Header.Get("Content-Range") - response.Body.Close() - watchdog.stop() - return r.jsTransferError(transferFailure{ - ErrorType: "integrity_failed", - Message: fmt.Sprintf( - "chunked response has unexpected Content-Range %q at %d", - contentRange, - chunkStart, - ), - HTTPStatus: response.StatusCode, - Attempts: attemptsUsed, - }) - } - if nextValidator := transferResponseValidator(response.Header); nextValidator != "" && - validator != "" && nextValidator != validator { - response.Body.Close() - watchdog.stop() - return r.jsTransferError(transferFailure{ - ErrorType: "integrity_failed", - Message: "chunked response validator changed during transfer", - Attempts: attemptsUsed, - }) - } - - if response.StatusCode == http.StatusOK && chunkStart > 0 { - if err := output.Truncate(0); err != nil { - response.Body.Close() - watchdog.stop() - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to restart ignored range: %v", err), - Attempts: attemptsUsed, - }) - } - if _, err := output.Seek(0, io.SeekStart); err != nil { - response.Body.Close() - watchdog.stop() - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to seek restarted range: %v", err), - Attempts: attemptsUsed, - }) - } - chunkStart = 0 - totalWritten = 0 - lastCheckpointBytes = 0 - os.Remove(checkpointPath) - if response.ContentLength > 0 { - totalSize = response.ContentLength - } - } - - chunkWritten := int64(0) - var readErr error - for { - readCount, bodyErr := response.Body.Read(buffer) - if readCount > 0 { - watchdog.reset() - if r.chunkedTransferCancelled(activeItemID) { - readErr = ErrDownloadCancelled - break - } - writeCount, writeErr := output.Write(buffer[:readCount]) - chunkWritten += int64(writeCount) - totalWritten += int64(writeCount) - if writeErr != nil || writeCount != readCount { - response.Body.Close() - watchdog.stop() - if writeErr == nil { - writeErr = io.ErrShortWrite - } - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to write chunked output: %v", writeErr), - Attempts: attemptsUsed, - }) - } - if shouldTrackBytes { - itemProgressReporter.Report(totalWritten, totalSize) - } - if onProgress != nil && totalSize > 0 && - (totalWritten-lastProgressNotify >= progressUpdateThreshold || totalWritten >= totalSize) { - lastProgressNotify = totalWritten - _, _ = onProgress( - goja.Undefined(), - r.vm.ToValue(totalWritten), - r.vm.ToValue(totalSize), - ) - } - } - if bodyErr != nil { - if bodyErr != io.EOF { - readErr = bodyErr - } - break - } - } - response.Body.Close() - stalled := watchdog.stalled.Load() - watchdog.stop() - expectedBytes := response.ContentLength - if response.StatusCode == http.StatusPartialContent && expectedBytes <= 0 { - expectedBytes = chunkEnd - chunkStart + 1 - } - if readErr == nil && expectedBytes > 0 && chunkWritten != expectedBytes { - readErr = io.ErrUnexpectedEOF - } - if readErr == nil && chunkWritten > 0 { - chunkComplete = true - completedChunk = true - fullResponse = response.StatusCode == http.StatusOK - break - } - - if err := output.Truncate(chunkStart); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to roll back incomplete chunk: %v", err), - Attempts: attemptsUsed, - }) - } - if _, err := output.Seek(chunkStart, io.SeekStart); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to seek rolled-back chunk: %v", err), - Attempts: attemptsUsed, - }) - } - totalWritten = chunkStart - if shouldTrackBytes && totalSize > 0 { - SetItemProgress( - activeItemID, - float64(totalWritten)/float64(totalSize), - totalWritten, - totalSize, - ) - } - message := fmt.Sprintf("failed to read chunk at %d: %v", chunkStart, readErr) - if chunkWritten == 0 && readErr == nil { - message = fmt.Sprintf("chunk at %d was empty", chunkStart) - } - if stalled { - message = fmt.Sprintf( - "chunk at %d stalled for %ds", - chunkStart, - int(downloadStallTimeout.Seconds()), - ) - } - lastFailure = transferFailure{ - ErrorType: "transient_network", - Message: message, - Attempts: attemptsUsed, - } - if attempt == policy.MaxAttempts || r.chunkedTransferCancelled(activeItemID) { - if r.chunkedTransferCancelled(activeItemID) { - lastFailure.ErrorType = "cancelled" - lastFailure.Message = "download cancelled" - } - break - } - if waitTransferRetry(retryContext, retryDelay) != nil { - lastFailure.ErrorType = "cancelled" - lastFailure.Message = "download cancelled" - return r.jsTransferError(lastFailure) - } - retryDelay = calculateNextDelay(retryDelay, config) - } - - if !chunkComplete { - if keepPartial && totalWritten > 0 { - _ = output.Sync() - _ = saveTransferCheckpoint(checkpointPath, transferCheckpoint{ - Fingerprint: fingerprint, - Validator: validator, - Bytes: totalWritten, - Total: totalSize, - }) - } - return r.jsTransferError(lastFailure) - } - if keepPartial && validator != "" && - (totalWritten-lastCheckpointBytes >= transferCheckpointBytes || - time.Since(lastCheckpointAt) >= transferCheckpointPeriod) { - if syncErr := output.Sync(); syncErr == nil { - if saveTransferCheckpoint(checkpointPath, transferCheckpoint{ - Fingerprint: fingerprint, - Validator: validator, - Bytes: totalWritten, - Total: totalSize, - }) == nil { - lastCheckpointBytes = totalWritten - lastCheckpointAt = time.Now() - } - } - } - if fullResponse { - break - } - if totalSize <= 0 && totalWritten-chunkStart < chunkSize { - break - } - } - - if totalWritten <= 0 || (totalSize > 0 && totalWritten != totalSize) { - return r.jsTransferError(transferFailure{ - ErrorType: "integrity_failed", - Message: fmt.Sprintf( - "chunked transfer size mismatch: expected %d bytes, wrote %d", - totalSize, - totalWritten, - ), - Attempts: attemptsUsed, - }) - } - if err := output.Sync(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to sync chunked output: %v", err), - Attempts: attemptsUsed, - }) - } - if err := output.Close(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to close chunked output: %v", err), - Attempts: attemptsUsed, - }) - } - if err := os.Rename(stagedPath, fullPath); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to publish chunked output: %v", err), - Attempts: attemptsUsed, - }) - } - promoted = true - os.Remove(checkpointPath) - syncDir(filepath.Dir(fullPath)) - if shouldTrackBytes { - SetItemProgress(activeItemID, 1, totalWritten, totalWritten) - } - return r.jsSuccess(map[string]any{ - "path": fullPath, - "size": totalWritten, - "attempts": attemptsUsed, - }) -} diff --git a/go_backend/extension_runtime_ffmpeg.go b/go_backend/extension_runtime_ffmpeg.go deleted file mode 100644 index 64b78df6..00000000 --- a/go_backend/extension_runtime_ffmpeg.go +++ /dev/null @@ -1,224 +0,0 @@ -package gobackend - -import ( - "context" - "fmt" - "regexp" - "sync" - "time" - - "github.com/dop251/goja" -) - -// FFmpegCommand holds a pending FFmpeg command for Flutter to execute. -type FFmpegCommand struct { - ExtensionID string - Arguments []string - InputPath string - OutputPath string - Completed bool - Claimed bool - Success bool - Error string - Output string - done chan struct{} -} - -var ( - ffmpegCommands = make(map[string]*FFmpegCommand) - ffmpegCommandsMu sync.RWMutex - ffmpegCommandID int64 - ffmpegCommandQueued = make(chan struct{}, 1) -) - -func notifyFFmpegCommandQueued() { - select { - case ffmpegCommandQueued <- struct{}{}: - default: - } -} - -func GetPendingFFmpegCommand(commandID string) *FFmpegCommand { - ffmpegCommandsMu.RLock() - defer ffmpegCommandsMu.RUnlock() - return ffmpegCommands[commandID] -} - -func SetFFmpegCommandResult(commandID string, success bool, output, errorMsg string) { - ffmpegCommandsMu.Lock() - defer ffmpegCommandsMu.Unlock() - if cmd, exists := ffmpegCommands[commandID]; exists { - if cmd.Completed { - return - } - cmd.Completed = true - cmd.Success = success - cmd.Output = output - cmd.Error = errorMsg - if cmd.done != nil { - close(cmd.done) - } - } -} - -func ClearFFmpegCommand(commandID string) { - ffmpegCommandsMu.Lock() - defer ffmpegCommandsMu.Unlock() - delete(ffmpegCommands, commandID) -} - -func (r *extensionRuntime) ffmpegExecute(call goja.FunctionCall) goja.Value { - // A raw command can introduce additional file inputs and network protocols, - // bypassing both validatePath and the extension network allow-list. Keep the - // API stub for compatibility, but never forward an unstructured command to - // the native FFmpeg process. - return r.jsError("raw FFmpeg execution is disabled; use ffmpeg.convert") -} - -func (r *extensionRuntime) executeFFmpegCommand(arguments []string, inputPath, outputPath string) goja.Value { - ctx := r.activeOperationContext(context.Background()) - if budget := r.currentResolutionBudget(); budget != nil { - defer budget.pause()() - } - if ctx.Err() != nil { - return r.jsError("FFmpeg command cancelled: %v", context.Cause(ctx)) - } - - ffmpegCommandsMu.Lock() - ffmpegCommandID++ - cmdID := fmt.Sprintf("%s_%d", r.extensionID, ffmpegCommandID) - queuedCommand := &FFmpegCommand{ - ExtensionID: r.extensionID, - Arguments: append([]string(nil), arguments...), - InputPath: inputPath, - OutputPath: outputPath, - Completed: false, - done: make(chan struct{}), - } - ffmpegCommands[cmdID] = queuedCommand - ffmpegCommandsMu.Unlock() - notifyFFmpegCommandQueued() - - GoLog("[Extension:%s] FFmpeg command queued: %s\n", r.extensionID, cmdID) - - select { - case <-queuedCommand.done: - ffmpegCommandsMu.Lock() - result := map[string]any{ - "success": queuedCommand.Success, - "output": queuedCommand.Output, - } - if queuedCommand.Error != "" { - result["error"] = queuedCommand.Error - } - delete(ffmpegCommands, cmdID) - ffmpegCommandsMu.Unlock() - return r.vm.ToValue(result) - case <-ctx.Done(): - ClearFFmpegCommand(cmdID) - return r.jsError("FFmpeg command cancelled: %v", context.Cause(ctx)) - case <-time.After(5 * time.Minute): - ClearFFmpegCommand(cmdID) - return r.jsError("FFmpeg command timed out") - } -} - -var ffmpegBitratePattern = regexp.MustCompile(`^[1-9][0-9]{0,7}[kKmM]?$`) - -var allowedFFmpegAudioCodecs = map[string]struct{}{ - "aac": {}, - "alac": {}, - "copy": {}, - "flac": {}, - "libmp3lame": {}, - "libopus": {}, - "opus": {}, - "pcm_s16le": {}, - "pcm_s24le": {}, -} - -func (r *extensionRuntime) ffmpegGetInfo(call goja.FunctionCall) goja.Value { - if r.manifest == nil || !r.manifest.Permissions.File { - return r.jsError("file permission denied") - } - if len(call.Arguments) < 1 { - return r.jsError("file path is required") - } - - filePath, err := r.validatePath(call.Arguments[0].String()) - if err != nil { - return r.jsError("%s", err.Error()) - } - - quality, err := GetAudioQuality(filePath) - if err != nil { - return r.jsError("%s", err.Error()) - } - - return r.jsSuccess(map[string]any{ - "bit_depth": quality.BitDepth, - "sample_rate": quality.SampleRate, - "total_samples": quality.TotalSamples, - "duration": float64(quality.TotalSamples) / float64(quality.SampleRate), - "codec": quality.Codec, - }) -} - -func (r *extensionRuntime) ffmpegConvert(call goja.FunctionCall) goja.Value { - if r.manifest == nil || !r.manifest.Permissions.File { - return r.jsError("file permission denied") - } - if len(call.Arguments) < 2 { - return r.jsError("input and output paths are required") - } - - inputPath, err := r.validatePath(call.Arguments[0].String()) - if err != nil { - return r.jsError("invalid input path: %v", err) - } - outputPath, err := r.validatePath(call.Arguments[1].String()) - if err != nil { - return r.jsError("invalid output path: %v", err) - } - - options := map[string]any{} - if len(call.Arguments) > 2 && !goja.IsUndefined(call.Arguments[2]) && !goja.IsNull(call.Arguments[2]) { - if opts, ok := call.Arguments[2].Export().(map[string]any); ok { - options = opts - } - } - - arguments := []string{"-hide_banner", "-nostdin", "-i", inputPath} - - if codec, ok := options["codec"].(string); ok { - if _, allowed := allowedFFmpegAudioCodecs[codec]; !allowed { - return r.jsError("unsupported audio codec") - } - arguments = append(arguments, "-c:a", codec) - } - - if bitrate, ok := options["bitrate"].(string); ok { - if !ffmpegBitratePattern.MatchString(bitrate) { - return r.jsError("invalid audio bitrate") - } - arguments = append(arguments, "-b:a", bitrate) - } - - if sampleRate, ok := options["sample_rate"].(float64); ok { - if sampleRate < 8_000 || sampleRate > 768_000 || sampleRate != float64(int(sampleRate)) { - return r.jsError("invalid sample rate") - } - arguments = append(arguments, "-ar", fmt.Sprintf("%d", int(sampleRate))) - } - - if channels, ok := options["channels"].(float64); ok { - if channels < 1 || channels > 32 || channels != float64(int(channels)) { - return r.jsError("invalid channel count") - } - arguments = append(arguments, "-ac", fmt.Sprintf("%d", int(channels))) - } - - arguments = append(arguments, "-y", outputPath) - - return r.executeFFmpegCommand(arguments, inputPath, outputPath) -} diff --git a/go_backend/extension_runtime_ffmpeg_test.go b/go_backend/extension_runtime_ffmpeg_test.go deleted file mode 100644 index bc7627fa..00000000 --- a/go_backend/extension_runtime_ffmpeg_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package gobackend - -import ( - "strings" - "testing" - - "github.com/dop251/goja" -) - -func TestWaitForPendingFFmpegCommandsClaimsCommandOnce(t *testing.T) { - const commandID = "wait-claim-test" - command := &FFmpegCommand{ - ExtensionID: "test-extension", - Arguments: []string{"-version"}, - done: make(chan struct{}), - } - ffmpegCommandsMu.Lock() - ffmpegCommands[commandID] = command - ffmpegCommandsMu.Unlock() - notifyFFmpegCommandQueued() - t.Cleanup(func() { ClearFFmpegCommand(commandID) }) - - first, err := WaitForPendingFFmpegCommandsJSON(50) - if err != nil || !strings.Contains(first, commandID) { - t.Fatalf("first wait = %q, %v", first, err) - } - second, err := WaitForPendingFFmpegCommandsJSON(1) - if err != nil || second != "[]" { - t.Fatalf("claimed command returned twice: %q, %v", second, err) - } - - SetFFmpegCommandResult(commandID, true, "ok", "") - select { - case <-command.done: - default: - t.Fatal("command completion did not signal waiter") - } -} - -func TestExtensionFFmpegRejectsRawAndInjectedOptions(t *testing.T) { - vm := goja.New() - runtime := &extensionRuntime{ - extensionID: "ffmpeg-security", - manifest: &ExtensionManifest{ - Permissions: ExtensionPermissions{File: true}, - Capabilities: map[string]any{"rawFfmpeg": true}, - }, - dataDir: t.TempDir(), - vm: vm, - } - raw := runtime.ffmpegExecute(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("-i /private/secret -f data out"), - }}).Export().(map[string]any) - if raw["success"] != false || !strings.Contains(raw["error"].(string), "disabled") { - t.Fatalf("raw FFmpeg was not rejected: %#v", raw) - } - - injected := runtime.ffmpegConvert(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("input.flac"), - vm.ToValue("output.flac"), - vm.ToValue(map[string]any{"codec": "flac -i /private/secret"}), - }}).Export().(map[string]any) - if injected["success"] != false || !strings.Contains(injected["error"].(string), "unsupported") { - t.Fatalf("FFmpeg option injection was not rejected: %#v", injected) - } - - injectedBitrate := runtime.ffmpegConvert(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("input.flac"), - vm.ToValue("output.m4a"), - vm.ToValue(map[string]any{"bitrate": "320k -i /private/secret"}), - }}).Export().(map[string]any) - if injectedBitrate["success"] != false || - !strings.Contains(injectedBitrate["error"].(string), "bitrate") { - t.Fatalf("FFmpeg bitrate injection was not rejected: %#v", injectedBitrate) - } -} diff --git a/go_backend/extension_runtime_file.go b/go_backend/extension_runtime_file.go deleted file mode 100644 index 45b75775..00000000 --- a/go_backend/extension_runtime_file.go +++ /dev/null @@ -1,645 +0,0 @@ -package gobackend - -import ( - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - - "github.com/dop251/goja" -) - -var ( - allowedDownloadDirs []string - allowedDownloadDirsMu sync.RWMutex -) - -func AddAllowedDownloadDir(dir string) { - absDir, err := filepath.Abs(dir) - if err != nil { - return - } - absDir = filepath.Clean(absDir) - - allowedDownloadDirsMu.Lock() - defer allowedDownloadDirsMu.Unlock() - for _, existing := range allowedDownloadDirs { - if existing == absDir { - return - } - } - allowedDownloadDirs = append(allowedDownloadDirs, absDir) -} - -// SetAllowedDownloadDirs replaces the whole allow-list in one call (passing nil -// clears it). Used by tests to reset the sandbox between cases; production code -// appends via AddAllowedDownloadDir. -func SetAllowedDownloadDirs(dirs []string) { - allowedDownloadDirsMu.Lock() - defer allowedDownloadDirsMu.Unlock() - allowedDownloadDirs = nil - seen := make(map[string]struct{}, len(dirs)) - for _, dir := range dirs { - absDir, err := filepath.Abs(dir) - if err != nil { - continue - } - absDir = filepath.Clean(absDir) - if _, duplicate := seen[absDir]; duplicate { - continue - } - seen[absDir] = struct{}{} - allowedDownloadDirs = append(allowedDownloadDirs, absDir) - } -} - -func isPathInAllowedDirs(absPath string) bool { - allowedDownloadDirsMu.RLock() - defer allowedDownloadDirsMu.RUnlock() - - for _, allowedDir := range allowedDownloadDirs { - if isPathWithinBase(allowedDir, absPath) { - return true - } - } - return false -} - -func isPathWithinBase(baseDir, targetPath string) bool { - baseAbs, err := filepath.Abs(baseDir) - if err != nil { - return false - } - targetAbs, err := filepath.Abs(targetPath) - if err != nil { - return false - } - - rel, err := filepath.Rel(baseAbs, targetAbs) - if err != nil { - return false - } - rel = filepath.Clean(rel) - if rel == "." { - return true - } - - prefix := ".." + string(filepath.Separator) - if rel == ".." || strings.HasPrefix(rel, prefix) { - return false - } - return true -} - -func (r *extensionRuntime) validatePath(path string) (string, error) { - if !r.manifest.Permissions.File { - return "", fmt.Errorf("file access denied: extension does not have 'file' permission") - } - - cleanPath := filepath.Clean(path) - - if filepath.IsAbs(cleanPath) { - absPath, err := filepath.Abs(cleanPath) - if err != nil { - return "", fmt.Errorf("invalid path: %w", err) - } - - if isPathInAllowedDirs(absPath) { - return absPath, nil - } - - return "", fmt.Errorf("file access denied: absolute paths are not allowed. Use relative paths within extension sandbox") - } - - fullPath := filepath.Join(r.dataDir, cleanPath) - - absPath, err := filepath.Abs(fullPath) - if err != nil { - return "", fmt.Errorf("invalid path: %w", err) - } - - absDataDir, _ := filepath.Abs(r.dataDir) - if !isPathWithinBase(absDataDir, absPath) { - return "", fmt.Errorf("file access denied: path '%s' is outside sandbox", path) - } - - return absPath, nil -} - -func (r *extensionRuntime) fileDownload(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.jsError("URL and output path are required") - } - - urlStr := call.Arguments[0].String() - outputPath := call.Arguments[1].String() - - if err := r.validateDomain(urlStr); err != nil { - return r.jsError("%s", err.Error()) - } - - fullPath, err := r.validatePath(outputPath) - if err != nil { - return r.jsError("%s", err.Error()) - } - - var onProgress goja.Callable - var headers map[string]string - var chunkedDownload bool - var resumeDownload bool - var resumeOptionSet bool - var persistentCheckpointOption *bool - var maxAttemptsOption int - trackItemBytes := true - var chunkSize int64 - if len(call.Arguments) > 2 && !goja.IsUndefined(call.Arguments[2]) && !goja.IsNull(call.Arguments[2]) { - optionsObj := call.Arguments[2].Export() - if opts, ok := optionsObj.(map[string]any); ok { - if h, ok := opts["headers"].(map[string]any); ok { - headers = make(map[string]string) - for k, v := range h { - headers[k] = fmt.Sprintf("%v", v) - } - } - if progressVal, ok := opts["onProgress"]; ok { - if callable, ok := goja.AssertFunction(r.vm.ToValue(progressVal)); ok { - onProgress = callable - } - } - if trackBytes, ok := opts["trackItemBytes"]; ok { - if v, ok := trackBytes.(bool); ok { - trackItemBytes = v - } - } else if trackBytes, ok := opts["track_item_bytes"]; ok { - if v, ok := trackBytes.(bool); ok { - trackItemBytes = v - } - } - if chunked, ok := opts["chunked"]; ok { - switch v := chunked.(type) { - case bool: - chunkedDownload = v - case int64: - if v > 0 { - chunkedDownload = true - chunkSize = v - } - case float64: - if v > 0 { - chunkedDownload = true - chunkSize = int64(v) - } - } - } - if resume, ok := opts["resume"]; ok { - if v, ok := resume.(bool); ok { - resumeDownload = v - resumeOptionSet = true - } - } - if checkpoint, ok := opts["persistentCheckpoint"]; ok { - if v, ok := checkpoint.(bool); ok { - persistentCheckpointOption = &v - } - } - if attempts, ok := opts["maxAttempts"]; ok { - switch v := attempts.(type) { - case int64: - maxAttemptsOption = int(v) - case float64: - maxAttemptsOption = int(v) - } - } - } - } - - // Default chunk size: 1MB (YouTube CDN max without poToken) - if chunkedDownload && chunkSize <= 0 { - chunkSize = 1024 * 1024 - } - - dir := filepath.Dir(fullPath) - if err := os.MkdirAll(dir, 0755); err != nil { - return r.jsError("failed to create directory: %v", err) - } - - client := r.downloadClient - if client == nil { - client = r.httpClient - } - - ua := appUserAgent() - if h, ok := headers["User-Agent"]; ok && h != "" { - ua = h - } - - policy := r.manifest.DownloadTransferPolicy() - if !resumeOptionSet { - resumeDownload = policy.ResumePolicy == "validated" - } - persistentCheckpoint := policy.PersistentCheckpoint - if persistentCheckpointOption != nil { - persistentCheckpoint = *persistentCheckpointOption && resumeDownload - } - if maxAttemptsOption > 0 { - policy.MaxAttempts = clampInt(maxAttemptsOption, 1, 8) - } - if chunkedDownload { - return r.fileDownloadChunked( - client, - urlStr, - fullPath, - headers, - ua, - chunkSize, - onProgress, - trackItemBytes, - persistentCheckpoint, - policy, - ) - } - return r.reliableFileDownload( - client, - urlStr, - fullPath, - headers, - onProgress, - trackItemBytes, - resumeDownload, - persistentCheckpoint, - policy, - ) -} - -func (r *extensionRuntime) fileExists(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue(false) - } - - path := call.Arguments[0].String() - fullPath, err := r.validatePath(path) - if err != nil { - return r.vm.ToValue(false) - } - - _, err = os.Stat(fullPath) - return r.vm.ToValue(err == nil) -} - -func (r *extensionRuntime) fileDelete(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.jsError("path is required") - } - - path := call.Arguments[0].String() - fullPath, err := r.validatePath(path) - if err != nil { - return r.jsError("%s", err.Error()) - } - - if err := os.Remove(fullPath); err != nil { - return r.jsError("%s", err.Error()) - } - - return r.jsSuccess(nil) -} - -func (r *extensionRuntime) fileRead(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.jsError("path is required") - } - - path := call.Arguments[0].String() - fullPath, err := r.validatePath(path) - if err != nil { - return r.jsError("%s", err.Error()) - } - - file, err := os.Open(fullPath) - if err != nil { - return r.jsError("%s", err.Error()) - } - defer file.Close() - - data, err := io.ReadAll(io.LimitReader(file, maxExtensionFileReadBytes+1)) - if err != nil { - return r.jsError("%s", err.Error()) - } - if int64(len(data)) > maxExtensionFileReadBytes { - return r.jsError(extensionFileReadLimitError) - } - - return r.jsSuccess(map[string]any{ - "data": string(data), - }) -} - -const ( - maxExtensionFileReadBytes = int64(16 << 20) - extensionFileReadLimitError = "file read exceeds 16 MiB limit; use file.readBytes with offset and length to read it in chunks" -) - -func extensionFileReadLength(size, offset, requested int64) (int64, error) { - remaining := size - offset - if remaining < 0 { - remaining = 0 - } - - if requested < 0 { - if remaining > maxExtensionFileReadBytes { - return 0, fmt.Errorf("%s", extensionFileReadLimitError) - } - return remaining, nil - } - if requested > maxExtensionFileReadBytes { - return 0, fmt.Errorf("%s", extensionFileReadLimitError) - } - return min(requested, remaining), nil -} - -func (r *extensionRuntime) fileReadBytes(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.jsError("path is required") - } - - path := call.Arguments[0].String() - fullPath, err := r.validatePath(path) - if err != nil { - return r.jsError("%s", err.Error()) - } - - options := parseRuntimeOptionsArgument(call, 1) - offset := runtimeOptionInt64(options, "offset", 0) - length := runtimeOptionInt64(options, "length", -1) - encoding := runtimeOptionString(options, "encoding", "base64") - if offset < 0 { - return r.jsError("offset must be >= 0") - } - file, err := os.Open(fullPath) - if err != nil { - return r.jsError("%s", err.Error()) - } - defer file.Close() - - info, err := file.Stat() - if err != nil { - return r.jsError("%s", err.Error()) - } - - size := info.Size() - if offset > size { - offset = size - } - if _, err := file.Seek(offset, io.SeekStart); err != nil { - return r.jsError("failed to seek file: %v", err) - } - - readLength, err := extensionFileReadLength(size, offset, length) - if err != nil { - return r.jsError("%s", err.Error()) - } - data := make([]byte, int(readLength)) - if readLength > 0 { - n, readErr := io.ReadFull(file, data) - if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF { - return r.jsError("failed to read file: %v", readErr) - } - data = data[:n] - } - - if strings.EqualFold(strings.TrimSpace(encoding), "bytes") || - strings.EqualFold(strings.TrimSpace(encoding), "raw") { - // Return raw bytes as an ArrayBuffer to avoid base64 encode/decode of - // large payloads under the goja interpreter. - return r.jsSuccess(map[string]any{ - "data": r.vm.NewArrayBuffer(data), - "bytes_read": len(data), - "offset": offset, - "size": size, - "eof": offset+int64(len(data)) >= size, - }) - } - - encoded, err := encodeRuntimeBytes(data, encoding) - if err != nil { - return r.jsError("%s", err.Error()) - } - - return r.jsSuccess(map[string]any{ - "data": encoded, - "bytes_read": len(data), - "offset": offset, - "size": size, - "eof": offset+int64(len(data)) >= size, - }) -} -func (r *extensionRuntime) fileWrite(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.jsError("path and data are required") - } - - path := call.Arguments[0].String() - data := call.Arguments[1].String() - - fullPath, err := r.validatePath(path) - if err != nil { - return r.jsError("%s", err.Error()) - } - - dir := filepath.Dir(fullPath) - if err := os.MkdirAll(dir, 0755); err != nil { - return r.jsError("failed to create directory: %v", err) - } - - // Full-content write: stage and rename so a kill mid-write cannot leave - // a truncated file under the final name. - stagedPath := stagedDownloadPath(fullPath) - if err := os.WriteFile(stagedPath, []byte(data), 0644); err != nil { - os.Remove(stagedPath) - return r.jsError("%s", err.Error()) - } - if err := os.Rename(stagedPath, fullPath); err != nil { - os.Remove(stagedPath) - return r.jsError("%s", err.Error()) - } - - return r.jsSuccess(map[string]any{ - "path": fullPath, - }) -} - -func (r *extensionRuntime) fileWriteBytes(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.jsError("path and data are required") - } - - path := call.Arguments[0].String() - fullPath, err := r.validatePath(path) - if err != nil { - return r.jsError("%s", err.Error()) - } - - options := parseRuntimeOptionsArgument(call, 2) - appendMode := runtimeOptionBool(options, "append", false) - truncate := runtimeOptionBool(options, "truncate", false) - hasOffset := runtimeOptionHasKey(options, "offset") - offset := runtimeOptionInt64(options, "offset", 0) - encoding := runtimeOptionString(options, "encoding", "base64") - - if appendMode && hasOffset { - return r.jsError("append and offset cannot be used together") - } - if offset < 0 { - return r.jsError("offset must be >= 0") - } - - data, err := decodeRuntimeBytesValue(call.Arguments[1].Export(), encoding) - if err != nil { - return r.jsError("%s", err.Error()) - } - - dir := filepath.Dir(fullPath) - if err := os.MkdirAll(dir, 0755); err != nil { - return r.jsError("failed to create directory: %v", err) - } - - flags := os.O_CREATE | os.O_WRONLY - if appendMode { - flags |= os.O_APPEND - } - if truncate { - flags |= os.O_TRUNC - } - - file, err := os.OpenFile(fullPath, flags, 0644) - if err != nil { - return r.jsError("%s", err.Error()) - } - defer file.Close() - - if hasOffset && !appendMode { - if _, err := file.Seek(offset, io.SeekStart); err != nil { - return r.jsError("failed to seek file: %v", err) - } - } - - written, err := file.Write(data) - if err != nil { - return r.jsError("%s", err.Error()) - } - - info, statErr := file.Stat() - size := int64(0) - if statErr == nil { - size = info.Size() - } - - return r.jsSuccess(map[string]any{ - "path": fullPath, - "bytes_written": written, - "size": size, - }) -} - -func (r *extensionRuntime) fileCopy(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.jsError("source and destination paths are required") - } - - srcPath := call.Arguments[0].String() - dstPath := call.Arguments[1].String() - - fullSrc, err := r.validatePath(srcPath) - if err != nil { - return r.jsError("%s", err.Error()) - } - - fullDst, err := r.validatePath(dstPath) - if err != nil { - return r.jsError("%s", err.Error()) - } - - srcFile, err := os.Open(fullSrc) - if err != nil { - return r.jsError("failed to read source: %v", err) - } - defer srcFile.Close() - - dir := filepath.Dir(fullDst) - if err := os.MkdirAll(dir, 0755); err != nil { - return r.jsError("failed to create directory: %v", err) - } - - dstFile, err := os.OpenFile(fullDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return r.jsError("failed to open destination: %v", err) - } - - if _, err := io.Copy(dstFile, srcFile); err != nil { - _ = dstFile.Close() - return r.jsError("failed to copy file: %v", err) - } - - if err := dstFile.Close(); err != nil { - return r.jsError("failed to finalize destination: %v", err) - } - - return r.jsSuccess(map[string]any{ - "path": fullDst, - }) -} - -func (r *extensionRuntime) fileMove(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.jsError("source and destination paths are required") - } - - srcPath := call.Arguments[0].String() - dstPath := call.Arguments[1].String() - - fullSrc, err := r.validatePath(srcPath) - if err != nil { - return r.jsError("%s", err.Error()) - } - - fullDst, err := r.validatePath(dstPath) - if err != nil { - return r.jsError("%s", err.Error()) - } - - dir := filepath.Dir(fullDst) - if err := os.MkdirAll(dir, 0755); err != nil { - return r.jsError("failed to create directory: %v", err) - } - - if err := os.Rename(fullSrc, fullDst); err != nil { - return r.jsError("failed to move file: %v", err) - } - - return r.jsSuccess(map[string]any{ - "path": fullDst, - }) -} - -func (r *extensionRuntime) fileGetSize(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.jsError("path is required") - } - - path := call.Arguments[0].String() - fullPath, err := r.validatePath(path) - if err != nil { - return r.jsError("%s", err.Error()) - } - - info, err := os.Stat(fullPath) - if err != nil { - return r.jsError("%s", err.Error()) - } - - return r.jsSuccess(map[string]any{ - "size": info.Size(), - }) -} diff --git a/go_backend/extension_runtime_file_download_integrity_test.go b/go_backend/extension_runtime_file_download_integrity_test.go deleted file mode 100644 index fe959833..00000000 --- a/go_backend/extension_runtime_file_download_integrity_test.go +++ /dev/null @@ -1,417 +0,0 @@ -package gobackend - -import ( - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/dop251/goja" -) - -// shortCleanEOFBodyReader simulates a transport that surfaces a mid-transfer -// connection drop as a normal io.EOF instead of io.ErrUnexpectedEOF (which is -// what a network reset looks like through some custom transports, e.g. the -// uTLS-based client this app uses for TLS-fingerprint spoofing). It sends -// data once and then reports a clean end of stream, even though fewer bytes -// were sent than the response's Content-Length promised. -type shortCleanEOFBodyReader struct { - data []byte - sent bool -} - -func (f *shortCleanEOFBodyReader) Read(p []byte) (int, error) { - if !f.sent { - f.sent = true - n := copy(p, f.data) - return n, io.EOF - } - return 0, io.EOF -} - -func TestFileDownloadShortCleanEOFFailsByDefaultEvenWithValidator(t *testing.T) { - var attempts int - var rangeSeen bool - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - attempts++ - rangeSeen = rangeSeen || req.Header.Get("Range") != "" - h := make(http.Header) - h.Set("ETag", `"v1"`) - return &http.Response{ - StatusCode: 200, - Header: h, - Body: io.NopCloser(&shortCleanEOFBodyReader{data: []byte("hello-")}), - ContentLength: int64(len("hello-world!")), - Request: req, - }, nil - }) - - // Resume is opt-in, so a short clean EOF must fail and clean up just like - // a real read error would. The generic transfer engine may retry the whole - // object, but must not send a Range request merely because a validator - // happens to be present. - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("https://cdn.example.com/track.flac"), - runtime.vm.ToValue("out/track.flac"), - }}).Export().(map[string]any) - if result["success"] != false { - t.Fatalf("expected failed download, got %#v", result) - } - if attempts != defaultTransferMaxAttempts { - t.Fatalf("attempts = %d, want %d full retries", attempts, defaultTransferMaxAttempts) - } - if rangeSeen { - t.Fatal("default retry unexpectedly sent a Range request") - } - - finalPath := filepath.Join(runtime.dataDir, "out", "track.flac") - if _, err := os.Stat(finalPath); !os.IsNotExist(err) { - t.Fatalf("truncated download was promoted to the final path: %v", err) - } - if _, err := os.Stat(stagedDownloadPath(finalPath)); !os.IsNotExist(err) { - t.Fatalf("staged partial file left behind: %v", err) - } -} - -func TestFileDownloadResumesAfterShortCleanEOFWhenEnabled(t *testing.T) { - const full = "hello-world!" - var attempts int - var resumeRange, resumeIfRange string - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - attempts++ - if attempts == 1 { - h := make(http.Header) - h.Set("ETag", `"v1"`) - return &http.Response{ - StatusCode: 200, - Header: h, - Body: io.NopCloser(&shortCleanEOFBodyReader{data: []byte(full[:6])}), - ContentLength: int64(len(full)), - Request: req, - }, nil - } - resumeRange = req.Header.Get("Range") - resumeIfRange = req.Header.Get("If-Range") - h := make(http.Header) - h.Set("Content-Range", fmt.Sprintf("bytes 6-%d/%d", len(full)-1, len(full))) - return &http.Response{ - StatusCode: 206, - Header: h, - Body: io.NopCloser(&shortCleanEOFBodyReader{data: []byte(full[6:])}), - ContentLength: int64(len(full) - 6), - Request: req, - }, nil - }) - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("https://cdn.example.com/track.flac"), - runtime.vm.ToValue("out/track.flac"), - runtime.vm.ToValue(map[string]any{"resume": true}), - }}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("download result = %#v", result) - } - if attempts != 2 || resumeRange != "bytes=6-" || resumeIfRange != `"v1"` { - t.Fatalf("attempts=%d range=%q if-range=%q", attempts, resumeRange, resumeIfRange) - } - - finalPath := filepath.Join(runtime.dataDir, "out", "track.flac") - data, err := os.ReadFile(finalPath) - if err != nil || string(data) != full { - t.Fatalf("final file = %q/%v (a truncated file was silently promoted)", data, err) - } -} - -func TestFileDownloadResumesPersistentCheckpointInNewRuntime(t *testing.T) { - const full = "hello-world!" - firstRuntime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - header := make(http.Header) - header.Set("ETag", `"v1"`) - return &http.Response{ - StatusCode: http.StatusOK, - Header: header, - Body: io.NopCloser(&shortCleanEOFBodyReader{data: []byte(full[:6])}), - ContentLength: int64(len(full)), - Request: req, - }, nil - }) - options := map[string]any{ - "resume": true, - "persistentCheckpoint": true, - "maxAttempts": float64(1), - } - firstResult := firstRuntime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - firstRuntime.vm.ToValue("https://cdn.example.com/track.flac?token=old"), - firstRuntime.vm.ToValue("out/track.flac"), - firstRuntime.vm.ToValue(options), - }}).Export().(map[string]any) - if firstResult["success"] != false { - t.Fatalf("first result = %#v", firstResult) - } - - var resumeRange, resumeIfRange string - secondRuntime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - resumeRange = req.Header.Get("Range") - resumeIfRange = req.Header.Get("If-Range") - header := make(http.Header) - header.Set("Content-Range", fmt.Sprintf("bytes 6-%d/%d", len(full)-1, len(full))) - return &http.Response{ - StatusCode: http.StatusPartialContent, - Header: header, - Body: io.NopCloser(strings.NewReader(full[6:])), - ContentLength: int64(len(full) - 6), - Request: req, - }, nil - }) - secondRuntime.dataDir = firstRuntime.dataDir - secondResult := secondRuntime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - secondRuntime.vm.ToValue("https://cdn.example.com/track.flac?token=fresh"), - secondRuntime.vm.ToValue("out/track.flac"), - secondRuntime.vm.ToValue(options), - }}).Export().(map[string]any) - if secondResult["success"] != true { - t.Fatalf("second result = %#v", secondResult) - } - if resumeRange != "bytes=6-" || resumeIfRange != `"v1"` { - t.Fatalf("range=%q if-range=%q", resumeRange, resumeIfRange) - } - data, err := os.ReadFile(filepath.Join(firstRuntime.dataDir, "out", "track.flac")) - if err != nil || string(data) != full { - t.Fatalf("resumed file = %q, err=%v", data, err) - } -} - -func TestFileDownloadShortCleanEOFWithoutValidatorFails(t *testing.T) { - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - // No ETag/Last-Modified, so the download cannot be resumed and must - // fail outright rather than promote a truncated file. - return &http.Response{ - StatusCode: 200, - Header: make(http.Header), - Body: io.NopCloser(&shortCleanEOFBodyReader{data: []byte("partial-aud")}), - ContentLength: 1 << 20, - Request: req, - }, nil - }) - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("https://cdn.example.com/track.flac"), - runtime.vm.ToValue("out/track.flac"), - }}).Export().(map[string]any) - if result["success"] != false { - t.Fatalf("expected a failed download for a short clean EOF with no validator, got %#v", result) - } - - finalPath := filepath.Join(runtime.dataDir, "out", "track.flac") - if _, err := os.Stat(finalPath); !os.IsNotExist(err) { - t.Fatalf("truncated download was promoted to the final path: %v", err) - } - if _, err := os.Stat(stagedDownloadPath(finalPath)); !os.IsNotExist(err) { - t.Fatalf("staged partial file left behind: %v", err) - } -} - -func TestChunkedDownloadRestartsWhenServerIgnoresLaterRange(t *testing.T) { - const full = "abcdefgh" - var requestedRanges []string - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - rangeHeader := req.Header.Get("Range") - requestedRanges = append(requestedRanges, rangeHeader) - header := make(http.Header) - header.Set("ETag", `"v1"`) - switch rangeHeader { - case "bytes=0-1": - header.Set("Content-Range", "bytes 0-1/8") - return &http.Response{ - StatusCode: http.StatusPartialContent, - Header: header, - Body: io.NopCloser(strings.NewReader(full[:2])), - ContentLength: 2, - Request: req, - }, nil - case "bytes=0-2": - header.Set("Content-Range", "bytes 0-2/8") - return &http.Response{ - StatusCode: http.StatusPartialContent, - Header: header, - Body: io.NopCloser(strings.NewReader(full[:3])), - ContentLength: 3, - Request: req, - }, nil - default: - // Some CDNs invalidate or ignore Range after the first request. The - // full response must replace the partial file instead of being appended. - return &http.Response{ - StatusCode: http.StatusOK, - Header: header, - Body: io.NopCloser(strings.NewReader(full)), - ContentLength: int64(len(full)), - Request: req, - }, nil - } - }) - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("https://cdn.example.com/track.flac"), - runtime.vm.ToValue("out/track.flac"), - runtime.vm.ToValue(map[string]any{"chunked": float64(3)}), - }}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("chunked result = %#v", result) - } - data, err := os.ReadFile(filepath.Join(runtime.dataDir, "out", "track.flac")) - if err != nil || string(data) != full { - t.Fatalf("chunked file = %q, err=%v", data, err) - } - if got := requestedRanges[len(requestedRanges)-1]; got != "bytes=3-5" { - t.Fatalf("last requested range = %q, all=%v", got, requestedRanges) - } -} - -func TestFileDownloadRejectsChangedValidatorDuringResume(t *testing.T) { - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - header := make(http.Header) - header.Set("ETag", `"v2"`) - header.Set("Content-Range", "bytes 3-5/6") - return &http.Response{ - StatusCode: http.StatusPartialContent, - Header: header, - Body: io.NopCloser(strings.NewReader("DEF")), - ContentLength: 3, - Request: req, - }, nil - }) - fullPath := filepath.Join(runtime.dataDir, "out", "track.flac") - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatal(err) - } - stagedPath := stagedDownloadPath(fullPath) - if err := os.WriteFile(stagedPath, []byte("ABC"), 0600); err != nil { - t.Fatal(err) - } - url := "https://cdn.example.com/track.flac" - if err := saveTransferCheckpoint(transferCheckpointPath(stagedPath), transferCheckpoint{ - Fingerprint: transferURLFingerprint(url), - Validator: `"v1"`, - Bytes: 3, - Total: 6, - }); err != nil { - t.Fatal(err) - } - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue(url), - runtime.vm.ToValue("out/track.flac"), - runtime.vm.ToValue(map[string]any{ - "resume": true, - "persistentCheckpoint": true, - }), - }}).Export().(map[string]any) - if result["success"] != false || result["error_type"] != "integrity_failed" { - t.Fatalf("changed-validator result = %#v", result) - } - if _, err := os.Stat(fullPath); !os.IsNotExist(err) { - t.Fatalf("changed entity was published: %v", err) - } -} - -func TestFileDownloadCallerRangeDoesNotReuseEngineCheckpoint(t *testing.T) { - var observedRange string - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - observedRange = req.Header.Get("Range") - header := make(http.Header) - header.Set("Content-Range", "bytes 5-7/8") - return &http.Response{ - StatusCode: http.StatusPartialContent, - Header: header, - Body: io.NopCloser(strings.NewReader("NEW")), - ContentLength: 3, - Request: req, - }, nil - }) - fullPath := filepath.Join(runtime.dataDir, "out", "fragment.bin") - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatal(err) - } - stagedPath := stagedDownloadPath(fullPath) - if err := os.WriteFile(stagedPath, []byte("OLD"), 0600); err != nil { - t.Fatal(err) - } - url := "https://cdn.example.com/track.flac" - if err := saveTransferCheckpoint(transferCheckpointPath(stagedPath), transferCheckpoint{ - Fingerprint: transferURLFingerprint(url), - Validator: `"v1"`, - Bytes: 3, - Total: 8, - }); err != nil { - t.Fatal(err) - } - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue(url), - runtime.vm.ToValue("out/fragment.bin"), - runtime.vm.ToValue(map[string]any{ - "headers": map[string]any{"Range": "bytes=5-7"}, - "resume": true, - "persistentCheckpoint": true, - }), - }}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("caller-range result = %#v", result) - } - data, err := os.ReadFile(fullPath) - if err != nil || string(data) != "NEW" { - t.Fatalf("caller-range file = %q, err=%v", data, err) - } - if observedRange != "bytes=5-7" { - t.Fatalf("observed Range = %q", observedRange) - } -} - -func TestFileDownloadPromotesFullyCheckpointedStagedFileWithoutNetwork(t *testing.T) { - var networkCalls int - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - networkCalls++ - return nil, fmt.Errorf("network should not be called") - }) - const full = "already-complete" - fullPath := filepath.Join(runtime.dataDir, "out", "track.flac") - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatal(err) - } - stagedPath := stagedDownloadPath(fullPath) - if err := os.WriteFile(stagedPath, []byte(full), 0600); err != nil { - t.Fatal(err) - } - url := "https://cdn.example.com/track.flac" - if err := saveTransferCheckpoint(transferCheckpointPath(stagedPath), transferCheckpoint{ - Fingerprint: transferURLFingerprint(url), - Validator: `"v1"`, - Bytes: int64(len(full)), - Total: int64(len(full)), - }); err != nil { - t.Fatal(err) - } - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue(url), - runtime.vm.ToValue("out/track.flac"), - runtime.vm.ToValue(map[string]any{ - "resume": true, - "persistentCheckpoint": true, - }), - }}).Export().(map[string]any) - if result["success"] != true || result["resumed"] != true { - t.Fatalf("fully checkpointed result = %#v", result) - } - if networkCalls != 0 { - t.Fatalf("network calls = %d", networkCalls) - } - data, err := os.ReadFile(fullPath) - if err != nil || string(data) != full { - t.Fatalf("promoted file = %q, err=%v", data, err) - } -} diff --git a/go_backend/extension_runtime_file_limits_test.go b/go_backend/extension_runtime_file_limits_test.go deleted file mode 100644 index d67547fe..00000000 --- a/go_backend/extension_runtime_file_limits_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package gobackend - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/dop251/goja" -) - -func TestExtensionFileReadLengthRequiresChunking(t *testing.T) { - if _, err := extensionFileReadLength(maxExtensionFileReadBytes+1, 0, -1); err == nil { - t.Fatal("unbounded large read was accepted") - } - if _, err := extensionFileReadLength(maxExtensionFileReadBytes+1, 0, maxExtensionFileReadBytes+1); err == nil { - t.Fatal("oversized explicit read was accepted") - } - if got, err := extensionFileReadLength(maxExtensionFileReadBytes+1, maxExtensionFileReadBytes, -1); err != nil || got != 1 { - t.Fatalf("tail read length = %d, %v", got, err) - } - if got, err := extensionFileReadLength(5, 2, 10); err != nil || got != 3 { - t.Fatalf("clamped read length = %d, %v", got, err) - } -} - -func TestExtensionFileAPIsRejectUnboundedLargeReads(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "large.bin") - file, err := os.Create(path) - if err != nil { - t.Fatal(err) - } - if err := file.Truncate(maxExtensionFileReadBytes + 1); err != nil { - file.Close() - t.Fatal(err) - } - file.Close() - - vm := goja.New() - runtime := &extensionRuntime{ - vm: vm, - dataDir: dir, - manifest: &ExtensionManifest{Permissions: ExtensionPermissions{File: true}}, - } - - readResult := runtime.fileRead(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("large.bin")}}).Export().(map[string]any) - if readResult["success"] != false || !strings.Contains(readResult["error"].(string), "chunks") { - t.Fatalf("fileRead result = %#v", readResult) - } - - readBytesResult := runtime.fileReadBytes(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("large.bin")}}).Export().(map[string]any) - if readBytesResult["success"] != false || !strings.Contains(readBytesResult["error"].(string), "chunks") { - t.Fatalf("fileReadBytes result = %#v", readBytesResult) - } -} diff --git a/go_backend/extension_runtime_file_transform.go b/go_backend/extension_runtime_file_transform.go deleted file mode 100644 index bd2c45e0..00000000 --- a/go_backend/extension_runtime_file_transform.go +++ /dev/null @@ -1,245 +0,0 @@ -package gobackend - -import ( - "context" - "crypto/cipher" - "io" - "os" - "path/filepath" - "strings" - - "github.com/dop251/goja" -) - -const ( - defaultPatternedTransformBufferSize = int64(1 << 20) - maxPatternedTransformBufferSize = int64(16 << 20) - maxPatternedTransformSegmentSize = int64(16 << 20) -) - -// fileTransformPatternedBlocks streams one file into another while applying an -// independent block-cipher transform to selected fixed-size segments. It keeps -// provider-specific layout knowledge in the extension: the host only receives -// a generic period/offset declaration and cipher parameters. -// -// JS signature: -// -// file.transformPatternedBlocks(inputPath, outputPath, { -// operation: "decrypt", algorithm: "blowfish", mode: "cbc", -// key: "...", keyEncoding: "hex", iv: "...", ivEncoding: "hex", -// segmentSize: 2048, transformEvery: 3, transformOffset: 0, -// bufferSize: 1048576, transformPartial: false -// }, function(processedBytes, totalBytes) {}) -func (r *extensionRuntime) fileTransformPatternedBlocks(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 3 { - return r.jsError("input path, output path, and options are required") - } - - inputPath, err := r.validatePath(call.Arguments[0].String()) - if err != nil { - return r.jsError("%s", err.Error()) - } - outputPath, err := r.validatePath(call.Arguments[1].String()) - if err != nil { - return r.jsError("%s", err.Error()) - } - options := parseRuntimeOptionsArgument(call, 2) - parsedCipher, err := parseRuntimeBlockCipherOptions(options) - if err != nil { - return r.jsError("%s", err.Error()) - } - if parsedCipher.Padding != "none" { - return r.jsError("patterned file transforms only support padding: none") - } - if parsedCipher.Mode != "cbc" && parsedCipher.Mode != "ctr" { - return r.jsError("unsupported block cipher mode: %s", parsedCipher.Mode) - } - operation := strings.ToLower(runtimeOptionString(options, "operation", "decrypt")) - if operation != "decrypt" && operation != "encrypt" { - return r.jsError("operation must be decrypt or encrypt") - } - - segmentSize := runtimeOptionInt64(options, "segmentSize", 0) - transformEvery := runtimeOptionInt64(options, "transformEvery", 1) - transformOffset := runtimeOptionInt64(options, "transformOffset", 0) - bufferSize := runtimeOptionInt64(options, "bufferSize", defaultPatternedTransformBufferSize) - transformPartial := runtimeOptionBool(options, "transformPartial", false) - if segmentSize <= 0 || segmentSize > maxPatternedTransformSegmentSize { - return r.jsError("segmentSize must be between 1 and %d bytes", maxPatternedTransformSegmentSize) - } - if transformEvery <= 0 { - return r.jsError("transformEvery must be greater than zero") - } - if transformOffset < 0 || transformOffset >= transformEvery { - return r.jsError("transformOffset must be between 0 and transformEvery - 1") - } - if bufferSize < segmentSize { - bufferSize = segmentSize - } - if bufferSize > maxPatternedTransformBufferSize { - bufferSize = maxPatternedTransformBufferSize - } - bufferSize -= bufferSize % segmentSize - if bufferSize == 0 { - bufferSize = segmentSize - } - - block, err := newRuntimeBlockCipher(parsedCipher) - if err != nil { - return r.jsError("%s", err.Error()) - } - if len(parsedCipher.IV) != block.BlockSize() { - return r.jsError("iv must be %d bytes for %s", block.BlockSize(), parsedCipher.Algorithm) - } - if parsedCipher.Mode == "cbc" && segmentSize%int64(block.BlockSize()) != 0 { - return r.jsError("segmentSize must be a multiple of %d bytes for CBC", block.BlockSize()) - } - - var onProgress goja.Callable - if len(call.Arguments) > 3 && !goja.IsUndefined(call.Arguments[3]) && !goja.IsNull(call.Arguments[3]) { - callback, ok := goja.AssertFunction(call.Arguments[3]) - if !ok { - return r.jsError("progress callback must be a function") - } - onProgress = callback - } - - unlock := lockDownloadOutputPath(outputPath) - defer unlock() - - input, err := os.Open(inputPath) - if err != nil { - return r.jsError("failed to open input file: %v", err) - } - info, err := input.Stat() - if err != nil { - input.Close() - return r.jsError("failed to stat input file: %v", err) - } - totalSize := info.Size() - - if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil { - input.Close() - return r.jsError("failed to create output directory: %v", err) - } - stagedPath := outputPath + ".transform.partial" - if filepath.Clean(stagedPath) == filepath.Clean(inputPath) { - input.Close() - return r.jsError("input path conflicts with transform staging path") - } - _ = os.Remove(stagedPath) - output, err := os.OpenFile(stagedPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) - if err != nil { - input.Close() - return r.jsError("failed to create staged output: %v", err) - } - - cleanup := func() { - input.Close() - output.Close() - _ = os.Remove(stagedPath) - } - ctx := r.activeOperationContext(context.Background()) - buffer := make([]byte, int(bufferSize)) - processed := int64(0) - segmentIndex := int64(0) - segmentsTransformed := int64(0) - - for { - if err := ctx.Err(); err != nil { - cleanup() - return r.jsError("patterned file transform cancelled: %v", err) - } - readCount, readErr := io.ReadFull(input, buffer) - if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF { - cleanup() - return r.jsError("failed to read input file: %v", readErr) - } - if readCount == 0 { - break - } - - chunk := buffer[:readCount] - for offset := 0; offset < readCount; offset += int(segmentSize) { - end := min(offset+int(segmentSize), readCount) - segment := chunk[offset:end] - selected := segmentIndex%transformEvery == transformOffset - fullSegment := len(segment) == int(segmentSize) - if selected && (fullSegment || transformPartial) { - if parsedCipher.Mode == "cbc" && len(segment)%block.BlockSize() != 0 { - cleanup() - return r.jsError("selected segment %d is not a multiple of %d bytes", segmentIndex, block.BlockSize()) - } - transformPatternedSegment(block, parsedCipher, operation, segment) - segmentsTransformed++ - } - segmentIndex++ - } - - written, writeErr := output.Write(chunk) - if writeErr != nil || written != len(chunk) { - if writeErr == nil { - writeErr = io.ErrShortWrite - } - cleanup() - return r.jsError("failed to write transformed file: %v", writeErr) - } - processed += int64(written) - if onProgress != nil { - if _, callbackErr := onProgress( - goja.Undefined(), - r.vm.ToValue(processed), - r.vm.ToValue(totalSize), - ); callbackErr != nil { - cleanup() - return r.jsError("progress callback failed: %v", callbackErr) - } - } - if readErr == io.EOF || readErr == io.ErrUnexpectedEOF { - break - } - } - - if err := input.Close(); err != nil { - output.Close() - _ = os.Remove(stagedPath) - return r.jsError("failed to close input file: %v", err) - } - if err := output.Sync(); err != nil { - output.Close() - _ = os.Remove(stagedPath) - return r.jsError("failed to sync transformed file: %v", err) - } - if err := output.Close(); err != nil { - _ = os.Remove(stagedPath) - return r.jsError("failed to close transformed file: %v", err) - } - if err := os.Rename(stagedPath, outputPath); err != nil { - _ = os.Remove(stagedPath) - return r.jsError("failed to publish transformed file: %v", err) - } - - return r.jsSuccess(map[string]any{ - "path": outputPath, - "bytes_processed": processed, - "segments_processed": segmentIndex, - "segments_transformed": segmentsTransformed, - }) -} - -func transformPatternedSegment( - block cipher.Block, - options *runtimeBlockCipherOptions, - operation string, - segment []byte, -) { - if options.Mode == "ctr" { - cipher.NewCTR(block, options.IV).XORKeyStream(segment, segment) - return - } - if operation == "encrypt" { - cipher.NewCBCEncrypter(block, options.IV).CryptBlocks(segment, segment) - return - } - cipher.NewCBCDecrypter(block, options.IV).CryptBlocks(segment, segment) -} diff --git a/go_backend/extension_runtime_http.go b/go_backend/extension_runtime_http.go deleted file mode 100644 index 32d0b811..00000000 --- a/go_backend/extension_runtime_http.go +++ /dev/null @@ -1,303 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - - "github.com/dop251/goja" -) - -type HTTPResponse struct { - StatusCode int `json:"statusCode"` - Body string `json:"body"` - Headers map[string]string `json:"headers"` -} - -const maxExtensionHTTPResponseBytes = 16 << 20 - -func readExtensionHTTPResponseBody(resp *http.Response) ([]byte, error) { - body, err := io.ReadAll( - io.LimitReader(resp.Body, maxExtensionHTTPResponseBytes+1), - ) - if err != nil { - return nil, err - } - if len(body) > maxExtensionHTTPResponseBytes { - return nil, fmt.Errorf( - "response body exceeds %d byte limit; use file.download for large media", - maxExtensionHTTPResponseBytes, - ) - } - return body, nil -} - -func setDefaultExtensionUA(req *http.Request) { - if req.Header.Get("User-Agent") == "" { - req.Header.Set("User-Agent", "Spotiflac-Extension/1.0") - } -} - -func (r *extensionRuntime) validateDomain(urlStr string) error { - parsed, err := url.Parse(urlStr) - if err != nil { - return fmt.Errorf("invalid URL: %w", err) - } - - if parsed.Scheme == "" { - return fmt.Errorf("invalid URL: scheme is required") - } - if parsed.Scheme != "https" && - !(parsed.Scheme == "http" && r.manifest.Permissions.AllowHTTP) { - return fmt.Errorf("network access denied: only https is allowed") - } - if parsed.User != nil { - return fmt.Errorf("invalid URL: embedded credentials are not allowed") - } - - domain := parsed.Hostname() - if domain == "" { - return fmt.Errorf("invalid URL: hostname is required") - } - - // Hostname answers are filtered and pinned by transportDialContext. Avoid a - // second net.LookupIP here: it was uncancellable and the answer was discarded - // before the transport resolved the same host again. - if isPrivateIPLiteralOrLocal(domain) { - return fmt.Errorf("network access denied: private/local network '%s' not allowed", domain) - } - - if !r.manifest.IsDomainAllowed(domain) { - return fmt.Errorf("network access denied: domain '%s' not in allowed list", domain) - } - - return nil -} - -// parseGojaHeaders converts an exported goja value (expected map[string]any) -// into string headers. Non-map values yield an empty map. -func parseGojaHeaders(v any) map[string]string { - headers := make(map[string]string) - if h, ok := v.(map[string]any); ok { - for k, val := range h { - headers[k] = fmt.Sprintf("%v", val) - } - } - return headers -} - -// coerceExportedBody stringifies a request body already exported from goja: -// strings pass through, maps/arrays are JSON-encoded, anything else is %v. -func coerceExportedBody(v any) (string, error) { - switch b := v.(type) { - case string: - return b, nil - case map[string]any, []any: - jsonBytes, err := json.Marshal(b) - if err != nil { - return "", fmt.Errorf("failed to stringify body: %v", err) - } - return string(jsonBytes), nil - default: - return fmt.Sprintf("%v", b), nil - } -} - -// coerceGojaBody is coerceExportedBody for a raw goja argument; undefined/null -// yield "", and the fallback uses goja's own String() conversion. -func coerceGojaBody(v goja.Value) (string, error) { - if v == nil || goja.IsUndefined(v) || goja.IsNull(v) { - return "", nil - } - switch b := v.Export().(type) { - case string: - return b, nil - case map[string]any, []any: - return coerceExportedBody(b) - default: - return v.String(), nil - } -} - -func flattenHTTPHeaders(h http.Header) map[string]any { - flat := make(map[string]any, len(h)) - for k, v := range h { - if len(v) == 1 { - flat[k] = v[0] - } else { - flat[k] = v - } - } - return flat -} - -// checkExtensionURL extracts and allowlist-validates the URL argument. -// On failure it returns a non-nil error value to hand back to JS. -func (r *extensionRuntime) checkExtensionURL(call goja.FunctionCall) (string, goja.Value) { - if len(call.Arguments) < 1 { - return "", r.vm.ToValue(map[string]any{ - "error": "URL is required", - }) - } - urlStr := call.Arguments[0].String() - if err := r.validateDomain(urlStr); err != nil { - GoLog("[Extension:%s] HTTP blocked: %v\n", r.extensionID, err) - return "", r.vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - return urlStr, nil -} - -// doExtensionHTTP builds and executes the request, returning the extension -// response map (or {"error": ...}). defaultJSON sets Content-Type -// application/json when the caller did not provide one. -func (r *extensionRuntime) doExtensionHTTP(method, urlStr string, body io.Reader, defaultJSON bool, headers map[string]string) goja.Value { - req, err := http.NewRequest(method, urlStr, body) - if err != nil { - return r.vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - req = r.bindDownloadCancelContext(req) - - for k, v := range headers { - req.Header.Set(k, v) - } - setDefaultExtensionUA(req) - if defaultJSON && req.Header.Get("Content-Type") == "" { - req.Header.Set("Content-Type", "application/json") - } - - resp, err := r.httpClient.Do(req) - if err != nil { - return r.vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - defer resp.Body.Close() - - respBody, err := readExtensionHTTPResponseBody(resp) - if err != nil { - return r.vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - - return r.vm.ToValue(map[string]any{ - "statusCode": resp.StatusCode, - "status": resp.StatusCode, - "ok": resp.StatusCode >= 200 && resp.StatusCode < 300, - "url": resp.Request.URL.String(), - "body": string(respBody), - "headers": flattenHTTPHeaders(resp.Header), - }) -} - -func (r *extensionRuntime) httpGet(call goja.FunctionCall) goja.Value { - urlStr, errVal := r.checkExtensionURL(call) - if errVal != nil { - return errVal - } - headers := parseGojaHeaders(call.Argument(1).Export()) - return r.doExtensionHTTP("GET", urlStr, nil, false, headers) -} - -func (r *extensionRuntime) httpPost(call goja.FunctionCall) goja.Value { - urlStr, errVal := r.checkExtensionURL(call) - if errVal != nil { - return errVal - } - bodyStr, err := coerceGojaBody(call.Argument(1)) - if err != nil { - return r.vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - headers := parseGojaHeaders(call.Argument(2).Export()) - // POST always sends a (possibly empty) body and defaults Content-Type. - return r.doExtensionHTTP("POST", urlStr, strings.NewReader(bodyStr), true, headers) -} - -func (r *extensionRuntime) httpRequest(call goja.FunctionCall) goja.Value { - urlStr, errVal := r.checkExtensionURL(call) - if errVal != nil { - return errVal - } - - method := "GET" - var bodyStr string - var headers map[string]string - - if opts, ok := call.Argument(1).Export().(map[string]any); ok { - if m, ok := opts["method"].(string); ok { - method = strings.ToUpper(m) - } - if bodyArg, ok := opts["body"]; ok && bodyArg != nil { - var err error - if bodyStr, err = coerceExportedBody(bodyArg); err != nil { - return r.vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - } - headers = parseGojaHeaders(opts["headers"]) - } - - var reqBody io.Reader - if bodyStr != "" { - reqBody = strings.NewReader(bodyStr) - } - return r.doExtensionHTTP(method, urlStr, reqBody, bodyStr != "", headers) -} - -func (r *extensionRuntime) httpPut(call goja.FunctionCall) goja.Value { - return r.httpMethodShortcut("PUT", call) -} - -func (r *extensionRuntime) httpDelete(call goja.FunctionCall) goja.Value { - return r.httpMethodShortcut("DELETE", call) -} - -func (r *extensionRuntime) httpPatch(call goja.FunctionCall) goja.Value { - return r.httpMethodShortcut("PATCH", call) -} - -func (r *extensionRuntime) httpMethodShortcut(method string, call goja.FunctionCall) goja.Value { - urlStr, errVal := r.checkExtensionURL(call) - if errVal != nil { - return errVal - } - - // DELETE takes (url, headers); other methods take (url, body, headers). - var bodyStr string - headerArg := 1 - if method != "DELETE" { - var err error - if bodyStr, err = coerceGojaBody(call.Argument(1)); err != nil { - return r.vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - headerArg = 2 - } - headers := parseGojaHeaders(call.Argument(headerArg).Export()) - - var reqBody io.Reader - if bodyStr != "" { - reqBody = strings.NewReader(bodyStr) - } - return r.doExtensionHTTP(method, urlStr, reqBody, bodyStr != "", headers) -} - -func (r *extensionRuntime) httpClearCookies(call goja.FunctionCall) goja.Value { - if jar, ok := r.cookieJar.(*simpleCookieJar); ok { - jar.Clear() - GoLog("[Extension:%s] Cookies cleared\n", r.extensionID) - return r.vm.ToValue(true) - } - return r.vm.ToValue(false) -} diff --git a/go_backend/extension_runtime_matching.go b/go_backend/extension_runtime_matching.go deleted file mode 100644 index 883ee9cf..00000000 --- a/go_backend/extension_runtime_matching.go +++ /dev/null @@ -1,133 +0,0 @@ -package gobackend - -import ( - "strings" - - "github.com/dop251/goja" -) - -func (r *extensionRuntime) matchingCompareStrings(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.vm.ToValue(0.0) - } - - str1 := strings.ToLower(strings.TrimSpace(call.Arguments[0].String())) - str2 := strings.ToLower(strings.TrimSpace(call.Arguments[1].String())) - - if str1 == str2 { - return r.vm.ToValue(1.0) - } - - similarity := calculateStringSimilarity(str1, str2) - return r.vm.ToValue(similarity) -} - -func (r *extensionRuntime) matchingCompareDuration(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.vm.ToValue(false) - } - - dur1 := int(call.Arguments[0].ToInteger()) - dur2 := int(call.Arguments[1].ToInteger()) - - tolerance := 3000 - if len(call.Arguments) > 2 && !goja.IsUndefined(call.Arguments[2]) { - tolerance = int(call.Arguments[2].ToInteger()) - } - - diff := dur1 - dur2 - if diff < 0 { - diff = -diff - } - - return r.vm.ToValue(diff <= tolerance) -} - -func (r *extensionRuntime) matchingNormalizeString(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue("") - } - - str := call.Arguments[0].String() - normalized := normalizeStringForMatching(str) - return r.vm.ToValue(normalized) -} - -func calculateStringSimilarity(s1, s2 string) float64 { - if len(s1) == 0 && len(s2) == 0 { - return 1.0 - } - if len(s1) == 0 || len(s2) == 0 { - return 0.0 - } - - distance := levenshteinDistance(s1, s2) - maxLen := len(s1) - if len(s2) > maxLen { - maxLen = len(s2) - } - - return 1.0 - float64(distance)/float64(maxLen) -} - -func levenshteinDistance(s1, s2 string) int { - if len(s1) == 0 { - return len(s2) - } - if len(s2) == 0 { - return len(s1) - } - - matrix := make([][]int, len(s1)+1) - for i := range matrix { - matrix[i] = make([]int, len(s2)+1) - matrix[i][0] = i - } - for j := range matrix[0] { - matrix[0][j] = j - } - - for i := 1; i <= len(s1); i++ { - for j := 1; j <= len(s2); j++ { - cost := 1 - if s1[i-1] == s2[j-1] { - cost = 0 - } - matrix[i][j] = min( - matrix[i-1][j]+1, - matrix[i][j-1]+1, - matrix[i-1][j-1]+cost, - ) - } - } - - return matrix[len(s1)][len(s2)] -} - -func normalizeStringForMatching(s string) string { - s = strings.ToLower(s) - - suffixes := []string{ - " (remastered)", " (remaster)", " - remastered", " - remaster", - " (deluxe)", " (deluxe edition)", " - deluxe", " - deluxe edition", - " (explicit)", " (clean)", " [explicit]", " [clean]", - " (album version)", " (single version)", " (radio edit)", - " (feat.", " (ft.", " feat.", " ft.", - } - for _, suffix := range suffixes { - if idx := strings.Index(s, suffix); idx != -1 { - s = s[:idx] - } - } - - var result strings.Builder - for _, r := range s { - if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == ' ' { - result.WriteRune(r) - } - } - - s = strings.Join(strings.Fields(result.String()), " ") - - return strings.TrimSpace(s) -} diff --git a/go_backend/extension_runtime_polyfills.go b/go_backend/extension_runtime_polyfills.go deleted file mode 100644 index 404a1e03..00000000 --- a/go_backend/extension_runtime_polyfills.go +++ /dev/null @@ -1,412 +0,0 @@ -package gobackend - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - - "github.com/dop251/goja" -) - -func (r *extensionRuntime) fetchPolyfill(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.createFetchError("URL is required") - } - - urlStr := call.Arguments[0].String() - if err := r.validateDomain(urlStr); err != nil { - GoLog("[Extension:%s] fetch blocked: %v\n", r.extensionID, err) - return r.createFetchError(err.Error()) - } - - method := "GET" - var bodyStr string - var headers map[string]string - - if opts, ok := call.Argument(1).Export().(map[string]any); ok { - if m, ok := opts["method"].(string); ok { - method = strings.ToUpper(m) - } - if bodyArg, ok := opts["body"]; ok && bodyArg != nil { - var err error - if bodyStr, err = coerceExportedBody(bodyArg); err != nil { - return r.createFetchError(err.Error()) - } - } - headers = parseGojaHeaders(opts["headers"]) - } - - var reqBody io.Reader - if bodyStr != "" { - reqBody = strings.NewReader(bodyStr) - } - - req, err := http.NewRequest(method, urlStr, reqBody) - if err != nil { - return r.createFetchError(err.Error()) - } - req = r.bindDownloadCancelContext(req) - - for k, v := range headers { - req.Header.Set(k, v) - } - if req.Header.Get("User-Agent") == "" { - req.Header.Set("User-Agent", appUserAgent()) - } - if bodyStr != "" && req.Header.Get("Content-Type") == "" { - req.Header.Set("Content-Type", "application/json") - } - - resp, err := r.httpClient.Do(req) - if err != nil { - return r.createFetchError(err.Error()) - } - defer resp.Body.Close() - - body, err := readExtensionHTTPResponseBody(resp) - if err != nil { - return r.createFetchError(err.Error()) - } - - respHeaders := flattenHTTPHeaders(resp.Header) - - responseObj := r.vm.NewObject() - responseObj.Set("ok", resp.StatusCode >= 200 && resp.StatusCode < 300) - responseObj.Set("status", resp.StatusCode) - responseObj.Set("statusText", http.StatusText(resp.StatusCode)) - responseObj.Set("headers", respHeaders) - responseObj.Set("url", resp.Request.URL.String()) - - var bodyString string - var bodyStringReady bool - responseObj.Set("text", func(call goja.FunctionCall) goja.Value { - // Avoid allocating a second full response copy when callers only use - // json() or arrayBuffer(). - if !bodyStringReady { - bodyString = string(body) - bodyStringReady = true - } - return r.vm.ToValue(bodyString) - }) - - responseObj.Set("json", func(call goja.FunctionCall) goja.Value { - var result any - if err := json.Unmarshal(body, &result); err != nil { - GoLog("[Extension:%s] fetch json() parse error: %v\n", r.extensionID, err) - return goja.Undefined() - } - return r.vm.ToValue(result) - }) - - responseObj.Set("arrayBuffer", func(call goja.FunctionCall) goja.Value { - // A Go-backed byte slice preserves the existing synchronous, array-like - // contract (length and numeric indexes) without one interface allocation - // per byte. - return r.vm.ToValue(body) - }) - - return responseObj -} - -func (r *extensionRuntime) createFetchError(message string) goja.Value { - errorObj := r.vm.NewObject() - errorObj.Set("ok", false) - errorObj.Set("status", 0) - errorObj.Set("statusText", "Network Error") - errorObj.Set("error", message) - errorObj.Set("text", func(call goja.FunctionCall) goja.Value { - return r.vm.ToValue("") - }) - errorObj.Set("json", func(call goja.FunctionCall) goja.Value { - return goja.Undefined() - }) - return errorObj -} - -func (r *extensionRuntime) atobPolyfill(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue("") - } - input := call.Arguments[0].String() - decoded, err := base64.StdEncoding.DecodeString(input) - if err != nil { - decoded, err = base64.URLEncoding.DecodeString(input) - if err != nil { - GoLog("[Extension:%s] atob decode error: %v\n", r.extensionID, err) - return r.vm.ToValue("") - } - } - return r.vm.ToValue(string(decoded)) -} - -func (r *extensionRuntime) btoaPolyfill(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue("") - } - input := call.Arguments[0].String() - return r.vm.ToValue(base64.StdEncoding.EncodeToString([]byte(input))) -} - -func (r *extensionRuntime) registerTextEncoderDecoder(vm *goja.Runtime) { - vm.Set("TextEncoder", func(call goja.ConstructorCall) *goja.Object { - encoder := call.This - encoder.Set("encoding", "utf-8") - - encoder.Set("encode", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return vm.ToValue([]byte{}) - } - input := call.Arguments[0].String() - bytes := []byte(input) - - result := make([]any, len(bytes)) - for i, b := range bytes { - result[i] = int(b) - } - return vm.ToValue(result) - }) - - encoder.Set("encodeInto", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return vm.ToValue(map[string]any{"read": 0, "written": 0}) - } - input := call.Arguments[0].String() - return vm.ToValue(map[string]any{ - "read": len(input), - "written": len([]byte(input)), - }) - }) - - return nil - }) - - vm.Set("TextDecoder", func(call goja.ConstructorCall) *goja.Object { - decoder := call.This - - encoding := "utf-8" - if len(call.Arguments) > 0 && !goja.IsUndefined(call.Arguments[0]) { - encoding = call.Arguments[0].String() - } - decoder.Set("encoding", encoding) - decoder.Set("fatal", false) - decoder.Set("ignoreBOM", false) - - decoder.Set("decode", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return vm.ToValue("") - } - - input := call.Arguments[0].Export() - var bytes []byte - - switch v := input.(type) { - case []byte: - bytes = v - case []any: - bytes = make([]byte, len(v)) - for i, val := range v { - switch n := val.(type) { - case int64: - bytes[i] = byte(n) - case float64: - bytes[i] = byte(n) - case int: - bytes[i] = byte(n) - } - } - case string: - return vm.ToValue(v) - default: - return vm.ToValue("") - } - - return vm.ToValue(string(bytes)) - }) - - return nil - }) -} - -func (r *extensionRuntime) registerURLClass(vm *goja.Runtime) { - vm.Set("URL", func(call goja.ConstructorCall) *goja.Object { - urlObj := call.This - - if len(call.Arguments) < 1 { - urlObj.Set("href", "") - return nil - } - - urlStr := call.Arguments[0].String() - - if len(call.Arguments) > 1 && !goja.IsUndefined(call.Arguments[1]) { - baseStr := call.Arguments[1].String() - baseURL, err := url.Parse(baseStr) - if err == nil { - relURL, err := url.Parse(urlStr) - if err == nil { - urlStr = baseURL.ResolveReference(relURL).String() - } - } - } - - parsed, err := url.Parse(urlStr) - if err != nil { - urlObj.Set("href", urlStr) - return nil - } - - urlObj.Set("href", parsed.String()) - urlObj.Set("protocol", parsed.Scheme+":") - urlObj.Set("host", parsed.Host) - urlObj.Set("hostname", parsed.Hostname()) - urlObj.Set("port", parsed.Port()) - urlObj.Set("pathname", parsed.Path) - urlObj.Set("search", "") - if parsed.RawQuery != "" { - urlObj.Set("search", "?"+parsed.RawQuery) - } - urlObj.Set("hash", "") - if parsed.Fragment != "" { - urlObj.Set("hash", "#"+parsed.Fragment) - } - urlObj.Set("origin", parsed.Scheme+"://"+parsed.Host) - urlObj.Set("username", parsed.User.Username()) - password, _ := parsed.User.Password() - urlObj.Set("password", password) - - queryValues := parsed.Query() - - searchParams := vm.NewObject() - searchParams.Set("get", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return goja.Null() - } - key := call.Arguments[0].String() - if val := queryValues.Get(key); val != "" { - return vm.ToValue(val) - } - return goja.Null() - }) - - searchParams.Set("getAll", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return vm.ToValue([]string{}) - } - key := call.Arguments[0].String() - return vm.ToValue(queryValues[key]) - }) - - searchParams.Set("has", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return vm.ToValue(false) - } - key := call.Arguments[0].String() - return vm.ToValue(queryValues.Has(key)) - }) - - searchParams.Set("toString", func(call goja.FunctionCall) goja.Value { - return vm.ToValue(queryValues.Encode()) - }) - - urlObj.Set("searchParams", searchParams) - - urlObj.Set("toString", func(call goja.FunctionCall) goja.Value { - return vm.ToValue(parsed.String()) - }) - - urlObj.Set("toJSON", func(call goja.FunctionCall) goja.Value { - return vm.ToValue(parsed.String()) - }) - - return nil - }) - - vm.Set("URLSearchParams", func(call goja.ConstructorCall) *goja.Object { - paramsObj := call.This - values := url.Values{} - - if len(call.Arguments) > 0 && !goja.IsUndefined(call.Arguments[0]) { - init := call.Arguments[0].Export() - switch v := init.(type) { - case string: - parsed, _ := url.ParseQuery(strings.TrimPrefix(v, "?")) - values = parsed - case map[string]any: - for k, val := range v { - values.Set(k, fmt.Sprintf("%v", val)) - } - } - } - - paramsObj.Set("append", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) >= 2 { - values.Add(call.Arguments[0].String(), call.Arguments[1].String()) - } - return goja.Undefined() - }) - - paramsObj.Set("delete", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) >= 1 { - values.Del(call.Arguments[0].String()) - } - return goja.Undefined() - }) - - paramsObj.Set("get", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return goja.Null() - } - if val := values.Get(call.Arguments[0].String()); val != "" { - return vm.ToValue(val) - } - return goja.Null() - }) - - paramsObj.Set("getAll", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return vm.ToValue([]string{}) - } - return vm.ToValue(values[call.Arguments[0].String()]) - }) - - paramsObj.Set("has", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return vm.ToValue(false) - } - return vm.ToValue(values.Has(call.Arguments[0].String())) - }) - - paramsObj.Set("set", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) >= 2 { - values.Set(call.Arguments[0].String(), call.Arguments[1].String()) - } - return goja.Undefined() - }) - - paramsObj.Set("toString", func(call goja.FunctionCall) goja.Value { - return vm.ToValue(values.Encode()) - }) - - return nil - }) -} - -func (r *extensionRuntime) registerJSONGlobal(vm *goja.Runtime) { - jsonScript := ` - if (typeof JSON === 'undefined') { - var JSON = { - parse: function(text) { - return utils.parseJSON(text); - }, - stringify: function(value, replacer, space) { - return utils.stringifyJSON(value); - } - }; - } - ` - _, _ = vm.RunString(jsonScript) -} diff --git a/go_backend/extension_runtime_segments.go b/go_backend/extension_runtime_segments.go deleted file mode 100644 index f41cd0c4..00000000 --- a/go_backend/extension_runtime_segments.go +++ /dev/null @@ -1,749 +0,0 @@ -package gobackend - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/dop251/goja" -) - -type segmentTransferSpec struct { - Index int - URL string - Headers map[string]string -} - -type segmentTransferResult struct { - Index int - Path string - Size int64 - Attempts int - Failure *transferFailure -} - -type segmentTransferCheckpoint struct { - Version int `json:"version"` - Fingerprint string `json:"fingerprint"` - NextIndex int `json:"next_index"` - Bytes int64 `json:"bytes"` - UpdatedAt int64 `json:"updated_at"` -} - -func parseStringHeaders(value any) map[string]string { - raw, ok := value.(map[string]any) - if !ok { - return nil - } - headers := make(map[string]string, len(raw)) - for key, entry := range raw { - headers[key] = fmt.Sprintf("%v", entry) - } - return headers -} - -func mergeStringHeaders(base, override map[string]string) map[string]string { - if len(base) == 0 && len(override) == 0 { - return nil - } - merged := make(map[string]string, len(base)+len(override)) - for key, value := range base { - merged[key] = value - } - for key, value := range override { - merged[key] = value - } - return merged -} - -func parseSegmentTransferSpecs(value any, commonHeaders map[string]string) ([]segmentTransferSpec, error) { - rawSegments, ok := value.([]any) - if !ok || len(rawSegments) == 0 { - return nil, fmt.Errorf("segments must be a non-empty array") - } - segments := make([]segmentTransferSpec, 0, len(rawSegments)) - for index, raw := range rawSegments { - var rawURL string - var headers map[string]string - switch typed := raw.(type) { - case string: - rawURL = typed - case map[string]any: - rawURL, _ = typed["url"].(string) - headers = parseStringHeaders(typed["headers"]) - default: - return nil, fmt.Errorf("segment %d must be a URL string or object", index) - } - rawURL = strings.TrimSpace(rawURL) - if rawURL == "" { - return nil, fmt.Errorf("segment %d URL is empty", index) - } - segments = append(segments, segmentTransferSpec{ - Index: index, - URL: rawURL, - Headers: mergeStringHeaders(commonHeaders, headers), - }) - } - return segments, nil -} - -func segmentListFingerprint(segments []segmentTransferSpec) string { - hash := sha256.New() - for _, segment := range segments { - // Segmented checkpoints have no per-segment ETag or Last-Modified - // validator. Include the complete URL, including its query, so a - // different media object served from the same CDN path can never reuse - // already-assembled bytes. Ordinary single-file checkpoints may ignore - // rotating query credentials because their validator still protects - // integrity. - hash.Write([]byte(segment.URL)) - hash.Write([]byte{0}) - } - return hex.EncodeToString(hash.Sum(nil)) -} - -func loadSegmentCheckpoint(path, fingerprint string) (segmentTransferCheckpoint, bool) { - var checkpoint segmentTransferCheckpoint - data, err := os.ReadFile(path) - if err != nil || json.Unmarshal(data, &checkpoint) != nil { - return segmentTransferCheckpoint{}, false - } - if checkpoint.Version != transferCheckpointVersion || - checkpoint.Fingerprint != fingerprint || - checkpoint.NextIndex < 0 || checkpoint.Bytes < 0 || - (checkpoint.NextIndex == 0 && checkpoint.Bytes != 0) || - (checkpoint.NextIndex > 0 && checkpoint.Bytes == 0) { - return segmentTransferCheckpoint{}, false - } - return checkpoint, true -} - -func saveSegmentCheckpoint(path string, checkpoint segmentTransferCheckpoint) error { - checkpoint.Version = transferCheckpointVersion - checkpoint.UpdatedAt = time.Now().UnixMilli() - data, err := json.Marshal(checkpoint) - if err != nil { - return err - } - tempPath := path + ".tmp" - file, err := os.OpenFile(tempPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) - if err != nil { - return err - } - if _, err = file.Write(data); err == nil { - err = file.Sync() - } - closeErr := file.Close() - if err == nil { - err = closeErr - } - if err != nil { - os.Remove(tempPath) - return err - } - if err := os.Rename(tempPath, path); err != nil { - os.Remove(tempPath) - return err - } - return nil -} - -func segmentTempPath(stagedPath string, index int) string { - return fmt.Sprintf("%s.segment.%06d", stagedPath, index) -} - -func (r *extensionRuntime) fetchSegmentToTemp( - ctx context.Context, - client *http.Client, - spec segmentTransferSpec, - tempPath string, - policy DownloadTransferPolicy, - received *atomic.Int64, - completed *atomic.Bool, - itemProgressReporter *ItemTransferProgressReporter, -) segmentTransferResult { - config := transferRetryConfig(policy) - retryDelay := config.InitialDelay - var lastFailure transferFailure - - for attempt := 1; attempt <= policy.MaxAttempts; attempt++ { - if ctx.Err() != nil { - return segmentTransferResult{ - Index: spec.Index, - Failure: &transferFailure{ - ErrorType: "cancelled", - Message: "download cancelled", - Attempts: attempt, - }, - } - } - os.Remove(tempPath) - output, err := os.OpenFile(tempPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) - if err != nil { - return segmentTransferResult{ - Index: spec.Index, - Failure: &transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to create segment file: %v", err), - Attempts: attempt, - }, - } - } - - req, err := http.NewRequestWithContext(ctx, "GET", spec.URL, nil) - if err != nil { - output.Close() - return segmentTransferResult{ - Index: spec.Index, - Failure: &transferFailure{ - ErrorType: "invalid_request", - Message: err.Error(), - Attempts: attempt, - }, - } - } - for key, value := range spec.Headers { - req.Header.Set(key, value) - } - if req.Header.Get("User-Agent") == "" { - req.Header.Set("User-Agent", appUserAgent()) - } - req, watchdog := bindStallWatchdog(req, downloadStallTimeout) - resp, err := r.doResolutionTransfer(client, req, attempt == 1 && completed.Load()) - if err != nil { - stalled := watchdog.stalled.Load() - watchdog.stop() - output.Close() - message := err.Error() - if stalled { - message = fmt.Sprintf( - "segment %d stalled for %ds", - spec.Index, - int(downloadStallTimeout.Seconds()), - ) - } - lastFailure = transferFailure{ - ErrorType: "transient_network", - Message: message, - Attempts: attempt, - } - if attempt == policy.MaxAttempts || ctx.Err() != nil { - if ctx.Err() != nil { - lastFailure.ErrorType = "cancelled" - lastFailure.Message = "download cancelled" - } - return segmentTransferResult{Index: spec.Index, Failure: &lastFailure} - } - if r.waitResolutionRetry(ctx, retryDelay) != nil { - lastFailure.ErrorType = "cancelled" - lastFailure.Message = "download cancelled" - return segmentTransferResult{Index: spec.Index, Failure: &lastFailure} - } - retryDelay = calculateNextDelay(retryDelay, config) - continue - } - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - retryAfter := retryAfterSeconds(resp) - io.Copy(io.Discard, io.LimitReader(resp.Body, 32*1024)) - resp.Body.Close() - watchdog.stop() - output.Close() - lastFailure = transferFailure{ - ErrorType: transferErrorTypeForStatus(resp.StatusCode, policy), - Message: fmt.Sprintf("segment %d HTTP error: %d", spec.Index, resp.StatusCode), - HTTPStatus: resp.StatusCode, - RetryAfterSeconds: retryAfter, - Attempts: attempt, - } - if !retryableTransferStatus(resp.StatusCode) || attempt == policy.MaxAttempts { - return segmentTransferResult{Index: spec.Index, Failure: &lastFailure} - } - delay := retryDelay - if retryAfter > 0 { - delay = time.Duration(retryAfter) * time.Second - } - if r.waitResolutionRetry(ctx, delay) != nil { - lastFailure.ErrorType = "cancelled" - lastFailure.Message = "download cancelled" - return segmentTransferResult{Index: spec.Index, Failure: &lastFailure} - } - retryDelay = calculateNextDelay(retryDelay, config) - continue - } - - buffer := make([]byte, 64*1024) - var size int64 - var readErr error - for { - readCount, bodyErr := resp.Body.Read(buffer) - if readCount > 0 { - watchdog.reset() - writeCount, writeErr := output.Write(buffer[:readCount]) - size += int64(writeCount) - received.Add(int64(writeCount)) - itemProgressReporter.Report(received.Load(), 0) - if writeErr != nil || writeCount != readCount { - if writeErr == nil { - writeErr = io.ErrShortWrite - } - readErr = writeErr - break - } - } - if bodyErr != nil { - if bodyErr != io.EOF { - readErr = bodyErr - } - break - } - } - resp.Body.Close() - stalled := watchdog.stalled.Load() - watchdog.stop() - closeErr := output.Close() - if readErr == nil { - readErr = closeErr - } - if readErr == nil && resp.ContentLength > 0 && size != resp.ContentLength { - readErr = io.ErrUnexpectedEOF - } - if readErr == nil && size > 0 { - completed.Store(true) - return segmentTransferResult{ - Index: spec.Index, - Path: tempPath, - Size: size, - Attempts: attempt, - } - } - if size > 0 { - received.Add(-size) - } - - message := fmt.Sprintf("failed to read segment %d: %v", spec.Index, readErr) - if size == 0 && readErr == nil { - message = fmt.Sprintf("segment %d response was empty", spec.Index) - } - if stalled { - message = fmt.Sprintf( - "segment %d stalled for %ds", - spec.Index, - int(downloadStallTimeout.Seconds()), - ) - } - lastFailure = transferFailure{ - ErrorType: "transient_network", - Message: message, - Attempts: attempt, - } - if attempt == policy.MaxAttempts { - return segmentTransferResult{Index: spec.Index, Failure: &lastFailure} - } - if r.waitResolutionRetry(ctx, retryDelay) != nil { - lastFailure.ErrorType = "cancelled" - lastFailure.Message = "download cancelled" - return segmentTransferResult{Index: spec.Index, Failure: &lastFailure} - } - retryDelay = calculateNextDelay(retryDelay, config) - } - - return segmentTransferResult{ - Index: spec.Index, - Failure: &transferFailure{ - ErrorType: "transient_network", - Message: fmt.Sprintf("segment %d exhausted retry budget", spec.Index), - Attempts: policy.MaxAttempts, - }, - } -} - -func (r *extensionRuntime) fileDownloadSegments(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.jsTransferError(transferFailure{ - ErrorType: "invalid_request", - Message: "segments and output path are required", - }) - } - - var commonHeaders map[string]string - var onProgress goja.Callable - var maxParallelOption int - var maxAttemptsOption int - var persistentCheckpointOption *bool - if len(call.Arguments) > 2 && - !goja.IsUndefined(call.Arguments[2]) && - !goja.IsNull(call.Arguments[2]) { - if options, ok := call.Arguments[2].Export().(map[string]any); ok { - commonHeaders = parseStringHeaders(options["headers"]) - if progressValue, ok := options["onProgress"]; ok { - if callable, ok := goja.AssertFunction(r.vm.ToValue(progressValue)); ok { - onProgress = callable - } - } - maxParallelOption = capabilityInt(options["maxParallel"], 0) - maxAttemptsOption = capabilityInt(options["maxAttempts"], 0) - if checkpoint, ok := options["persistentCheckpoint"].(bool); ok { - persistentCheckpointOption = &checkpoint - } - } - } - - segments, err := parseSegmentTransferSpecs(call.Arguments[0].Export(), commonHeaders) - if err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "invalid_request", - Message: err.Error(), - }) - } - for _, segment := range segments { - if err := r.validateDomain(segment.URL); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "permission", - Message: err.Error(), - }) - } - } - fullPath, err := r.validatePath(call.Arguments[1].String()) - if err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "permission", - Message: err.Error(), - }) - } - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to create output directory: %v", err), - }) - } - - policy := r.manifest.DownloadTransferPolicy() - if maxParallelOption > 0 { - policy.MaxParallelSegments = clampInt(maxParallelOption, 1, maxParallelSegments) - } - if maxAttemptsOption > 0 { - policy.MaxAttempts = clampInt(maxAttemptsOption, 1, 8) - } - persistentCheckpoint := policy.PersistentCheckpoint - if persistentCheckpointOption != nil { - persistentCheckpoint = *persistentCheckpointOption - } - - client := r.downloadClient - if client == nil { - client = r.httpClient - } - unlock := lockDownloadOutputPath(fullPath) - defer unlock() - - stagedPath := stagedDownloadPath(fullPath) - checkpointPath := transferCheckpointPath(stagedPath) + ".segments" - fingerprint := segmentListFingerprint(segments) - checkpoint, checkpointOK := loadSegmentCheckpoint(checkpointPath, fingerprint) - if !persistentCheckpoint || !checkpointOK || checkpoint.NextIndex > len(segments) { - checkpoint = segmentTransferCheckpoint{Fingerprint: fingerprint} - checkpointOK = false - os.Remove(stagedPath) - os.Remove(checkpointPath) - } - - output, err := os.OpenFile(stagedPath, os.O_CREATE|os.O_RDWR, 0600) - if err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to create segmented output: %v", err), - }) - } - promoted := false - defer func() { - output.Close() - for index := range segments { - os.Remove(segmentTempPath(stagedPath, index)) - } - if promoted { - os.Remove(checkpointPath) - } else if !persistentCheckpoint { - os.Remove(stagedPath) - os.Remove(checkpointPath) - } - }() - - nextIndex := 0 - var totalWritten int64 - if checkpointOK { - if info, statErr := output.Stat(); statErr == nil && info.Size() >= checkpoint.Bytes { - totalWritten = checkpoint.Bytes - nextIndex = checkpoint.NextIndex - } else { - // Segment boundaries cannot be reconstructed from a shorter file. - // Restart instead of skipping segments named by a stale checkpoint. - checkpointOK = false - checkpoint = segmentTransferCheckpoint{Fingerprint: fingerprint} - os.Remove(checkpointPath) - } - } - if err := output.Truncate(totalWritten); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to restore segmented output: %v", err), - }) - } - if _, err := output.Seek(totalWritten, io.SeekStart); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to seek segmented output: %v", err), - }) - } - - activeItemID := r.getActiveDownloadItemID() - if activeItemID != "" { - SetItemDownloading(activeItemID) - SetItemProgress( - activeItemID, - float64(nextIndex)/float64(len(segments)), - totalWritten, - 0, - ) - } - if nextIndex == len(segments) && totalWritten > 0 { - if err := output.Sync(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to sync restored segmented output: %v", err), - }) - } - if err := output.Close(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to close restored segmented output: %v", err), - }) - } - if err := os.Rename(stagedPath, fullPath); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to publish restored segmented output: %v", err), - }) - } - promoted = true - os.Remove(checkpointPath) - syncDir(filepath.Dir(fullPath)) - if activeItemID != "" { - SetItemProgress(activeItemID, 1, totalWritten, totalWritten) - } - return r.jsSuccess(map[string]any{ - "path": fullPath, - "size": totalWritten, - "segments": len(segments), - "resumed": true, - }) - } - - baseRequest, requestErr := http.NewRequest("GET", segments[nextIndex].URL, nil) - if requestErr != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "invalid_request", - Message: requestErr.Error(), - }) - } - baseRequest = r.bindDownloadCancelContext(baseRequest) - ctx, cancel := context.WithCancel(baseRequest.Context()) - defer cancel() - - jobs := make(chan segmentTransferSpec) - results := make(chan segmentTransferResult, policy.MaxParallelSegments) - var received atomic.Int64 - var completed atomic.Bool - received.Store(totalWritten) - itemProgressReporter := NewItemTransferProgressReporter(activeItemID, totalWritten, 0) - var workers sync.WaitGroup - workerCount := min(policy.MaxParallelSegments, len(segments)-nextIndex) - for workerIndex := 0; workerIndex < workerCount; workerIndex++ { - workers.Add(1) - go func() { - defer workers.Done() - for spec := range jobs { - result := r.fetchSegmentToTemp( - ctx, - client, - spec, - segmentTempPath(stagedPath, spec.Index), - policy, - &received, - &completed, - itemProgressReporter, - ) - select { - case results <- result: - case <-ctx.Done(): - return - } - if result.Failure != nil { - return - } - } - }() - } - go func() { - defer close(jobs) - for index := nextIndex; index < len(segments); index++ { - select { - case jobs <- segments[index]: - case <-ctx.Done(): - return - } - } - }() - go func() { - workers.Wait() - close(results) - }() - - pending := make(map[int]segmentTransferResult) - completedSegments := nextIndex - lastCheckpointBytes := totalWritten - lastCheckpointAt := time.Now() - var firstFailure *transferFailure - for result := range results { - if result.Failure != nil { - if firstFailure == nil { - failureCopy := *result.Failure - firstFailure = &failureCopy - cancel() - } - continue - } - pending[result.Index] = result - for { - ready, ok := pending[nextIndex] - if !ok { - break - } - segmentFile, openErr := os.Open(ready.Path) - if openErr != nil { - firstFailure = &transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to open downloaded segment %d: %v", nextIndex, openErr), - Attempts: ready.Attempts, - } - cancel() - break - } - copied, copyErr := io.CopyBuffer(output, segmentFile, make([]byte, 128*1024)) - segmentFile.Close() - if copyErr != nil || copied != ready.Size { - if copyErr == nil { - copyErr = io.ErrShortWrite - } - firstFailure = &transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to append segment %d: %v", nextIndex, copyErr), - Attempts: ready.Attempts, - } - cancel() - break - } - totalWritten += copied - os.Remove(ready.Path) - delete(pending, nextIndex) - nextIndex++ - completedSegments++ - if persistentCheckpoint && - (totalWritten-lastCheckpointBytes >= transferCheckpointBytes || - time.Since(lastCheckpointAt) >= transferCheckpointPeriod) { - if output.Sync() == nil && saveSegmentCheckpoint( - checkpointPath, - segmentTransferCheckpoint{ - Fingerprint: fingerprint, - NextIndex: nextIndex, - Bytes: totalWritten, - }, - ) == nil { - lastCheckpointBytes = totalWritten - lastCheckpointAt = time.Now() - } - } - if activeItemID != "" { - SetItemProgress( - activeItemID, - float64(completedSegments)/float64(len(segments)), - received.Load(), - 0, - ) - } - if onProgress != nil { - func() { - if b := r.currentResolutionBudget(); b != nil { - defer b.charge()() - } - _, _ = onProgress( - goja.Undefined(), - r.vm.ToValue(received.Load()), - r.vm.ToValue(int64(0)), - r.vm.ToValue(completedSegments), - r.vm.ToValue(len(segments)), - ) - }() - } - } - } - if firstFailure != nil { - if persistentCheckpoint && nextIndex > 0 && totalWritten > 0 && output.Sync() == nil { - _ = saveSegmentCheckpoint(checkpointPath, segmentTransferCheckpoint{ - Fingerprint: fingerprint, - NextIndex: nextIndex, - Bytes: totalWritten, - }) - } - return r.jsTransferError(*firstFailure) - } - if nextIndex != len(segments) || totalWritten <= 0 { - return r.jsTransferError(transferFailure{ - ErrorType: "integrity_failed", - Message: fmt.Sprintf( - "segmented transfer incomplete: assembled %d of %d segments", - nextIndex, - len(segments), - ), - }) - } - if err := output.Sync(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to sync segmented output: %v", err), - }) - } - if err := output.Close(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to close segmented output: %v", err), - }) - } - if err := os.Rename(stagedPath, fullPath); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to publish segmented output: %v", err), - }) - } - promoted = true - os.Remove(checkpointPath) - syncDir(filepath.Dir(fullPath)) - if activeItemID != "" { - SetItemProgress(activeItemID, 1, totalWritten, totalWritten) - } - return r.jsSuccess(map[string]any{ - "path": fullPath, - "size": totalWritten, - "segments": len(segments), - }) -} diff --git a/go_backend/extension_runtime_segments_test.go b/go_backend/extension_runtime_segments_test.go deleted file mode 100644 index 535b3a7f..00000000 --- a/go_backend/extension_runtime_segments_test.go +++ /dev/null @@ -1,257 +0,0 @@ -package gobackend - -import ( - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/dop251/goja" -) - -func segmentTestRuntime(t *testing.T, transport roundTripFunc) *extensionRuntime { - t.Helper() - runtime := newFileDownloadTestRuntime(t, transport) - runtime.manifest.Capabilities = map[string]any{ - "downloadTransfer": map[string]any{ - "maxAttempts": float64(3), - "initialRetryDelayMs": float64(100), - "maxRetryDelayMs": float64(100), - "resumePolicy": "validated", - "persistentCheckpoint": true, - "maxParallelSegments": float64(3), - }, - } - return runtime -} - -func TestFileDownloadSegmentsPreservesOrderRetriesAndRunsConcurrently(t *testing.T) { - var active atomic.Int32 - var maxActive atomic.Int32 - var mu sync.Mutex - attempts := map[string]int{} - runtime := segmentTestRuntime(t, func(req *http.Request) (*http.Response, error) { - name := strings.TrimPrefix(req.URL.Path, "/") - mu.Lock() - attempts[name]++ - attempt := attempts[name] - mu.Unlock() - - current := active.Add(1) - defer active.Add(-1) - for { - previous := maxActive.Load() - if current <= previous || maxActive.CompareAndSwap(previous, current) { - break - } - } - time.Sleep(40 * time.Millisecond) - if name == "segment-1" && attempt == 1 { - return &http.Response{ - StatusCode: http.StatusServiceUnavailable, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader("retry")), - Request: req, - }, nil - } - body := map[string]string{ - "segment-0": "A", - "segment-1": "B", - "segment-2": "C", - }[name] - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(body)), - ContentLength: int64(len(body)), - Request: req, - }, nil - }) - - segments := []any{ - "https://cdn.example.com/segment-0", - "https://cdn.example.com/segment-1", - "https://cdn.example.com/segment-2", - } - result := runtime.fileDownloadSegments(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue(segments), - runtime.vm.ToValue("out/track.flac"), - }}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("segmented result = %#v", result) - } - data, err := os.ReadFile(filepath.Join(runtime.dataDir, "out", "track.flac")) - if err != nil || string(data) != "ABC" { - t.Fatalf("assembled data = %q, err=%v", data, err) - } - if maxActive.Load() < 2 { - t.Fatalf("segments did not overlap; max active = %d", maxActive.Load()) - } - mu.Lock() - segmentOneAttempts := attempts["segment-1"] - mu.Unlock() - if segmentOneAttempts != 2 { - t.Fatalf("segment-1 attempts = %d, want 2", segmentOneAttempts) - } -} - -func TestFileDownloadSegmentsResumesAssembledCheckpoint(t *testing.T) { - requested := make(chan string, 2) - runtime := segmentTestRuntime(t, func(req *http.Request) (*http.Response, error) { - name := strings.TrimPrefix(req.URL.Path, "/") - requested <- name - body := "B" - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(body)), - ContentLength: int64(len(body)), - Request: req, - }, nil - }) - - segments := []segmentTransferSpec{ - {Index: 0, URL: "https://cdn.example.com/segment-0"}, - {Index: 1, URL: "https://cdn.example.com/segment-1"}, - } - fullPath := filepath.Join(runtime.dataDir, "out", "track.flac") - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatal(err) - } - stagedPath := stagedDownloadPath(fullPath) - if err := os.WriteFile(stagedPath, []byte("A"), 0600); err != nil { - t.Fatal(err) - } - checkpointPath := transferCheckpointPath(stagedPath) + ".segments" - if err := saveSegmentCheckpoint(checkpointPath, segmentTransferCheckpoint{ - Fingerprint: segmentListFingerprint(segments), - NextIndex: 1, - Bytes: 1, - }); err != nil { - t.Fatal(err) - } - - result := runtime.fileDownloadSegments(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue([]any{segments[0].URL, segments[1].URL}), - runtime.vm.ToValue("out/track.flac"), - }}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("segmented resume result = %#v", result) - } - close(requested) - requests := []string{} - for name := range requested { - requests = append(requests, name) - } - if len(requests) != 1 || requests[0] != "segment-1" { - t.Fatalf("requests after checkpoint = %v", requests) - } - data, err := os.ReadFile(fullPath) - if err != nil || string(data) != "AB" { - t.Fatalf("resumed data = %q, err=%v", data, err) - } - if _, err := os.Stat(checkpointPath); !os.IsNotExist(err) { - t.Fatalf("checkpoint not removed after publish: %v", err) - } -} - -func TestFileDownloadSegmentsReturnsTypedExpiredStreamError(t *testing.T) { - runtime := segmentTestRuntime(t, func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: http.StatusForbidden, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader("expired")), - Request: req, - }, nil - }) - result := runtime.fileDownloadSegments(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue([]any{"https://cdn.example.com/segment-0"}), - runtime.vm.ToValue("out/track.flac"), - }}).Export().(map[string]any) - if result["success"] != false || result["error_type"] != "expired_stream" || - fmt.Sprint(result["http_status"]) != fmt.Sprint(http.StatusForbidden) { - t.Fatalf("typed error = %#v", result) - } - if message := fmt.Sprint(result["error"]); !strings.Contains(message, "403") { - t.Fatalf("typed error message = %q", message) - } -} - -func TestFileDownloadSegmentsRestartsWhenCheckpointExceedsStagedFile(t *testing.T) { - var mu sync.Mutex - requested := []string{} - runtime := segmentTestRuntime(t, func(req *http.Request) (*http.Response, error) { - name := strings.TrimPrefix(req.URL.Path, "/") - mu.Lock() - requested = append(requested, name) - mu.Unlock() - body := map[string]string{"segment-0": "A", "segment-1": "B"}[name] - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(body)), - ContentLength: int64(len(body)), - Request: req, - }, nil - }) - segments := []segmentTransferSpec{ - {Index: 0, URL: "https://cdn.example.com/segment-0"}, - {Index: 1, URL: "https://cdn.example.com/segment-1"}, - } - fullPath := filepath.Join(runtime.dataDir, "out", "track.flac") - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatal(err) - } - stagedPath := stagedDownloadPath(fullPath) - if err := os.WriteFile(stagedPath, nil, 0600); err != nil { - t.Fatal(err) - } - if err := saveSegmentCheckpoint( - transferCheckpointPath(stagedPath)+".segments", - segmentTransferCheckpoint{ - Fingerprint: segmentListFingerprint(segments), - NextIndex: 1, - Bytes: 1, - }, - ); err != nil { - t.Fatal(err) - } - - result := runtime.fileDownloadSegments(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue([]any{segments[0].URL, segments[1].URL}), - runtime.vm.ToValue("out/track.flac"), - }}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("stale-checkpoint result = %#v", result) - } - data, err := os.ReadFile(fullPath) - if err != nil || string(data) != "AB" { - t.Fatalf("restarted segmented data = %q, err=%v", data, err) - } - mu.Lock() - requestCount := len(requested) - mu.Unlock() - if requestCount != 2 { - t.Fatalf("requested segments = %v", requested) - } -} - -func TestSegmentCheckpointFingerprintIncludesQueryIdentity(t *testing.T) { - first := []segmentTransferSpec{{ - Index: 0, - URL: "https://cdn.example.com/audio?media=first", - }} - second := []segmentTransferSpec{{ - Index: 0, - URL: "https://cdn.example.com/audio?media=second", - }} - if segmentListFingerprint(first) == segmentListFingerprint(second) { - t.Fatal("different segment query identities shared a checkpoint fingerprint") - } -} diff --git a/go_backend/extension_runtime_stall_test.go b/go_backend/extension_runtime_stall_test.go deleted file mode 100644 index 80ec4271..00000000 --- a/go_backend/extension_runtime_stall_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package gobackend - -import ( - "context" - "errors" - "net/http" - "net/http/httptest" - "testing" - "time" -) - -// TestStallWatchdogCancelsOnNoData verifies the watchdog aborts a transfer that -// stops sending bytes, marks itself stalled (not user-cancelled), and does so -// after resetting on the initial byte. -func TestStallWatchdogCancelsOnNoData(t *testing.T) { - block := make(chan struct{}) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Length", "1000") - w.WriteHeader(http.StatusOK) - w.Write([]byte("x")) - w.(http.Flusher).Flush() - <-block // simulate a dead radio mid-transfer: stop sending - })) - defer srv.Close() - defer close(block) - - req, _ := http.NewRequestWithContext(context.Background(), "GET", srv.URL, nil) - req, wd := bindStallWatchdog(req, 150*time.Millisecond) - defer wd.stop() - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("do: %v", err) - } - defer resp.Body.Close() - - buf := make([]byte, 32) - var readErr error - for { - n, er := resp.Body.Read(buf) - if n > 0 { - wd.reset() - } - if er != nil { - readErr = er - break - } - } - if !errors.Is(readErr, context.Canceled) { - t.Fatalf("read error = %v, want watchdog context cancellation", readErr) - } - if !wd.stalled.Load() { - t.Fatalf("watchdog did not mark stalled; err=%v", readErr) - } -} diff --git a/go_backend/extension_runtime_storage.go b/go_backend/extension_runtime_storage.go deleted file mode 100644 index c74448a8..00000000 --- a/go_backend/extension_runtime_storage.go +++ /dev/null @@ -1,552 +0,0 @@ -package gobackend - -import ( - "crypto/aes" - "crypto/cipher" - "crypto/rand" - "crypto/sha256" - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - "sync" - - "github.com/dop251/goja" -) - -// Isolated per-download runtimes of the same extension share the storage, -// credentials, and salt files on disk, so writers must be serialized -// process-wide; the per-runtime mutexes only cover a single VM. -var extensionFileMus sync.Map // file path -> *sync.Mutex - -type extensionFileIdentity struct { - exists bool - size int64 - modified int64 -} - -type extensionJSONCacheEntry struct { - identity extensionFileIdentity - snapshot map[string]any -} - -// Shared by all isolated runtimes so repeated storage/credential reads avoid -// reading, decoding, and (for credentials) decrypting the complete file. The -// corresponding extensionFileMu must be held while accessing an entry. -var extensionJSONCaches sync.Map // file path -> *extensionJSONCacheEntry - -func extensionFileMu(path string) *sync.Mutex { - mu, _ := extensionFileMus.LoadOrStore(path, &sync.Mutex{}) - return mu.(*sync.Mutex) -} - -// writeExtensionFileLocked writes data via a temp file + rename so a reader -// never observes a torn write. Callers must hold extensionFileMu(path). -func writeExtensionFileLocked(path string, data []byte) error { - tmp := path + ".tmp" - if err := os.WriteFile(tmp, data, 0600); err != nil { - return err - } - return os.Rename(tmp, path) -} - -func extensionFileIdentityForPath(path string) (extensionFileIdentity, error) { - info, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return extensionFileIdentity{}, nil - } - return extensionFileIdentity{}, err - } - return extensionFileIdentity{ - exists: true, - size: info.Size(), - modified: info.ModTime().UnixNano(), - }, nil -} - -func cloneJSONValue(value any) any { - switch typed := value.(type) { - case map[string]any: - return cloneJSONMap(typed) - case []any: - result := make([]any, len(typed)) - for index, item := range typed { - result[index] = cloneJSONValue(item) - } - return result - default: - return typed - } -} - -func cloneJSONMap(source map[string]any) map[string]any { - result := make(map[string]any, len(source)) - for key, value := range source { - result[key] = cloneJSONValue(value) - } - return result -} - -// readCachedJSONMapLocked returns an isolated snapshot. The path-specific -// mutex must be held, which makes the stat/load/update sequence coherent with -// writers from all runtimes. -func readCachedJSONMapLocked( - path string, - load func() (map[string]any, error), -) (map[string]any, error) { - snapshot, err := cachedJSONMapLocked(path, load) - if err != nil { - return nil, err - } - return cloneJSONMap(snapshot), nil -} - -// cachedJSONMapLocked returns the shared, read-only cache entry. Never expose -// it to a VM or mutate it. Callers must hold the path-specific file mutex. -func cachedJSONMapLocked( - path string, - load func() (map[string]any, error), -) (map[string]any, error) { - identity, err := extensionFileIdentityForPath(path) - if err != nil { - return nil, err - } - if cached, ok := extensionJSONCaches.Load(path); ok { - entry := cached.(*extensionJSONCacheEntry) - if entry.identity == identity { - return entry.snapshot, nil - } - } - - snapshot, err := load() - if err != nil { - return nil, err - } - extensionJSONCaches.Store(path, &extensionJSONCacheEntry{ - identity: identity, - snapshot: snapshot, - }) - return snapshot, nil -} - -func readCachedJSONValueLocked(path, key string, load func() (map[string]any, error)) (any, bool, error) { - snapshot, err := cachedJSONMapLocked(path, load) - if err != nil { - return nil, false, err - } - value, exists := snapshot[key] - return cloneJSONValue(value), exists, nil -} - -func storeCachedJSONMapLocked(path string, snapshot map[string]any) error { - identity, err := extensionFileIdentityForPath(path) - if err != nil { - extensionJSONCaches.Delete(path) - return err - } - extensionJSONCaches.Store(path, &extensionJSONCacheEntry{ - identity: identity, - snapshot: cloneJSONMap(snapshot), - }) - return nil -} - -func (r *extensionRuntime) getStoragePath() string { - return filepath.Join(r.dataDir, "storage.json") -} - -func readJSONMapFile(path string) (map[string]any, error) { - data, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return make(map[string]any), nil - } - return nil, err - } - result := make(map[string]any) - if err := json.Unmarshal(data, &result); err != nil { - return nil, err - } - if result == nil { - result = make(map[string]any) - } - return result, nil -} - -func (r *extensionRuntime) mutateStorage(mutate func(map[string]any) bool) error { - r.storageMu.RLock() - closed := r.storageClosed - r.storageMu.RUnlock() - if closed { - return fmt.Errorf("storage is closed") - } - - path := r.getStoragePath() - fileMu := extensionFileMu(path) - fileMu.Lock() - snapshot, err := readCachedJSONMapLocked(path, func() (map[string]any, error) { - return readJSONMapFile(path) - }) - if err == nil && mutate(snapshot) { - var data []byte - data, err = json.Marshal(snapshot) - if err == nil { - err = writeExtensionFileLocked(path, data) - } - if err == nil { - err = storeCachedJSONMapLocked(path, snapshot) - } - } - fileMu.Unlock() - if err != nil { - return err - } - - r.storageMu.Lock() - r.storageCache = snapshot - r.storageMu.Unlock() - return nil -} - -func (r *extensionRuntime) flushStorageNow() error { - // Mutations are persisted synchronously under the process-wide file lock. - return nil -} - -func (r *extensionRuntime) closeStorageFlusher() { - r.storageMu.Lock() - r.storageClosed = true - r.storageMu.Unlock() -} - -func (r *extensionRuntime) storageGet(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return goja.Undefined() - } - - key := call.Arguments[0].String() - - path := r.getStoragePath() - fileMu := extensionFileMu(path) - fileMu.Lock() - value, exists, err := readCachedJSONValueLocked(path, key, func() (map[string]any, error) { - return readJSONMapFile(path) - }) - fileMu.Unlock() - if err != nil { - GoLog("[Extension:%s] Storage load error: %v\n", r.extensionID, err) - return goja.Undefined() - } - - if !exists { - if len(call.Arguments) > 1 { - return call.Arguments[1] - } - return goja.Undefined() - } - - return r.vm.ToValue(value) -} - -func (r *extensionRuntime) storageSet(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.vm.ToValue(false) - } - - key := call.Arguments[0].String() - value := call.Arguments[1].Export() - - if err := r.mutateStorage(func(storage map[string]any) bool { - storage[key] = value - return true - }); err != nil { - GoLog("[Extension:%s] Storage save error: %v\n", r.extensionID, err) - return r.vm.ToValue(false) - } - - return r.vm.ToValue(true) -} - -func (r *extensionRuntime) storageRemove(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue(false) - } - - key := call.Arguments[0].String() - - if err := r.mutateStorage(func(storage map[string]any) bool { - if _, exists := storage[key]; !exists { - return false - } - delete(storage, key) - return true - }); err != nil { - GoLog("[Extension:%s] Storage save error: %v\n", r.extensionID, err) - return r.vm.ToValue(false) - } - - return r.vm.ToValue(true) -} - -func (r *extensionRuntime) getCredentialsPath() string { - return filepath.Join(r.dataDir, ".credentials.enc") -} - -func (r *extensionRuntime) getSaltPath() string { - return filepath.Join(r.dataDir, ".cred_salt") -} - -func (r *extensionRuntime) getOrCreateSalt() ([]byte, error) { - saltPath := r.getSaltPath() - - // Serialize concurrent runtimes: if two generated different salts, the - // loser's credentials would become undecryptable. - mu := extensionFileMu(saltPath) - mu.Lock() - defer mu.Unlock() - - salt, err := os.ReadFile(saltPath) - if err == nil && len(salt) == 32 { - return salt, nil - } - - salt = make([]byte, 32) - if _, err := io.ReadFull(rand.Reader, salt); err != nil { - return nil, fmt.Errorf("failed to generate salt: %w", err) - } - - if err := writeExtensionFileLocked(saltPath, salt); err != nil { - return nil, fmt.Errorf("failed to save salt: %w", err) - } - - return salt, nil -} - -func (r *extensionRuntime) getEncryptionKey() ([]byte, error) { - return deriveExtensionStorageKey(r.extensionID, "credentials") -} - -func (r *extensionRuntime) getLegacyEncryptionKey() ([]byte, error) { - salt, err := r.getOrCreateSalt() - if err != nil { - return nil, err - } - - combined := append([]byte(r.extensionID), salt...) - hash := sha256.Sum256(combined) - return hash[:], nil -} - -func (r *extensionRuntime) readCredentialsFileLocked() (map[string]any, error) { - data, err := os.ReadFile(r.getCredentialsPath()) - if err != nil { - if os.IsNotExist(err) { - return make(map[string]any), nil - } - return nil, err - } - key, err := r.getEncryptionKey() - if err != nil { - return nil, fmt.Errorf("failed to get encryption key: %w", err) - } - decrypted, err := decryptAES(data, key) - if err != nil { - legacyKey, legacyKeyErr := r.getLegacyEncryptionKey() - if legacyKeyErr != nil { - return nil, fmt.Errorf("failed to decrypt credentials: %w", err) - } - decrypted, legacyKeyErr = decryptAES(data, legacyKey) - if legacyKeyErr != nil { - return nil, fmt.Errorf("failed to decrypt credentials: %w", err) - } - // Transparently replace the legacy extension-id-derived ciphertext while - // the caller holds the per-file lock. - migrated, migrateErr := encryptAES(decrypted, key) - if migrateErr != nil { - return nil, fmt.Errorf("failed to migrate credentials: %w", migrateErr) - } - if migrateErr = writeExtensionFileLocked(r.getCredentialsPath(), migrated); migrateErr != nil { - return nil, fmt.Errorf("failed to migrate credentials: %w", migrateErr) - } - } - creds := make(map[string]any) - if err := json.Unmarshal(decrypted, &creds); err != nil { - return nil, err - } - if creds == nil { - creds = make(map[string]any) - } - return creds, nil -} - -func (r *extensionRuntime) refreshCredentials() error { - path := r.getCredentialsPath() - fileMu := extensionFileMu(path) - fileMu.Lock() - snapshot, err := readCachedJSONMapLocked(path, r.readCredentialsFileLocked) - fileMu.Unlock() - if err != nil { - return err - } - r.credentialsMu.Lock() - r.credentialsCache = snapshot - r.credentialsMu.Unlock() - return nil -} - -func (r *extensionRuntime) mutateCredentials(mutate func(map[string]any)) error { - path := r.getCredentialsPath() - fileMu := extensionFileMu(path) - fileMu.Lock() - snapshot, err := readCachedJSONMapLocked(path, r.readCredentialsFileLocked) - if err == nil { - mutate(snapshot) - var data []byte - data, err = json.Marshal(snapshot) - if err == nil { - var key []byte - key, err = r.getEncryptionKey() - if err == nil { - data, err = encryptAES(data, key) - } - } - if err == nil { - err = writeExtensionFileLocked(path, data) - } - if err == nil { - err = storeCachedJSONMapLocked(path, snapshot) - } - } - fileMu.Unlock() - if err != nil { - return err - } - r.credentialsMu.Lock() - r.credentialsCache = snapshot - r.credentialsMu.Unlock() - return nil -} - -func (r *extensionRuntime) credentialsStore(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.jsError("key and value are required") - } - - key := call.Arguments[0].String() - value := call.Arguments[1].Export() - - if err := r.mutateCredentials(func(credentials map[string]any) { - credentials[key] = value - }); err != nil { - GoLog("[Extension:%s] Credentials save error: %v\n", r.extensionID, err) - return r.jsError("%s", err.Error()) - } - - return r.jsSuccess(nil) -} - -func (r *extensionRuntime) credentialsGet(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return goja.Undefined() - } - - key := call.Arguments[0].String() - - if err := r.refreshCredentials(); err != nil { - GoLog("[Extension:%s] Credentials load error: %v\n", r.extensionID, err) - return goja.Undefined() - } - - r.credentialsMu.RLock() - value, exists := r.credentialsCache[key] - r.credentialsMu.RUnlock() - if !exists { - if len(call.Arguments) > 1 { - return call.Arguments[1] - } - return goja.Undefined() - } - - return r.vm.ToValue(value) -} - -func (r *extensionRuntime) credentialsRemove(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue(false) - } - - key := call.Arguments[0].String() - - if err := r.mutateCredentials(func(credentials map[string]any) { - delete(credentials, key) - }); err != nil { - GoLog("[Extension:%s] Credentials save error: %v\n", r.extensionID, err) - return r.vm.ToValue(false) - } - - return r.vm.ToValue(true) -} - -func (r *extensionRuntime) credentialsHas(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue(false) - } - - key := call.Arguments[0].String() - - if err := r.refreshCredentials(); err != nil { - return r.vm.ToValue(false) - } - - r.credentialsMu.RLock() - _, exists := r.credentialsCache[key] - r.credentialsMu.RUnlock() - return r.vm.ToValue(exists) -} - -func encryptAES(plaintext []byte, key []byte) ([]byte, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, err - } - - nonce := make([]byte, gcm.NonceSize()) - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { - return nil, err - } - - ciphertext := gcm.Seal(nonce, nonce, plaintext, nil) - return ciphertext, nil -} - -func decryptAES(ciphertext []byte, key []byte) ([]byte, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, err - } - - nonceSize := gcm.NonceSize() - if len(ciphertext) < nonceSize { - return nil, fmt.Errorf("ciphertext too short") - } - - nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] - plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) - if err != nil { - return nil, err - } - - return plaintext, nil -} diff --git a/go_backend/extension_runtime_storage_test.go b/go_backend/extension_runtime_storage_test.go deleted file mode 100644 index 75df73d5..00000000 --- a/go_backend/extension_runtime_storage_test.go +++ /dev/null @@ -1,307 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "github.com/dop251/goja" -) - -func setStorageValue(t *testing.T, runtime *extensionRuntime, key string, value any) { - t.Helper() - result := runtime.storageSet(goja.FunctionCall{ - Arguments: []goja.Value{ - runtime.vm.ToValue(key), - runtime.vm.ToValue(value), - }, - }) - if !result.ToBoolean() { - t.Fatalf("storage.set(%q) returned false", key) - } -} - -func TestExtensionJSONCacheUsesIdentityAndIsolatesSnapshots(t *testing.T) { - path := filepath.Join(t.TempDir(), "storage.json") - if err := os.WriteFile(path, []byte(`{"value":"first"}`), 0600); err != nil { - t.Fatal(err) - } - loads := 0 - load := func() (map[string]any, error) { - loads++ - return readJSONMapFile(path) - } - - mu := extensionFileMu(path) - mu.Lock() - first, err := readCachedJSONMapLocked(path, load) - mu.Unlock() - if err != nil { - t.Fatal(err) - } - first["value"] = "mutated locally" - - mu.Lock() - second, err := readCachedJSONMapLocked(path, load) - mu.Unlock() - if err != nil { - t.Fatal(err) - } - if loads != 1 || second["value"] != "first" { - t.Fatalf("cache did not isolate/reuse snapshot: loads=%d data=%#v", loads, second) - } - - if err := os.WriteFile(path, []byte(`{"value":"external update"}`), 0600); err != nil { - t.Fatal(err) - } - mu.Lock() - third, err := readCachedJSONMapLocked(path, load) - mu.Unlock() - if err != nil { - t.Fatal(err) - } - if loads != 2 || third["value"] != "external update" { - t.Fatalf("external update was not reloaded: loads=%d data=%#v", loads, third) - } -} - -func TestExtensionStorageKeyReadIsolatedAndFresh(t *testing.T) { - dataDir := t.TempDir() - ext := &loadedExtension{ID: "key-read", Manifest: &ExtensionManifest{Name: "key-read"}, DataDir: dataDir} - a := newExtensionRuntime(ext) - b := newExtensionRuntime(ext) - a.RegisterAPIs(goja.New()) - b.RegisterAPIs(goja.New()) - setStorageValue(t, a, "nested", map[string]any{"items": []any{map[string]any{"value": "original"}}}) - read := func(r *extensionRuntime, key string) goja.Value { - return r.storageGet(goja.FunctionCall{Arguments: []goja.Value{r.vm.ToValue(key)}}) - } - value := read(a, "nested").ToObject(a.vm) - items := value.Get("items").ToObject(a.vm) - if err := items.Get("0").ToObject(a.vm).Set("value", "local mutation"); err != nil { - t.Fatal(err) - } - for _, runtime := range []*extensionRuntime{a, b} { - got := read(runtime, "nested").ToObject(runtime.vm).Get("items").ToObject(runtime.vm).Get("0").ToObject(runtime.vm).Get("value").String() - if got != "original" { - t.Fatalf("VM mutation leaked: %q", got) - } - } - setStorageValue(t, b, "nested", "updated") - if got := read(a, "nested").String(); got != "updated" { - t.Fatalf("cross-runtime update not visible: %q", got) - } - if err := os.WriteFile(filepath.Join(dataDir, "storage.json"), []byte(`{"nested":"external replacement","nullValue":null}`), 0600); err != nil { - t.Fatal(err) - } - if got := read(a, "nested").String(); got != "external replacement" { - t.Fatalf("external replacement not visible: %q", got) - } - if !goja.IsNull(read(a, "nullValue")) || !goja.IsUndefined(read(a, "absent")) { - t.Fatal("null and absent values must remain distinct") - } - if err := os.Remove(filepath.Join(dataDir, "storage.json")); err != nil { - t.Fatal(err) - } - if !goja.IsUndefined(read(a, "nested")) { - t.Fatal("removed storage file reused stale value") - } -} - -func BenchmarkExtensionCachedSingleKeyRead(b *testing.B) { - for _, size := range []int{10, 10000} { - b.Run(fmt.Sprintf("entries_%d", size), func(b *testing.B) { - path := filepath.Join(b.TempDir(), "storage.json") - snapshot := map[string]any{"token": "value"} - for i := 0; i < size; i++ { - snapshot[fmt.Sprint(i)] = map[string]any{"items": []any{"large cached value", i}} - } - data, _ := json.Marshal(snapshot) - if err := os.WriteFile(path, data, 0600); err != nil { - b.Fatal(err) - } - load := func() (map[string]any, error) { return readJSONMapFile(path) } - mu := extensionFileMu(path) - mu.Lock() - defer mu.Unlock() - if _, _, err := readCachedJSONValueLocked(path, "token", load); err != nil { - b.Fatal(err) - } - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, _, err := readCachedJSONValueLocked(path, "token", load); err != nil { - b.Fatal(err) - } - } - }) - } -} - -func TestExtensionRuntimeStorageConcurrentRuntimesMergeWrites(t *testing.T) { - dataDir := t.TempDir() - ext := &loadedExtension{ID: "merge-test", Manifest: &ExtensionManifest{Name: "merge-test"}, DataDir: dataDir} - runtimeA := newExtensionRuntime(ext) - runtimeB := newExtensionRuntime(ext) - runtimeA.RegisterAPIs(goja.New()) - runtimeB.RegisterAPIs(goja.New()) - - start := make(chan struct{}) - done := make(chan bool, 2) - go func() { - <-start - result := runtimeA.storageSet(goja.FunctionCall{Arguments: []goja.Value{ - runtimeA.vm.ToValue("from_a"), runtimeA.vm.ToValue("a"), - }}) - done <- result.ToBoolean() - }() - go func() { - <-start - result := runtimeB.storageSet(goja.FunctionCall{Arguments: []goja.Value{ - runtimeB.vm.ToValue("from_b"), runtimeB.vm.ToValue("b"), - }}) - done <- result.ToBoolean() - }() - close(start) - firstSucceeded, secondSucceeded := <-done, <-done - if !firstSucceeded || !secondSucceeded { - t.Fatal("concurrent storage write failed") - } - - storage := readStorageMap(t, filepath.Join(dataDir, "storage.json")) - if storage["from_a"] != "a" || storage["from_b"] != "b" { - t.Fatalf("concurrent storage writes were not merged: %#v", storage) - } - - credStart := make(chan struct{}) - credDone := make(chan struct{}, 2) - for _, item := range []struct { - runtime *extensionRuntime - key string - }{ - {runtimeA, "token_a"}, - {runtimeB, "token_b"}, - } { - item := item - go func() { - <-credStart - result := item.runtime.credentialsStore(goja.FunctionCall{Arguments: []goja.Value{ - item.runtime.vm.ToValue(item.key), - item.runtime.vm.ToValue(item.key + "_value"), - }}) - if success, _ := result.Export().(map[string]any)["success"].(bool); !success { - t.Errorf("credentialsStore(%s) failed", item.key) - } - credDone <- struct{}{} - }() - } - close(credStart) - <-credDone - <-credDone - - reader := newExtensionRuntime(ext) - reader.RegisterAPIs(goja.New()) - for _, key := range []string{"token_a", "token_b"} { - got := reader.credentialsGet(goja.FunctionCall{Arguments: []goja.Value{reader.vm.ToValue(key)}}).String() - if got != key+"_value" { - t.Fatalf("credential %s = %q", key, got) - } - } -} - -func readStorageMap(t *testing.T, storagePath string) map[string]any { - t.Helper() - data, err := os.ReadFile(storagePath) - if err != nil { - t.Fatalf("failed to read storage file: %v", err) - } - - var parsed map[string]any - if err := json.Unmarshal(data, &parsed); err != nil { - t.Fatalf("failed to unmarshal storage file: %v", err) - } - return parsed -} - -func TestExtensionRuntimeStorage_AtomicWriteCompactJSON(t *testing.T) { - ext := &loadedExtension{ - ID: "storage-test", - Manifest: &ExtensionManifest{ - Name: "storage-test", - }, - DataDir: t.TempDir(), - } - - runtime := newExtensionRuntime(ext) - runtime.RegisterAPIs(goja.New()) - - setStorageValue(t, runtime, "k1", "v1") - setStorageValue(t, runtime, "k2", 2) - - storagePath := filepath.Join(ext.DataDir, "storage.json") - deadline := time.Now().Add(1500 * time.Millisecond) - - var raw []byte - for time.Now().Before(deadline) { - data, err := os.ReadFile(storagePath) - if err == nil { - raw = data - break - } - time.Sleep(20 * time.Millisecond) - } - if len(raw) == 0 { - t.Fatalf("storage.json was not written within timeout") - } - - var parsed map[string]any - if err := json.Unmarshal(raw, &parsed); err != nil { - t.Fatalf("failed to unmarshal storage file: %v", err) - } - if parsed["k1"] != "v1" { - t.Fatalf("expected k1=v1, got %v", parsed["k1"]) - } - if parsed["k2"] != float64(2) { - t.Fatalf("expected k2=2, got %v", parsed["k2"]) - } - if bytes.Contains(raw, []byte("\n")) { - t.Fatalf("expected compact JSON without indentation, got: %q", string(raw)) - } -} - -func TestUnloadExtension_FlushesPendingStorage(t *testing.T) { - ext := &loadedExtension{ - ID: "unload-storage-test", - Manifest: &ExtensionManifest{ - Name: "unload-storage-test", - }, - DataDir: t.TempDir(), - VM: goja.New(), - } - - runtime := newExtensionRuntime(ext) - runtime.RegisterAPIs(ext.VM) - ext.runtime = runtime - - manager := &extensionManager{ - extensions: map[string]*loadedExtension{ - ext.ID: ext, - }, - } - - setStorageValue(t, runtime, "persist_on_unload", true) - - if err := manager.UnloadExtension(ext.ID); err != nil { - t.Fatalf("UnloadExtension failed: %v", err) - } - - storagePath := filepath.Join(ext.DataDir, "storage.json") - parsed := readStorageMap(t, storagePath) - if parsed["persist_on_unload"] != true { - t.Fatalf("expected pending storage value to be flushed on unload, got %v", parsed["persist_on_unload"]) - } -} diff --git a/go_backend/extension_runtime_supplement_test.go b/go_backend/extension_runtime_supplement_test.go deleted file mode 100644 index 1f804f90..00000000 --- a/go_backend/extension_runtime_supplement_test.go +++ /dev/null @@ -1,1195 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/dop251/goja" -) - -func TestPendingAuthRequestExpiresAfterThreeMinutes(t *testing.T) { - if pendingAuthRequestTTL != 3*time.Minute { - t.Fatalf("pending auth request TTL = %v", pendingAuthRequestTTL) - } -} - -func TestExtensionRuntimeAuthAndPolyfills(t *testing.T) { - vm := goja.New() - runtime := &extensionRuntime{ - extensionID: "auth-ext", - manifest: &ExtensionManifest{ - Name: "auth-ext", - Description: "Auth extension", - Version: "1.0.0", - Permissions: ExtensionPermissions{ - Network: []string{"auth.example.com", "token.example.com", "api.example.com"}, - }, - }, - settings: map[string]any{}, - httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch req.URL.Host { - case "token.example.com": - return &http.Response{ - StatusCode: 200, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(`{"access_token":"access","refresh_token":"refresh","expires_in":3600}`)), - Request: req, - }, nil - case "api.example.com": - if req.URL.Path == "/huge" { - return &http.Response{ - StatusCode: 200, - Header: make(http.Header), - Body: io.NopCloser(io.LimitReader( - strings.NewReader(strings.Repeat("x", maxExtensionHTTPResponseBytes+1)), - maxExtensionHTTPResponseBytes+1, - )), - Request: req, - }, nil - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"X-Test": []string{"yes"}}, - Body: io.NopCloser(strings.NewReader(`{"ok":true,"items":[1,2]}`)), - Request: req, - }, nil - default: - return &http.Response{StatusCode: 404, Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}, - vm: vm, - } - - if err := validateExtensionAuthURL("https://user:pass@auth.example.com/login"); err == nil { - t.Fatal("expected embedded credential error") - } - if err := validateExtensionAuthURL("http://auth.example.com/login"); err == nil { - t.Fatal("expected non-https auth URL error") - } - if got := summarizeURLForLog("https://auth.example.com/login?token=secret"); got != "https://auth.example.com/login" { - t.Fatalf("summary = %q", got) - } - - openResult := runtime.authOpenUrl(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("https://auth.example.com/login"), - vm.ToValue("app://callback"), - }}).Export().(map[string]any) - if openResult["success"] != true { - t.Fatalf("authOpenUrl = %#v", openResult) - } - pending := GetPendingAuthRequest("auth-ext") - if pending == nil || pending.AuthURL == "" || pending.State == "" || !strings.Contains(pending.AuthURL, "state=") { - t.Fatalf("pending auth = %#v", pending) - } - if extensionID, err := ResolveExtensionCallbackState(pending.State); err != nil || extensionID != "auth-ext" { - t.Fatalf("resolve callback state = %q/%v", extensionID, err) - } - if extensionID, err := ResolveExtensionCallbackState(pending.State); err != nil || extensionID != "auth-ext" { - t.Fatalf("resolve callback state retry = %q/%v", extensionID, err) - } - if extensionID, err := ConsumeExtensionCallbackState(pending.State); err != nil || extensionID != "auth-ext" { - t.Fatalf("consume callback state = %q/%v", extensionID, err) - } - if _, err := ConsumeExtensionCallbackState(pending.State); err == nil { - t.Fatal("callback state replay should be rejected") - } - collisionState := "shared-callback-state" - collisionCreatedAt := time.Now() - if err := registerPendingAuthRequest(&PendingAuthRequest{ - ExtensionID: "auth-ext", - State: collisionState, - CreatedAt: collisionCreatedAt, - }); err != nil { - t.Fatalf("register first callback state: %v", err) - } - if err := registerPendingAuthRequest(&PendingAuthRequest{ - ExtensionID: "other-ext", - State: collisionState, - CreatedAt: collisionCreatedAt.Add(time.Second), - }); err == nil { - t.Fatal("callback state collision should be rejected") - } - ClearPendingAuthRequest("auth-ext") - if code := runtime.authGetCode(goja.FunctionCall{}); !goja.IsUndefined(code) { - t.Fatalf("expected undefined code, got %v", code) - } - if ok := runtime.authSetCode(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(map[string]any{"code": "abc", "access_token": "access", "refresh_token": "refresh", "expires_in": float64(60)})}}); !ok.ToBoolean() { - t.Fatal("authSetCode returned false") - } - if code := runtime.authGetCode(goja.FunctionCall{}).String(); code != "abc" { - t.Fatalf("code = %q", code) - } - if !runtime.authIsAuthenticated(goja.FunctionCall{}).ToBoolean() { - t.Fatal("expected authenticated runtime") - } - tokens := runtime.authGetTokens(goja.FunctionCall{}).Export().(map[string]any) - if tokens["access_token"] != "access" { - t.Fatalf("tokens = %#v", tokens) - } - - pkce := runtime.authGeneratePKCE(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(float64(50))}}).Export().(map[string]any) - if pkce["method"] != "S256" || pkce["verifier"] == "" || pkce["challenge"] == "" { - t.Fatalf("pkce = %#v", pkce) - } - if current := runtime.authGetPKCE(goja.FunctionCall{}).Export().(map[string]any); current["verifier"] == "" { - t.Fatalf("current pkce = %#v", current) - } - oauthConfig := map[string]any{ - "authUrl": "https://auth.example.com/oauth", - "clientId": "client", - "redirectUri": "app://callback", - "scope": "read", - "extraParams": map[string]any{"prompt": "login"}, - } - oauth := runtime.authStartOAuthWithPKCE(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(oauthConfig)}}).Export().(map[string]any) - if oauth["success"] != true || !strings.Contains(oauth["authUrl"].(string), "code_challenge") { - t.Fatalf("oauth = %#v", oauth) - } - tokenConfig := map[string]any{ - "tokenUrl": "https://token.example.com/token", - "clientId": "client", - "redirectUri": "app://callback", - "code": "abc", - } - token := runtime.authExchangeCodeWithPKCE(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(tokenConfig)}}).Export().(map[string]any) - if token["success"] != true || token["access_token"] != "access" { - t.Fatalf("token = %#v", token) - } - - runtime.registerTextEncoderDecoder(vm) - runtime.registerURLClass(vm) - runtime.registerJSONGlobal(vm) - vm.Set("fetch", func(call goja.FunctionCall) goja.Value { - return runtime.fetchPolyfill(call) - }) - vm.Set("atob", func(call goja.FunctionCall) goja.Value { - return runtime.atobPolyfill(call) - }) - vm.Set("btoa", func(call goja.FunctionCall) goja.Value { - return runtime.btoaPolyfill(call) - }) - - value, err := vm.RunString(` - var encoded = btoa("hello"); - var decoded = atob(encoded); - var te = new TextEncoder(); - var bytes = te.encode("hi"); - var into = te.encodeInto("hi", []); - var td = new TextDecoder(); - var text = td.decode(bytes); - var url = new URL("/path?a=1&a=2#frag", "https://api.example.com/base"); - var params = new URLSearchParams("?x=1"); - params.append("x", "2"); - params.set("y", "3"); - var response = fetch("https://api.example.com/data", {method: "POST", body: {q: "x"}, headers: {"X-Client": "test"}}); - JSON.stringify({ - encoded: encoded, - decoded: decoded, - text: text, - read: into.read, - host: url.hostname, - first: url.searchParams.get("a"), - all: url.searchParams.getAll("a").length, - params: params.toString(), - ok: response.ok, - status: response.status, - jsonOk: response.json().ok, - bufferLen: response.arrayBuffer().length, - bufferFirst: response.arrayBuffer()[0] - }); - `) - if err != nil { - t.Fatalf("polyfill script: %v", err) - } - var result map[string]any - if err := json.Unmarshal([]byte(value.String()), &result); err != nil { - t.Fatalf("decode polyfill result: %v", err) - } - if result["decoded"] != "hello" || result["host"] != "api.example.com" || result["ok"] != true || - result["bufferLen"] != float64(len(`{"ok":true,"items":[1,2]}`)) || result["bufferFirst"] != float64('{') { - t.Fatalf("polyfill result = %#v", result) - } - - blocked := runtime.fetchPolyfill(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("https://blocked.example.com")}}).ToObject(vm) - if blocked.Get("ok").ToBoolean() { - t.Fatal("expected blocked fetch") - } - huge := runtime.fetchPolyfill(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("https://api.example.com/huge")}}).ToObject(vm) - if huge.Get("ok").ToBoolean() || !strings.Contains(huge.Get("error").String(), "exceeds") { - t.Fatalf("expected bounded fetch response, got %s", huge.String()) - } - runtime.authClear(goja.FunctionCall{}) - if runtime.authIsAuthenticated(goja.FunctionCall{}).ToBoolean() { - t.Fatal("expected auth cleared") - } -} - -type failingBodyReader struct { - data []byte - sent bool -} - -func (f *failingBodyReader) Read(p []byte) (int, error) { - if !f.sent { - f.sent = true - n := copy(p, f.data) - return n, nil - } - return 0, fmt.Errorf("connection reset") -} - -func newFileDownloadTestRuntime(t *testing.T, transport roundTripFunc) *extensionRuntime { - t.Helper() - return &extensionRuntime{ - extensionID: "dl-ext", - manifest: &ExtensionManifest{ - Name: "dl-ext", - Version: "1.0.0", - Permissions: ExtensionPermissions{ - File: true, - Network: []string{"cdn.example.com"}, - }, - }, - dataDir: t.TempDir(), - vm: goja.New(), - httpClient: &http.Client{Transport: transport}, - } -} - -func TestFileDownloadStagesAndPromotes(t *testing.T) { - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: 200, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader("audio-bytes")), - ContentLength: int64(len("audio-bytes")), - Request: req, - }, nil - }) - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("https://cdn.example.com/track.flac"), - runtime.vm.ToValue("out/track.flac"), - }}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("download result = %#v", result) - } - - finalPath := filepath.Join(runtime.dataDir, "out", "track.flac") - data, err := os.ReadFile(finalPath) - if err != nil || string(data) != "audio-bytes" { - t.Fatalf("final file = %q/%v", data, err) - } - if _, err := os.Stat(stagedDownloadPath(finalPath)); !os.IsNotExist(err) { - t.Fatalf("staged file left behind: %v", err) - } -} - -func TestFileDownloadFailureLeavesNoFinalFile(t *testing.T) { - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: 200, - Header: make(http.Header), - Body: io.NopCloser(&failingBodyReader{data: []byte("partial-aud")}), - ContentLength: 1 << 20, - Request: req, - }, nil - }) - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("https://cdn.example.com/track.flac"), - runtime.vm.ToValue("out/track.flac"), - }}).Export().(map[string]any) - if result["success"] != false { - t.Fatalf("expected failed download, got %#v", result) - } - - finalPath := filepath.Join(runtime.dataDir, "out", "track.flac") - if _, err := os.Stat(finalPath); !os.IsNotExist(err) { - t.Fatalf("partial download visible at final path: %v", err) - } - if _, err := os.Stat(stagedDownloadPath(finalPath)); !os.IsNotExist(err) { - t.Fatalf("staged file left behind: %v", err) - } -} - -func TestFileDownloadDoesNotResumeMidBodyCutByDefault(t *testing.T) { - const full = "hello-world!" - var attempts int - var rangeSeen bool - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - attempts++ - rangeSeen = rangeSeen || req.Header.Get("Range") != "" - h := make(http.Header) - h.Set("ETag", `"v1"`) - return &http.Response{ - StatusCode: 200, - Header: h, - Body: io.NopCloser(&failingBodyReader{data: []byte(full[:6])}), - ContentLength: int64(len(full)), - Request: req, - }, nil - }) - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("https://cdn.example.com/track.flac"), - runtime.vm.ToValue("out/track.flac"), - }}).Export().(map[string]any) - if result["success"] != false { - t.Fatalf("expected failed download, got %#v", result) - } - if attempts != defaultTransferMaxAttempts { - t.Fatalf("attempts = %d, want %d full retries", attempts, defaultTransferMaxAttempts) - } - if rangeSeen { - t.Fatal("default retry unexpectedly sent a Range request") - } - finalPath := filepath.Join(runtime.dataDir, "out", "track.flac") - if _, err := os.Stat(finalPath); !os.IsNotExist(err) { - t.Fatalf("partial download visible at final path: %v", err) - } - if _, err := os.Stat(stagedDownloadPath(finalPath)); !os.IsNotExist(err) { - t.Fatalf("staged file left behind: %v", err) - } -} - -func TestFileDownloadResumesAfterMidBodyCutWhenEnabled(t *testing.T) { - const full = "hello-world!" - var attempts int - var resumeRange, resumeIfRange string - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - attempts++ - if attempts == 1 { - h := make(http.Header) - h.Set("ETag", `"v1"`) - return &http.Response{ - StatusCode: 200, - Header: h, - Body: io.NopCloser(&failingBodyReader{data: []byte(full[:6])}), - ContentLength: int64(len(full)), - Request: req, - }, nil - } - resumeRange = req.Header.Get("Range") - resumeIfRange = req.Header.Get("If-Range") - h := make(http.Header) - h.Set("Content-Range", fmt.Sprintf("bytes 6-%d/%d", len(full)-1, len(full))) - return &http.Response{ - StatusCode: 206, - Header: h, - Body: io.NopCloser(strings.NewReader(full[6:])), - ContentLength: int64(len(full) - 6), - Request: req, - }, nil - }) - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("https://cdn.example.com/track.flac"), - runtime.vm.ToValue("out/track.flac"), - runtime.vm.ToValue(map[string]any{"resume": true}), - }}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("download result = %#v", result) - } - if attempts != 2 || resumeRange != "bytes=6-" || resumeIfRange != `"v1"` { - t.Fatalf("attempts=%d range=%q if-range=%q", attempts, resumeRange, resumeIfRange) - } - data, err := os.ReadFile(filepath.Join(runtime.dataDir, "out", "track.flac")) - if err != nil || string(data) != full { - t.Fatalf("final file = %q/%v", data, err) - } -} - -func TestFileDownloadResumeRestartsWhenRangeIgnored(t *testing.T) { - const full = "hello-world!" - var attempts int - runtime := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) { - attempts++ - h := make(http.Header) - h.Set("ETag", `"v1"`) - if attempts == 1 { - return &http.Response{ - StatusCode: 200, - Header: h, - Body: io.NopCloser(&failingBodyReader{data: []byte(full[:6])}), - ContentLength: int64(len(full)), - Request: req, - }, nil - } - // Server ignores the Range header and replays the full body. - return &http.Response{ - StatusCode: 200, - Header: h, - Body: io.NopCloser(strings.NewReader(full)), - ContentLength: int64(len(full)), - Request: req, - }, nil - }) - - result := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("https://cdn.example.com/track.flac"), - runtime.vm.ToValue("out/track.flac"), - runtime.vm.ToValue(map[string]any{"resume": true}), - }}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("download result = %#v", result) - } - if attempts != 2 { - t.Fatalf("attempts = %d, want 2", attempts) - } - data, err := os.ReadFile(filepath.Join(runtime.dataDir, "out", "track.flac")) - if err != nil || string(data) != full { - t.Fatalf("final file = %q/%v", data, err) - } -} - -func TestResetDownloadCancelDropsStaleFlag(t *testing.T) { - const itemID = "reset-cancel-item" - - // A cancel issued while nothing is downloading pre-registers a flag that - // the next attempt would consume and abort; reset must drop it. - cancelDownload(itemID) - if !isDownloadCancelled(itemID) { - t.Fatal("expected pre-registered cancel flag") - } - resetDownloadCancel(itemID) - if isDownloadCancelled(itemID) { - t.Fatal("expected stale cancel flag to be dropped") - } - - // A cancel attached to an active download must survive reset. - ctx := initDownloadCancel(itemID) - cancelDownload(itemID) - resetDownloadCancel(itemID) - if !isDownloadCancelled(itemID) { - t.Fatal("expected active cancel to survive reset") - } - select { - case <-ctx.Done(): - default: - t.Fatal("expected cancelled context") - } - clearDownloadCancel(itemID) -} - -func TestParseExtensionTrackValueExplicit(t *testing.T) { - vm := goja.New() - - value, err := vm.RunString(`({name: "Song", explicit: true})`) - if err != nil { - t.Fatalf("RunString: %v", err) - } - if track := parseExtensionTrackValue(vm, value); !track.Explicit { - t.Fatalf("expected explicit track, got %#v", track) - } - - value, err = vm.RunString(`({name: "Song", isExplicit: true})`) - if err != nil { - t.Fatalf("RunString: %v", err) - } - if track := parseExtensionTrackValue(vm, value); !track.Explicit { - t.Fatalf("expected explicit track via isExplicit, got %#v", track) - } - - value, err = vm.RunString(`({name: "Song"})`) - if err != nil { - t.Fatalf("RunString: %v", err) - } - if track := parseExtensionTrackValue(vm, value); track.Explicit { - t.Fatalf("expected clean track, got %#v", track) - } -} - -func TestDeezerTrackIsExplicit(t *testing.T) { - if deezerTrackIsExplicit(deezerTrack{}) { - t.Fatal("expected clean track by default") - } - if !deezerTrackIsExplicit(deezerTrack{ExplicitLyrics: true}) { - t.Fatal("expected explicit via explicit_lyrics") - } - if !deezerTrackIsExplicit(deezerTrack{ExplicitContentLyrics: 1}) { - t.Fatal("expected explicit via explicit_content_lyrics == 1") - } - if deezerTrackIsExplicit(deezerTrack{ExplicitContentLyrics: 2}) { - t.Fatal("expected unknown (2) to be treated as clean") - } -} - -func TestExtensionStoreDiskCacheSurvivesRestart(t *testing.T) { - dir := t.TempDir() - registryURL := "https://registry.example.com/registry.json" - store := &extensionRepo{ - registryURL: registryURL, - cacheDir: dir, - cacheTTL: time.Hour, - cache: &repoRegistry{ - Version: 1, - Extensions: []repoExtension{{ID: "ext", Name: "ext", Version: "1.0.0"}}, - }, - cacheTime: time.Now(), - etag: `"registry-v1"`, - lastModified: "Sun, 30 Aug 2026 00:00:00 GMT", - } - store.saveDiskCache() - - // Simulates an app restart: a fresh store loads the disk cache, then the - // Dart layer re-applies the same registry URL. - restarted := &extensionRepo{cacheDir: dir, cacheTTL: time.Hour} - restarted.loadDiskCache() - if restarted.getRegistryURL() != registryURL { - t.Fatalf("registry URL after restart = %q", restarted.getRegistryURL()) - } - if restarted.etag != `"registry-v1"` || restarted.lastModified == "" { - t.Fatalf("conditional cache metadata was not restored: %q / %q", restarted.etag, restarted.lastModified) - } - restarted.setRegistryURL(registryURL) - if restarted.cache == nil || len(restarted.cache.Extensions) != 1 { - t.Fatalf("expected cache to survive re-applying the same registry URL, got %#v", restarted.cache) - } - - restarted.setRegistryURL("https://other.example.com/registry.json") - if restarted.cache != nil { - t.Fatal("expected cache reset after registry URL change") - } -} - -func TestParseRegistryBody(t *testing.T) { - registry, err := parseRegistryBody([]byte(`{"version":1,"extensions":[{"id":"ext","name":"ext","version":"1.0.0"}]}`)) - if err != nil || len(registry.Extensions) != 1 { - t.Fatalf("parseRegistryBody = %#v/%v", registry, err) - } - - if _, err := parseRegistryBody([]byte("<!DOCTYPE html><html></html>")); err == nil || !strings.Contains(err.Error(), "web page") { - t.Fatalf("expected web page error, got %v", err) - } - - if _, err := parseRegistryBody([]byte("not json")); err == nil || !strings.Contains(err.Error(), "failed to parse registry") { - t.Fatalf("expected parse error, got %v", err) - } -} - -func TestExtensionStoreSettingsAndRuntimeStorage(t *testing.T) { - dir := t.TempDir() - store := &extensionRepo{ - registryURL: "https://registry.example.com/registry.json", - cacheDir: dir, - cacheTTL: time.Hour, - cache: &repoRegistry{ - Version: 1, - UpdatedAt: "2026-05-04", - Extensions: []repoExtension{ - { - ID: "coverage-ext", - Name: "coverage-ext", - DisplayNameAlt: "Coverage Extension", - Version: "2.0.0", - Description: "Metadata and lyrics provider", - DownloadURLAlt: "https://registry.example.com/coverage.spotiflac-ext", - IconURLAlt: "https://registry.example.com/icon.png", - Category: CategoryMetadata, - Tags: []string{"metadata", "lyrics"}, - Downloads: 10, - UpdatedAt: "2026-05-04", - MinAppVersionAlt: "4.5.0", - }, - { - ID: "utility-ext", - Name: "utility-ext", - Version: "1.0.0", - Description: "Utility", - DownloadURL: "https://registry.example.com/utility.spotiflac-ext", - Category: CategoryUtility, - UpdatedAt: "2026-05-04", - }, - }, - }, - cacheTime: time.Now(), - } - store.saveDiskCache() - loadedStore := &extensionRepo{cacheDir: dir} - loadedStore.loadDiskCache() - if loadedStore.cache == nil || len(loadedStore.cache.Extensions) != 2 { - t.Fatalf("loaded cache = %#v", loadedStore.cache) - } - if got := store.getRegistryURL(); got != "https://registry.example.com/registry.json" { - t.Fatalf("registry URL = %q", got) - } - store.setRegistryURL("https://registry.example.com/new.json") - if store.cache != nil { - t.Fatal("expected cache reset after registry URL change") - } - store.cache = loadedStore.cache - store.cacheTime = time.Now() - - manager := getExtensionManager() - manager.mu.Lock() - if manager.extensions == nil { - manager.extensions = map[string]*loadedExtension{} - } - manager.extensions["coverage-ext"] = &loadedExtension{ - ID: "coverage-ext", - Manifest: &ExtensionManifest{ - Name: "coverage-ext", - DisplayName: "Coverage Extension", - Version: "1.0.0", - Description: "Installed", - Types: []ExtensionType{ExtensionTypeMetadataProvider}, - }, - Enabled: true, - } - manager.mu.Unlock() - defer func() { - manager.mu.Lock() - delete(manager.extensions, "coverage-ext") - manager.mu.Unlock() - }() - - extensions, err := store.getExtensionsWithStatus(false) - if err != nil { - t.Fatalf("getExtensionsWithStatus: %v", err) - } - if len(extensions) != 2 || !extensions[0].IsInstalled || !extensions[0].HasUpdate { - t.Fatalf("extensions = %#v", extensions) - } - found, err := store.searchExtensions("lyrics", CategoryMetadata) - if err != nil || len(found) != 1 || found[0].ID != "coverage-ext" { - t.Fatalf("search = %#v/%v", found, err) - } - all, err := store.searchExtensions("", "") - if err != nil || len(all) != 2 { - t.Fatalf("all search = %#v/%v", all, err) - } - if cats := store.getCategories(); len(cats) != 5 { - t.Fatalf("categories = %#v", cats) - } - if err := requireHTTPSURL("http://example.com", "registry"); err == nil { - t.Fatal("expected HTTPS validation error") - } - if _, err := resolveRegistryURL(""); err == nil { - t.Fatal("expected empty registry URL error") - } - if resolved, err := resolveRegistryURL("http://github.com/owner/repo"); err != nil || !strings.Contains(resolved, "raw.githubusercontent.com/owner/repo") { - t.Fatalf("resolved registry = %q/%v", resolved, err) - } - store.clearCache() - if store.cache != nil { - t.Fatal("expected cleared store cache") - } - - settingsStore := &ExtensionSettingsStore{settings: map[string]map[string]any{}} - if err := settingsStore.SetDataDir(filepath.Join(dir, "settings")); err != nil { - t.Fatalf("SetDataDir: %v", err) - } - if err := settingsStore.Set("ext", "quality", "lossless"); err != nil { - t.Fatalf("settings Set: %v", err) - } - if value, err := settingsStore.Get("ext", "quality"); err != nil || value != "lossless" { - t.Fatalf("settings Get = %#v/%v", value, err) - } - if _, err := settingsStore.Get("ext", "missing"); err == nil { - t.Fatal("expected missing setting error") - } - if err := settingsStore.SetAll("ext", map[string]any{"a": float64(1), "_secret": "hidden"}); err != nil { - t.Fatalf("settings SetAll: %v", err) - } - if all := settingsStore.GetAll("ext"); all["a"] != float64(1) { - t.Fatalf("settings all = %#v", all) - } - settingsCiphertext, err := os.ReadFile(settingsStore.getSettingsPath("ext")) - if err != nil { - t.Fatalf("read encrypted settings: %v", err) - } - if bytes.Contains(settingsCiphertext, []byte("hidden")) || bytes.Contains(settingsCiphertext, []byte("quality")) { - t.Fatal("extension settings were stored as plaintext") - } - if _, err := os.Stat(settingsStore.getLegacySettingsPath("ext")); !os.IsNotExist(err) { - t.Fatalf("plaintext settings file still exists: %v", err) - } - reloadedWithData := &ExtensionSettingsStore{settings: map[string]map[string]any{}} - if err := reloadedWithData.SetDataDir(settingsStore.dataDir); err != nil { - t.Fatalf("reload encrypted settings: %v", err) - } - if reloadedWithData.GetAll("ext")["_secret"] != "hidden" { - t.Fatal("encrypted settings did not round-trip") - } - if err := settingsStore.Remove("ext", "a"); err != nil { - t.Fatalf("settings Remove: %v", err) - } - if err := settingsStore.RemoveAll("ext"); err != nil { - t.Fatalf("settings RemoveAll: %v", err) - } - if jsonText, err := settingsStore.GetAllExtensionSettingsJSON(); err != nil || jsonText == "" { - t.Fatalf("settings JSON = %q/%v", jsonText, err) - } - reloaded := &ExtensionSettingsStore{settings: map[string]map[string]any{}} - if err := reloaded.SetDataDir(settingsStore.dataDir); err != nil { - t.Fatalf("reload settings: %v", err) - } - - vm := goja.New() - runtime := &extensionRuntime{ - extensionID: "storage-ext", - dataDir: filepath.Join(dir, "runtime"), - vm: vm, - } - if err := os.MkdirAll(runtime.dataDir, 0755); err != nil { - t.Fatal(err) - } - if got := runtime.storageGet(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("missing"), vm.ToValue("fallback")}}).String(); got != "fallback" { - t.Fatalf("storage fallback = %q", got) - } - if ok := runtime.storageSet(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("key"), vm.ToValue(map[string]any{"nested": "value"})}}); !ok.ToBoolean() { - t.Fatal("storageSet false") - } - if ok := runtime.storageSet(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("key"), vm.ToValue(map[string]any{"nested": "value"})}}); !ok.ToBoolean() { - t.Fatal("storageSet equal false") - } - if err := runtime.flushStorageNow(); err != nil { - t.Fatalf("flushStorageNow: %v", err) - } - if ok := runtime.storageRemove(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("key")}}); !ok.ToBoolean() { - t.Fatal("storageRemove false") - } - runtime.closeStorageFlusher() - if ok := runtime.storageSet(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("after_close"), vm.ToValue("x")}}); ok.ToBoolean() { - t.Fatal("expected storageSet false after close") - } - - credRuntime := &extensionRuntime{ - extensionID: "cred-ext", - dataDir: filepath.Join(dir, "creds"), - vm: vm, - } - if err := os.MkdirAll(credRuntime.dataDir, 0755); err != nil { - t.Fatal(err) - } - if result := credRuntime.credentialsStore(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("token"), vm.ToValue("secret")}}).Export().(map[string]any); result["success"] != true { - t.Fatalf("credentialsStore = %#v", result) - } - if got := credRuntime.credentialsGet(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("token")}}).String(); got != "secret" { - t.Fatalf("credential = %q", got) - } - if !credRuntime.credentialsHas(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("token")}}).ToBoolean() { - t.Fatal("expected credential") - } - if ok := credRuntime.credentialsRemove(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("token")}}); !ok.ToBoolean() { - t.Fatal("credentialsRemove false") - } - if credRuntime.credentialsHas(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("token")}}).ToBoolean() { - t.Fatal("expected credential removed") - } - key, err := credRuntime.getEncryptionKey() - if err != nil { - t.Fatalf("getEncryptionKey: %v", err) - } - encrypted, err := encryptAES([]byte("plain"), key) - if err != nil { - t.Fatalf("encryptAES: %v", err) - } - decrypted, err := decryptAES(encrypted, key) - if err != nil || string(decrypted) != "plain" { - t.Fatalf("decryptAES = %q/%v", decrypted, err) - } - if _, err := decryptAES([]byte("short"), key); err == nil { - t.Fatal("expected short ciphertext error") - } -} - -func TestExtensionRuntimeHTTPMatchingAndMetadataHelpers(t *testing.T) { - vm := goja.New() - jar, _ := newSimpleCookieJar() - runtime := &extensionRuntime{ - extensionID: "http-ext", - manifest: &ExtensionManifest{ - Name: "http-ext", - Description: "HTTP extension", - Version: "1.0.0", - Permissions: ExtensionPermissions{ - Network: []string{"api.example.com"}, - }, - }, - vm: vm, - cookieJar: jar, - httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - var body []byte - if req.Body != nil { - body, _ = io.ReadAll(req.Body) - } - header := make(http.Header) - header.Set("X-Method", req.Method) - if req.URL.Path == "/huge" { - return &http.Response{StatusCode: 200, Header: header, Body: io.NopCloser(io.LimitReader(strings.NewReader(strings.Repeat("x", maxExtensionHTTPResponseBytes+2)), maxExtensionHTTPResponseBytes+2)), Request: req}, nil - } - return &http.Response{ - StatusCode: 201, - Header: header, - Body: io.NopCloser(strings.NewReader(req.Method + ":" + string(body))), - Request: req, - }, nil - })}, - } - - if err := runtime.validateDomain("https://api.example.com/path"); err != nil { - t.Fatalf("validateDomain allowed: %v", err) - } - for _, rawURL := range []string{"notaurl", "http://api.example.com", "https://user:pass@api.example.com", "https://127.0.0.1/x", "https://blocked.example.com/x"} { - if err := runtime.validateDomain(rawURL); err == nil { - t.Fatalf("expected domain validation error for %s", rawURL) - } - } - if got := runtime.httpGet(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("https://api.example.com/get"), vm.ToValue(map[string]any{"X-Test": "yes"})}}).Export().(map[string]any); got["status"] != 201 || !strings.Contains(got["body"].(string), "GET") { - t.Fatalf("httpGet = %#v", got) - } - if got := runtime.httpPost(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("https://api.example.com/post"), vm.ToValue(map[string]any{"a": "b"})}}).Export().(map[string]any); !strings.Contains(got["body"].(string), "POST") { - t.Fatalf("httpPost = %#v", got) - } - requestOptions := map[string]any{"method": "patch", "body": []any{"x"}, "headers": map[string]any{"X-Req": "1"}} - if got := runtime.httpRequest(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("https://api.example.com/request"), vm.ToValue(requestOptions)}}).Export().(map[string]any); !strings.Contains(got["body"].(string), "PATCH") { - t.Fatalf("httpRequest = %#v", got) - } - for _, method := range []struct { - name string - call func(goja.FunctionCall) goja.Value - args []goja.Value - }{ - {name: "PUT", call: runtime.httpPut, args: []goja.Value{vm.ToValue("https://api.example.com/put"), vm.ToValue("body")}}, - {name: "DELETE", call: runtime.httpDelete, args: []goja.Value{vm.ToValue("https://api.example.com/delete"), vm.ToValue(map[string]any{"X-Delete": "1"})}}, - {name: "PATCH", call: runtime.httpPatch, args: []goja.Value{vm.ToValue("https://api.example.com/patch"), vm.ToValue(map[string]any{"p": "q"})}}, - } { - if got := method.call(goja.FunctionCall{Arguments: method.args}).Export().(map[string]any); !strings.Contains(got["body"].(string), method.name) { - t.Fatalf("%s = %#v", method.name, got) - } - } - if got := runtime.httpGet(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("https://api.example.com/huge")}}).Export().(map[string]any); !strings.Contains(got["error"].(string), "exceeds") { - t.Fatalf("huge response = %#v", got) - } - if !runtime.httpClearCookies(goja.FunctionCall{}).ToBoolean() { - t.Fatal("expected cookies cleared") - } - - if runtime.matchingCompareStrings(goja.FunctionCall{}).ToFloat() != 0 { - t.Fatal("missing string compare args should be zero") - } - if runtime.matchingCompareStrings(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("Song"), vm.ToValue("song")}}).ToFloat() != 1 { - t.Fatal("expected exact string similarity") - } - if runtime.matchingCompareDuration(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(180000), vm.ToValue(182000)}}).ToBoolean() != true { - t.Fatal("expected duration match") - } - if runtime.matchingNormalizeString(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("Song (Remastered) feat. Guest!")}}).String() != "song" { - t.Fatalf("normalized = %q", runtime.matchingNormalizeString(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("Song (Remastered) feat. Guest!")}}).String()) - } - - if formatMusicBrainzGenre([]musicBrainzTag{{Count: 1, Name: "rock"}, {Count: 5, Name: "electronic"}, {Count: 10, Name: "rock"}}) != "Electronic" { - t.Fatal("unexpected genre selection") - } - credits := []musicBrainzArtistCredit{{Name: "A", JoinPhrase: " & "}, {Name: "B"}} - if formatMusicBrainzArtistCredit(credits) != "A & B" { - t.Fatal("artist credit format mismatch") - } - releases := []musicBrainzRelease{ - {Title: "Other", ArtistCredit: []musicBrainzArtistCredit{{Name: "Fallback"}}}, - {Title: "Album", ArtistCredit: credits}, - } - if selectMusicBrainzAlbumArtist(releases, "Album") != "A & B" || selectMusicBrainzAlbumArtist(releases, "") != "Fallback" { - t.Fatal("album artist selection mismatch") - } -} - -func TestExtensionRuntimeFileAPIs(t *testing.T) { - vm := goja.New() - dir := t.TempDir() - SetAllowedDownloadDirs(nil) - defer SetAllowedDownloadDirs(nil) - - fileBody := "chunk" - runtime := &extensionRuntime{ - extensionID: "file-ext", - manifest: &ExtensionManifest{ - Name: "file-ext", - Description: "File extension", - Version: "1.0.0", - Permissions: ExtensionPermissions{ - File: true, - Network: []string{"files.example.com"}, - }, - }, - dataDir: dir, - vm: vm, - httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - if req.Header.Get("Range") == "" { - body := "downloaded" - return &http.Response{ - StatusCode: 200, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(body)), - ContentLength: int64(len(body)), - Request: req, - }, nil - } - rangeHeader := req.Header.Get("Range") - start, end := 0, len(fileBody)-1 - if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end); err != nil { - start, end = 0, 1 - } - if start < 0 { - start = 0 - } - if end >= len(fileBody) { - end = len(fileBody) - 1 - } - if start > len(fileBody) { - start = len(fileBody) - } - body := fileBody[start : end+1] - header := http.Header{"Content-Range": []string{fmt.Sprintf("bytes %d-%d/%d", start, end, len(fileBody))}} - return &http.Response{StatusCode: 206, Header: header, Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil - })}, - } - runtime.downloadClient = runtime.httpClient - - if _, err := (&extensionRuntime{manifest: &ExtensionManifest{}}).validatePath("x"); err == nil { - t.Fatal("expected file permission error") - } - if _, err := runtime.validatePath("../escape.txt"); err == nil { - t.Fatal("expected sandbox escape error") - } - AddAllowedDownloadDir(dir) - AddAllowedDownloadDir(filepath.Clean(dir)) - allowedDownloadDirsMu.RLock() - allowedDirCount := len(allowedDownloadDirs) - allowedDownloadDirsMu.RUnlock() - if allowedDirCount != 1 { - t.Fatalf("duplicate allowed directories retained: %d", allowedDirCount) - } - absolutePath := filepath.Join(dir, "allowed.txt") - if got, err := runtime.validatePath(absolutePath); err != nil || got != absolutePath { - t.Fatalf("absolute validatePath = %q/%v", got, err) - } - - write := runtime.fileWrite(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("nested/a.txt"), vm.ToValue("hello")}}).Export().(map[string]any) - if write["success"] != true { - t.Fatalf("fileWrite = %#v", write) - } - if !runtime.fileExists(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("nested/a.txt")}}).ToBoolean() { - t.Fatal("expected written file to exist") - } - read := runtime.fileRead(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("nested/a.txt")}}).Export().(map[string]any) - if read["data"] != "hello" { - t.Fatalf("fileRead = %#v", read) - } - - writeBytes := runtime.fileWriteBytes(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("nested/bytes.bin"), - vm.ToValue("4869"), - vm.ToValue(map[string]any{"encoding": "hex", "truncate": true}), - }}).Export().(map[string]any) - if writeBytes["success"] != true { - t.Fatalf("fileWriteBytes = %#v", writeBytes) - } - appendBytes := runtime.fileWriteBytes(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("nested/bytes.bin"), - vm.ToValue([]any{float64('!')}), - vm.ToValue(map[string]any{"append": true}), - }}).Export().(map[string]any) - if appendBytes["success"] != true { - t.Fatalf("append fileWriteBytes = %#v", appendBytes) - } - readBytes := runtime.fileReadBytes(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("nested/bytes.bin"), - vm.ToValue(map[string]any{"encoding": "text", "offset": float64(1), "length": float64(2)}), - }}).Export().(map[string]any) - if readBytes["data"] != "i!" || readBytes["bytes_read"] != 2 { - t.Fatalf("fileReadBytes = %#v", readBytes) - } - if bad := runtime.fileWriteBytes(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("nested/bad.bin"), - vm.ToValue("x"), - vm.ToValue(map[string]any{"append": true, "offset": float64(1)}), - }}).Export().(map[string]any); bad["success"] != false { - t.Fatalf("expected append+offset failure, got %#v", bad) - } - if bad := runtime.fileReadBytes(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("nested/bytes.bin"), - vm.ToValue(map[string]any{"encoding": "bad"}), - }}).Export().(map[string]any); bad["success"] != false { - t.Fatalf("expected bad encoding failure, got %#v", bad) - } - - copyResult := runtime.fileCopy(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("nested/bytes.bin"), vm.ToValue("nested/copy.bin")}}).Export().(map[string]any) - if copyResult["success"] != true { - t.Fatalf("fileCopy = %#v", copyResult) - } - moveResult := runtime.fileMove(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("nested/copy.bin"), vm.ToValue("nested/moved.bin")}}).Export().(map[string]any) - if moveResult["success"] != true { - t.Fatalf("fileMove = %#v", moveResult) - } - sizeResult := runtime.fileGetSize(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("nested/moved.bin")}}).Export().(map[string]any) - if sizeResult["success"] != true || sizeResult["size"] != int64(3) { - t.Fatalf("fileGetSize = %#v", sizeResult) - } - deleteResult := runtime.fileDelete(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("nested/moved.bin")}}).Export().(map[string]any) - if deleteResult["success"] != true { - t.Fatalf("fileDelete = %#v", deleteResult) - } - - download := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("https://files.example.com/file"), - vm.ToValue("downloads/file.bin"), - }}).Export().(map[string]any) - if download["success"] != true { - t.Fatalf("fileDownload = %#v", download) - } - if data, err := os.ReadFile(filepath.Join(dir, "downloads/file.bin")); err != nil || string(data) != "downloaded" { - t.Fatalf("downloaded data = %q/%v", data, err) - } - - chunked := runtime.fileDownload(goja.FunctionCall{Arguments: []goja.Value{ - vm.ToValue("https://files.example.com/chunk"), - vm.ToValue("downloads/chunk.bin"), - vm.ToValue(map[string]any{"chunked": float64(2), "headers": map[string]any{"X-Test": "yes"}}), - }}).Export().(map[string]any) - if chunked["success"] != true { - t.Fatalf("chunked fileDownload = %#v", chunked) - } - if data, err := os.ReadFile(filepath.Join(dir, "downloads/chunk.bin")); err != nil || string(data) != fileBody { - t.Fatalf("chunked data = %q/%v", data, err) - } - - if missing := runtime.fileDownload(goja.FunctionCall{}).Export().(map[string]any); missing["success"] != false { - t.Fatalf("expected missing download args error, got %#v", missing) - } -} - -func TestExtensionRuntimeUtilityAPIs(t *testing.T) { - vm := goja.New() - runtime := &extensionRuntime{extensionID: "utils-ext", vm: vm} - - if runtime.sha256Hash(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("abc")}}).String() == "" { - t.Fatal("expected sha256") - } - if runtime.hmacSHA256(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("msg"), vm.ToValue("key")}}).String() == "" { - t.Fatal("expected hmac sha256") - } - if runtime.hmacSHA256Base64(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("msg"), vm.ToValue("key")}}).String() == "" { - t.Fatal("expected hmac sha256 base64") - } - if value := runtime.hmacSHA1(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue([]any{float64(1), float64(2)}), vm.ToValue([]any{float64(3)})}}); len(value.Export().([]any)) == 0 { - t.Fatal("expected hmac sha1 bytes") - } - if !goja.IsUndefined(runtime.parseJSON(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(`{bad`)}})) { - t.Fatal("expected invalid JSON to return undefined") - } - parsed := runtime.parseJSON(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(`{"ok":true}`)}}).Export().(map[string]any) - if parsed["ok"] != true { - t.Fatalf("parseJSON = %#v", parsed) - } - if text := runtime.stringifyJSON(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(map[string]any{"ok": true})}}).String(); !strings.Contains(text, "ok") { - t.Fatalf("stringifyJSON = %q", text) - } - encrypted := runtime.cryptoEncrypt(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("plain"), vm.ToValue("secret")}}).Export().(map[string]any) - if encrypted["success"] != true || encrypted["data"] == "" { - t.Fatalf("cryptoEncrypt = %#v", encrypted) - } - decrypted := runtime.cryptoDecrypt(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(encrypted["data"]), vm.ToValue("secret")}}).Export().(map[string]any) - if decrypted["success"] != true || decrypted["data"] != "plain" { - t.Fatalf("cryptoDecrypt = %#v", decrypted) - } - if bad := runtime.cryptoDecrypt(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("bad"), vm.ToValue("secret")}}).Export().(map[string]any); bad["success"] != false { - t.Fatalf("expected bad decrypt failure, got %#v", bad) - } - key := runtime.cryptoGenerateKey(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(float64(8))}}).Export().(map[string]any) - if key["success"] != true || key["key"] == "" || key["hex"] == "" { - t.Fatalf("cryptoGenerateKey = %#v", key) - } - for _, invalidLength := range []float64{-1, 0, 1.5, 4097} { - invalidKey := runtime.cryptoGenerateKey(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(invalidLength)}}).Export().(map[string]any) - if invalidKey["success"] != false { - t.Fatalf("cryptoGenerateKey(%v) should fail: %#v", invalidLength, invalidKey) - } - } - if runtime.randomUserAgent(goja.FunctionCall{}).String() == "" || runtime.appUserAgent(goja.FunctionCall{}).String() == "" { - t.Fatal("expected user agents") - } - SetAppVersion("9.9.9") - if runtime.appVersion(goja.FunctionCall{}).String() != "9.9.9" { - t.Fatal("appVersion mismatch") - } - if !runtime.sleep(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(float64(0))}}).ToBoolean() { - t.Fatal("zero sleep should succeed") - } - - itemID := "utils-item" - runtime.setActiveDownloadItemID(itemID) - initDownloadCancel(itemID) - if runtime.isDownloadCancelled(goja.FunctionCall{}).ToBoolean() { - t.Fatal("item should not be cancelled yet") - } - runtime.setDownloadStatus(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue(itemProgressStatusDownloading)}}) - cancelDownload(itemID) - if !runtime.isDownloadCancelled(goja.FunctionCall{}).ToBoolean() { - t.Fatal("item should be cancelled") - } - clearDownloadCancel(itemID) - runtime.clearActiveDownloadItemID() - - requestID := "utils-request" - runtime.setActiveRequestID(requestID) - initExtensionRequestCancel(requestID) - if runtime.isRequestCancelled(goja.FunctionCall{}).ToBoolean() { - t.Fatal("request should not be cancelled yet") - } - cancelExtensionRequest(requestID) - if !runtime.isRequestCancelled(goja.FunctionCall{}).ToBoolean() { - t.Fatal("request should be cancelled") - } - clearExtensionRequestCancel(requestID) - runtime.clearActiveRequestID() - - if msg := runtime.formatLogArgs([]goja.Value{vm.ToValue("a"), vm.ToValue(1)}); msg != "a 1" { - t.Fatalf("formatLogArgs = %q", msg) - } - objectLog := runtime.formatLogArgs([]goja.Value{ - vm.ToValue(map[string]any{"access_token": "must-not-be-exported"}), - }) - if strings.Contains(objectLog, "must-not-be-exported") || !strings.HasPrefix(objectLog, "<") { - t.Fatalf("object log was not summarized: %q", objectLog) - } - runtime.logDebug(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("debug")}}) - runtime.logInfo(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("info")}}) - runtime.logWarn(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("warn")}}) - runtime.logError(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("error")}}) -} - -func TestClassifySignedSessionExpiredAsVerification(t *testing.T) { - got := classifyDownloadErrorType("Failed to resolve provider download: signed session expired") - if got != "verification_required" { - t.Fatalf("expected verification_required, got %q", got) - } -} - -func TestVerificationRequiredNoteConsume(t *testing.T) { - r := &extensionRuntime{} - if got := r.consumeVerificationRequired(); got != "" { - t.Fatalf("expected empty before note, got %q", got) - } - r.noteVerificationRequired("") - if got := r.consumeVerificationRequired(); got != "pending" { - t.Fatalf("expected pending sentinel, got %q", got) - } - if got := r.consumeVerificationRequired(); got != "" { - t.Fatalf("expected cleared after consume, got %q", got) - } - r.noteVerificationRequired("https://x/auth") - if got := r.consumeVerificationRequired(); got != "https://x/auth" { - t.Fatalf("expected auth url, got %q", got) - } -} diff --git a/go_backend/extension_runtime_transfer.go b/go_backend/extension_runtime_transfer.go deleted file mode 100644 index bd838375..00000000 --- a/go_backend/extension_runtime_transfer.go +++ /dev/null @@ -1,712 +0,0 @@ -package gobackend - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/dop251/goja" -) - -const ( - transferCheckpointVersion = 1 - transferCheckpointBytes = 8 * 1024 * 1024 - transferCheckpointPeriod = 5 * time.Second -) - -type transferCheckpoint struct { - Version int `json:"version"` - Fingerprint string `json:"fingerprint"` - Validator string `json:"validator"` - Bytes int64 `json:"bytes"` - Total int64 `json:"total,omitempty"` - UpdatedAt int64 `json:"updated_at"` -} - -type transferFailure struct { - ErrorType string - Message string - HTTPStatus int - RetryAfterSeconds int - Attempts int -} - -func (r *extensionRuntime) jsTransferError(failure transferFailure) goja.Value { - values := map[string]any{ - "success": false, - "error": failure.Message, - "error_type": failure.ErrorType, - "attempts": failure.Attempts, - } - if failure.HTTPStatus > 0 { - values["http_status"] = failure.HTTPStatus - } - if failure.RetryAfterSeconds > 0 { - values["retry_after_seconds"] = failure.RetryAfterSeconds - } - return r.vm.ToValue(values) -} - -func transferURLFingerprint(rawURL string) string { - parsed, err := url.Parse(rawURL) - if err != nil { - return "" - } - // Query strings commonly contain short-lived CDN credentials. Excluding - // them both avoids persisting a secret-derived value and permits a freshly - // signed URL for the same object to continue a validator-protected partial. - identity := strings.ToLower(parsed.Scheme) + "://" + - strings.ToLower(parsed.Host) + parsed.EscapedPath() - sum := sha256.Sum256([]byte(identity)) - return hex.EncodeToString(sum[:]) -} - -func transferCheckpointPath(stagedPath string) string { - return stagedPath + ".checkpoint.json" -} - -func loadTransferCheckpoint(path, fingerprint string) (transferCheckpoint, bool) { - var checkpoint transferCheckpoint - data, err := os.ReadFile(path) - if err != nil || json.Unmarshal(data, &checkpoint) != nil { - return transferCheckpoint{}, false - } - if checkpoint.Version != transferCheckpointVersion || - checkpoint.Fingerprint == "" || - checkpoint.Fingerprint != fingerprint || - checkpoint.Validator == "" || - checkpoint.Bytes <= 0 { - return transferCheckpoint{}, false - } - return checkpoint, true -} - -func saveTransferCheckpoint(path string, checkpoint transferCheckpoint) error { - checkpoint.Version = transferCheckpointVersion - checkpoint.UpdatedAt = time.Now().UnixMilli() - data, err := json.Marshal(checkpoint) - if err != nil { - return err - } - tempPath := path + ".tmp" - file, err := os.OpenFile(tempPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) - if err != nil { - return err - } - if _, err = file.Write(data); err == nil { - err = file.Sync() - } - closeErr := file.Close() - if err == nil { - err = closeErr - } - if err != nil { - os.Remove(tempPath) - return err - } - if err := os.Rename(tempPath, path); err != nil { - os.Remove(tempPath) - return err - } - return nil -} - -func transferResponseValidator(headers http.Header) string { - if etag := strings.TrimSpace(headers.Get("ETag")); etag != "" && !strings.HasPrefix(strings.ToUpper(etag), "W/") { - return etag - } - return strings.TrimSpace(headers.Get("Last-Modified")) -} - -func transferTotalLength(resp *http.Response, rangeFrom int64) int64 { - if resp == nil { - return 0 - } - if resp.StatusCode == http.StatusPartialContent { - contentRange := resp.Header.Get("Content-Range") - if slash := strings.LastIndex(contentRange, "/"); slash >= 0 { - if total, err := strconv.ParseInt(contentRange[slash+1:], 10, 64); err == nil { - return total - } - } - if resp.ContentLength > 0 { - return rangeFrom + resp.ContentLength - } - return 0 - } - return resp.ContentLength -} - -func validTransferContentRange(resp *http.Response, rangeFrom int64) bool { - if rangeFrom <= 0 || resp == nil || resp.StatusCode != http.StatusPartialContent { - return true - } - want := fmt.Sprintf("bytes %d-", rangeFrom) - return strings.HasPrefix(resp.Header.Get("Content-Range"), want) -} - -func retryableTransferStatus(status int) bool { - return status == http.StatusRequestTimeout || - status == http.StatusTooEarly || - status == http.StatusTooManyRequests || - status >= 500 -} - -func retryAfterSeconds(resp *http.Response) int { - if resp == nil { - return 0 - } - delay := getRetryAfterDuration(resp) - if delay <= 0 { - return 0 - } - seconds := int(delay.Round(time.Second) / time.Second) - if seconds < 1 { - return 1 - } - return seconds -} - -func transferErrorTypeForStatus(status int, policy DownloadTransferPolicy) string { - if status == http.StatusTooManyRequests { - return "rate_limited" - } - if policy.RefreshStreamOnStatus[status] { - return "expired_stream" - } - if status >= 500 || status == http.StatusRequestTimeout || status == http.StatusTooEarly { - return "transient_network" - } - return "http_error" -} - -func waitTransferRetry(ctx context.Context, delay time.Duration) error { - if delay <= 0 { - return nil - } - return sleepRetry(ctx, delay) -} - -func transferRetryConfig(policy DownloadTransferPolicy) RetryConfig { - return RetryConfig{ - MaxRetries: max(0, policy.MaxAttempts-1), - InitialDelay: policy.InitialRetryDelay, - MaxDelay: policy.MaxRetryDelay, - BackoffFactor: 2, - } -} - -func (r *extensionRuntime) reliableFileDownload( - client *http.Client, - urlStr string, - fullPath string, - headers map[string]string, - onProgress goja.Callable, - trackItemBytes bool, - resumeDownload bool, - persistentCheckpoint bool, - policy DownloadTransferPolicy, -) goja.Value { - unlock := lockDownloadOutputPath(fullPath) - defer unlock() - - callerSetRange := false - for key := range headers { - if strings.EqualFold(key, "Range") { - callerSetRange = true - break - } - } - if callerSetRange { - // A caller-defined range describes a standalone output fragment. It - // cannot safely be combined with a checkpoint owned by this engine. - resumeDownload = false - persistentCheckpoint = false - } - - stagedPath := stagedDownloadPath(fullPath) - checkpointPath := transferCheckpointPath(stagedPath) - fingerprint := transferURLFingerprint(urlStr) - keepPartial := resumeDownload && persistentCheckpoint && fingerprint != "" - checkpoint, checkpointOK := loadTransferCheckpoint(checkpointPath, fingerprint) - if !keepPartial || !checkpointOK { - os.Remove(stagedPath) - os.Remove(checkpointPath) - checkpoint = transferCheckpoint{} - } - - out, err := os.OpenFile(stagedPath, os.O_CREATE|os.O_RDWR, 0600) - if err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to create staged file: %v", err), - }) - } - promoted := false - defer func() { - out.Close() - if promoted { - os.Remove(checkpointPath) - return - } - if !keepPartial { - os.Remove(stagedPath) - os.Remove(checkpointPath) - } - }() - - var written int64 - var validator string - var contentLength int64 - if checkpointOK { - if info, statErr := out.Stat(); statErr == nil { - written = min(checkpoint.Bytes, info.Size()) - validator = checkpoint.Validator - contentLength = checkpoint.Total - if truncateErr := out.Truncate(written); truncateErr != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to restore partial download: %v", truncateErr), - }) - } - } - } - if _, err := out.Seek(written, io.SeekStart); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to seek staged file: %v", err), - }) - } - - activeItemID := r.getActiveDownloadItemID() - if activeItemID != "" { - SetItemDownloading(activeItemID) - } - shouldTrackItemBytes := activeItemID != "" && trackItemBytes - itemProgressReporter := NewItemTransferProgressReporter(activeItemID, written, contentLength) - if shouldTrackItemBytes { - if contentLength > 0 { - SetItemProgress(activeItemID, float64(written)/float64(contentLength), written, contentLength) - } else if written > 0 { - SetItemBytesReceived(activeItemID, written) - } - } - if checkpointOK && written > 0 && contentLength > 0 && written == contentLength { - // The process may have died after the last durable checkpoint but - // before the atomic rename. Publish that already-complete staged file - // without issuing an unsatisfiable Range request at EOF. - if err := out.Sync(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to sync restored download: %v", err), - }) - } - if err := out.Close(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to close restored download: %v", err), - }) - } - if err := os.Rename(stagedPath, fullPath); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to publish restored download: %v", err), - }) - } - promoted = true - os.Remove(checkpointPath) - syncDir(filepath.Dir(fullPath)) - if shouldTrackItemBytes { - SetItemProgress(activeItemID, 1, written, contentLength) - } - return r.jsSuccess(map[string]any{ - "path": fullPath, - "size": written, - "attempts": 0, - "resumed": true, - }) - } - - config := transferRetryConfig(policy) - retryDelay := config.InitialDelay - var lastFailure transferFailure - attemptsUsed := 0 - var lastProgressNotify int64 - lastCheckpointBytes := written - lastCheckpointAt := time.Now() - saveCheckpoint := func() { - if !keepPartial || validator == "" || written <= 0 { - return - } - // Rate-limit checkpoint attempts too: a transient storage failure must - // not turn every subsequent 64 KiB network read into another fsync. - lastCheckpointBytes = written - lastCheckpointAt = time.Now() - // Persist data before the pointer to it. After a power loss, a valid - // checkpoint must never advertise bytes that were only in page cache. - if syncErr := out.Sync(); syncErr != nil { - return - } - _ = saveTransferCheckpoint(checkpointPath, transferCheckpoint{ - Fingerprint: fingerprint, - Validator: validator, - Bytes: written, - Total: contentLength, - }) - } - - for attempt := 1; attempt <= policy.MaxAttempts; attempt++ { - attemptsUsed = attempt - rangeFrom := int64(0) - if resumeDownload && written > 0 && validator != "" { - rangeFrom = written - } - - req, requestErr := http.NewRequest("GET", urlStr, nil) - if requestErr != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "invalid_request", - Message: requestErr.Error(), - Attempts: attempt, - }) - } - req = r.bindDownloadCancelContext(req) - for key, value := range headers { - req.Header.Set(key, value) - } - if req.Header.Get("User-Agent") == "" { - req.Header.Set("User-Agent", appUserAgent()) - } - if rangeFrom > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", rangeFrom)) - req.Header.Set("If-Range", validator) - } - - retryContext := req.Context() - req, watchdog := bindStallWatchdog(req, downloadStallTimeout) - resp, requestErr := r.doResolutionTransfer(client, req, false) - if requestErr != nil { - stalled := watchdog.stalled.Load() - watchdog.stop() - if activeItemID != "" && isDownloadCancelled(activeItemID) { - return r.jsTransferError(transferFailure{ - ErrorType: "cancelled", - Message: "download cancelled", - Attempts: attempt, - }) - } - message := requestErr.Error() - if stalled { - message = fmt.Sprintf( - "download stalled: no data received for %ds (network timeout)", - int(downloadStallTimeout.Seconds()), - ) - } - lastFailure = transferFailure{ - ErrorType: "transient_network", - Message: message, - Attempts: attempt, - } - if attempt == policy.MaxAttempts || retryContext.Err() != nil { - return r.jsTransferError(lastFailure) - } - if written > 0 && (!resumeDownload || validator == "") { - if err := out.Truncate(0); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to restart transfer: %v", err), - Attempts: attempt, - }) - } - written = 0 - if _, err := out.Seek(0, io.SeekStart); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to seek restarted transfer: %v", err), - Attempts: attempt, - }) - } - } - if err := waitTransferRetry(retryContext, retryDelay); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "cancelled", - Message: "download cancelled", - Attempts: attempt, - }) - } - retryDelay = calculateNextDelay(retryDelay, config) - continue - } - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - retryAfter := retryAfterSeconds(resp) - io.Copy(io.Discard, io.LimitReader(resp.Body, 32*1024)) - resp.Body.Close() - watchdog.stop() - errorType := transferErrorTypeForStatus(resp.StatusCode, policy) - lastFailure = transferFailure{ - ErrorType: errorType, - Message: fmt.Sprintf("HTTP error: %d", resp.StatusCode), - HTTPStatus: resp.StatusCode, - RetryAfterSeconds: retryAfter, - Attempts: attempt, - } - if !retryableTransferStatus(resp.StatusCode) || attempt == policy.MaxAttempts { - return r.jsTransferError(lastFailure) - } - delay := retryDelay - if retryAfter > 0 { - delay = time.Duration(retryAfter) * time.Second - } - if err := waitTransferRetry(retryContext, delay); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "cancelled", - Message: "download cancelled", - Attempts: attempt, - }) - } - retryDelay = calculateNextDelay(retryDelay, config) - continue - } - - if rangeFrom > 0 && resp.StatusCode == http.StatusPartialContent && - !validTransferContentRange(resp, rangeFrom) { - contentRange := resp.Header.Get("Content-Range") - resp.Body.Close() - watchdog.stop() - return r.jsTransferError(transferFailure{ - ErrorType: "integrity_failed", - Message: fmt.Sprintf( - "resume failed: unexpected Content-Range %q at %d bytes", - contentRange, - rangeFrom, - ), - HTTPStatus: resp.StatusCode, - Attempts: attempt, - }) - } - if rangeFrom > 0 && resp.StatusCode == http.StatusPartialContent { - nextValidator := transferResponseValidator(resp.Header) - if nextValidator != "" && validator != "" && nextValidator != validator { - resp.Body.Close() - watchdog.stop() - return r.jsTransferError(transferFailure{ - ErrorType: "integrity_failed", - Message: "resume failed: response validator changed", - HTTPStatus: resp.StatusCode, - Attempts: attempt, - }) - } - } - if rangeFrom > 0 && resp.StatusCode == http.StatusOK { - if err := out.Truncate(0); err != nil { - resp.Body.Close() - watchdog.stop() - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to restart changed transfer: %v", err), - Attempts: attempt, - }) - } - if _, err := out.Seek(0, io.SeekStart); err != nil { - resp.Body.Close() - watchdog.stop() - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to seek restarted transfer: %v", err), - Attempts: attempt, - }) - } - written = 0 - rangeFrom = 0 - } - - if nextValidator := transferResponseValidator(resp.Header); nextValidator != "" { - validator = nextValidator - } - contentLength = transferTotalLength(resp, rangeFrom) - if callerSetRange { - // The caller asked file.download to materialize only this range; the - // response length, not the complete object's Content-Range total, is - // therefore the integrity boundary for the output file. - contentLength = resp.ContentLength - } - if shouldTrackItemBytes && contentLength > 0 { - itemProgressReporter.Report(written, contentLength) - } - var readErr error - buffer := make([]byte, 64*1024) - for { - readCount, bodyErr := resp.Body.Read(buffer) - if readCount > 0 { - watchdog.reset() - writeCount, writeErr := out.Write(buffer[:readCount]) - written += int64(writeCount) - if writeErr != nil || writeCount != readCount { - resp.Body.Close() - watchdog.stop() - if writeErr == nil { - writeErr = io.ErrShortWrite - } - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to write staged file: %v", writeErr), - Attempts: attempt, - }) - } - if shouldTrackItemBytes { - itemProgressReporter.Report(written, contentLength) - } - if onProgress != nil && contentLength > 0 && - (written-lastProgressNotify >= progressUpdateThreshold || written >= contentLength) { - lastProgressNotify = written - _, _ = onProgress( - goja.Undefined(), - r.vm.ToValue(written), - r.vm.ToValue(contentLength), - ) - } - if keepPartial && validator != "" && - (written-lastCheckpointBytes >= transferCheckpointBytes || - time.Since(lastCheckpointAt) >= transferCheckpointPeriod) { - saveCheckpoint() - } - } - if bodyErr != nil { - if bodyErr != io.EOF { - readErr = bodyErr - } - break - } - } - resp.Body.Close() - stalled := watchdog.stalled.Load() - watchdog.stop() - - if readErr == nil && contentLength > 0 && written != contentLength { - readErr = io.ErrUnexpectedEOF - } - if readErr == nil { - break - } - - saveCheckpoint() - message := fmt.Sprintf("failed to read response: %v", readErr) - if stalled { - message = fmt.Sprintf( - "download stalled: no data received for %ds (network timeout)", - int(downloadStallTimeout.Seconds()), - ) - } - lastFailure = transferFailure{ - ErrorType: "transient_network", - Message: message, - Attempts: attempt, - } - if attempt == policy.MaxAttempts || - (activeItemID != "" && isDownloadCancelled(activeItemID)) { - if activeItemID != "" && isDownloadCancelled(activeItemID) { - lastFailure.ErrorType = "cancelled" - lastFailure.Message = "download cancelled" - } - return r.jsTransferError(lastFailure) - } - - if !resumeDownload || validator == "" { - if err := out.Truncate(0); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to restart transfer: %v", err), - Attempts: attempt, - }) - } - if _, err := out.Seek(0, io.SeekStart); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to seek restarted transfer: %v", err), - Attempts: attempt, - }) - } - written = 0 - validator = "" - os.Remove(checkpointPath) - } - if err := waitTransferRetry(retryContext, retryDelay); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "cancelled", - Message: "download cancelled", - Attempts: attempt, - }) - } - retryDelay = calculateNextDelay(retryDelay, config) - } - - if written <= 0 { - return r.jsTransferError(transferFailure{ - ErrorType: "integrity_failed", - Message: "download response was empty", - Attempts: attemptsUsed, - }) - } - if contentLength > 0 && written != contentLength { - return r.jsTransferError(transferFailure{ - ErrorType: "integrity_failed", - Message: fmt.Sprintf( - "download size mismatch: expected %d bytes, wrote %d", - contentLength, - written, - ), - Attempts: attemptsUsed, - }) - } - if err := out.Sync(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to sync staged file: %v", err), - }) - } - if err := out.Close(); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to finalize staged file: %v", err), - }) - } - if err := os.Rename(stagedPath, fullPath); err != nil { - return r.jsTransferError(transferFailure{ - ErrorType: "storage_error", - Message: fmt.Sprintf("failed to publish file: %v", err), - }) - } - promoted = true - os.Remove(checkpointPath) - syncDir(filepath.Dir(fullPath)) - if shouldTrackItemBytes { - if contentLength > 0 { - SetItemProgress(activeItemID, 1, written, contentLength) - } else if written > 0 { - SetItemBytesReceived(activeItemID, written) - } - } - - GoLog( - "[Extension:%s] Reliable transfer downloaded %d bytes to %s\n", - r.extensionID, - written, - fullPath, - ) - return r.jsSuccess(map[string]any{ - "path": fullPath, - "size": written, - "attempts": attemptsUsed, - }) -} diff --git a/go_backend/extension_runtime_utils.go b/go_backend/extension_runtime_utils.go deleted file mode 100644 index a9db6d5e..00000000 --- a/go_backend/extension_runtime_utils.go +++ /dev/null @@ -1,561 +0,0 @@ -package gobackend - -import ( - "context" - "crypto/hmac" - "crypto/md5" - "crypto/rand" - "crypto/sha1" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "math" - "reflect" - "strings" - "time" - - "github.com/dop251/goja" -) - -// jsError returns the standard {"success": false, "error": ...} extension -// response. -func (r *extensionRuntime) jsError(format string, args ...any) goja.Value { - return r.vm.ToValue(map[string]any{ - "success": false, - "error": fmt.Sprintf(format, args...), - }) -} - -// jsSuccess returns kv with "success": true added. -func (r *extensionRuntime) jsSuccess(kv map[string]any) goja.Value { - if kv == nil { - kv = map[string]any{} - } - kv["success"] = true - return r.vm.ToValue(kv) -} - -func (r *extensionRuntime) base64Encode(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue("") - } - input := call.Arguments[0].String() - return r.vm.ToValue(base64.StdEncoding.EncodeToString([]byte(input))) -} - -func (r *extensionRuntime) base64Decode(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue("") - } - input := call.Arguments[0].String() - decoded, err := base64.StdEncoding.DecodeString(input) - if err != nil { - return r.vm.ToValue("") - } - return r.vm.ToValue(string(decoded)) -} - -func (r *extensionRuntime) md5Hash(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue("") - } - input := call.Arguments[0].String() - hash := md5.Sum([]byte(input)) - return r.vm.ToValue(hex.EncodeToString(hash[:])) -} - -func (r *extensionRuntime) sha256Hash(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue("") - } - input := call.Arguments[0].String() - hash := sha256.Sum256([]byte(input)) - return r.vm.ToValue(hex.EncodeToString(hash[:])) -} - -func (r *extensionRuntime) hmacSHA256(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.vm.ToValue("") - } - message := call.Arguments[0].String() - key := call.Arguments[1].String() - - mac := hmac.New(sha256.New, []byte(key)) - mac.Write([]byte(message)) - return r.vm.ToValue(hex.EncodeToString(mac.Sum(nil))) -} - -func (r *extensionRuntime) hmacSHA256Base64(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.vm.ToValue("") - } - message := call.Arguments[0].String() - key := call.Arguments[1].String() - - mac := hmac.New(sha256.New, []byte(key)) - mac.Write([]byte(message)) - return r.vm.ToValue(base64.StdEncoding.EncodeToString(mac.Sum(nil))) -} - -func (r *extensionRuntime) hmacSHA1(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.vm.ToValue([]byte{}) - } - - var keyBytes []byte - keyArg := call.Arguments[0].Export() - switch k := keyArg.(type) { - case string: - keyBytes = []byte(k) - case []any: - keyBytes = make([]byte, len(k)) - for i, v := range k { - if num, ok := v.(int64); ok { - keyBytes[i] = byte(num) - } else if num, ok := v.(float64); ok { - keyBytes[i] = byte(int(num)) - } - } - default: - return r.vm.ToValue([]byte{}) - } - - var msgBytes []byte - msgArg := call.Arguments[1].Export() - switch m := msgArg.(type) { - case string: - msgBytes = []byte(m) - case []any: - msgBytes = make([]byte, len(m)) - for i, v := range m { - if num, ok := v.(int64); ok { - msgBytes[i] = byte(num) - } else if num, ok := v.(float64); ok { - msgBytes[i] = byte(int(num)) - } - } - default: - return r.vm.ToValue([]byte{}) - } - - mac := hmac.New(sha1.New, keyBytes) - mac.Write(msgBytes) - result := mac.Sum(nil) - - jsArray := make([]any, len(result)) - for i, b := range result { - jsArray[i] = int(b) - } - return r.vm.ToValue(jsArray) -} - -func (r *extensionRuntime) parseJSON(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return goja.Undefined() - } - input := call.Arguments[0].String() - - var result any - if err := json.Unmarshal([]byte(input), &result); err != nil { - GoLog("[Extension:%s] JSON parse error: %v\n", r.extensionID, err) - return goja.Undefined() - } - - return r.vm.ToValue(result) -} - -func (r *extensionRuntime) stringifyJSON(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue("") - } - input := call.Arguments[0].Export() - - data, err := json.Marshal(input) - if err != nil { - GoLog("[Extension:%s] JSON stringify error: %v\n", r.extensionID, err) - return r.vm.ToValue("") - } - - return r.vm.ToValue(string(data)) -} - -func (r *extensionRuntime) cryptoEncrypt(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.jsError("plaintext and key are required") - } - - plaintext := call.Arguments[0].String() - keyStr := call.Arguments[1].String() - - keyHash := sha256.Sum256([]byte(keyStr)) - - encrypted, err := encryptAES([]byte(plaintext), keyHash[:]) - if err != nil { - return r.jsError("%s", err.Error()) - } - - return r.jsSuccess(map[string]any{ - "data": base64.StdEncoding.EncodeToString(encrypted), - }) -} - -func (r *extensionRuntime) cryptoDecrypt(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.jsError("ciphertext and key are required") - } - - ciphertextB64 := call.Arguments[0].String() - keyStr := call.Arguments[1].String() - - ciphertext, err := base64.StdEncoding.DecodeString(ciphertextB64) - if err != nil { - return r.jsError("invalid base64 ciphertext") - } - - keyHash := sha256.Sum256([]byte(keyStr)) - - decrypted, err := decryptAES(ciphertext, keyHash[:]) - if err != nil { - return r.jsError("invalid base64 ciphertext") - } - - return r.jsSuccess(map[string]any{ - "data": string(decrypted), - }) -} - -func (r *extensionRuntime) cryptoGenerateKey(call goja.FunctionCall) goja.Value { - length := 32 - if len(call.Arguments) > 0 && !goja.IsUndefined(call.Arguments[0]) { - requested := call.Arguments[0].ToFloat() - if math.IsNaN(requested) || math.IsInf(requested, 0) || - requested != math.Trunc(requested) || requested < 1 || requested > 4096 { - return r.jsError("key length must be an integer between 1 and 4096 bytes") - } - length = int(requested) - } - - key := make([]byte, length) - if _, err := rand.Read(key); err != nil { - return r.jsError("%s", err.Error()) - } - - return r.jsSuccess(map[string]any{ - "key": base64.StdEncoding.EncodeToString(key), - "hex": hex.EncodeToString(key), - }) -} - -func (r *extensionRuntime) randomUserAgent(call goja.FunctionCall) goja.Value { - return r.vm.ToValue(getRandomUserAgent()) -} - -func (r *extensionRuntime) appVersion(call goja.FunctionCall) goja.Value { - return r.vm.ToValue(GetAppVersion()) -} - -func (r *extensionRuntime) appUserAgent(call goja.FunctionCall) goja.Value { - return r.vm.ToValue(appUserAgent()) -} - -func (r *extensionRuntime) sleep(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return r.vm.ToValue(true) - } - - sleepMs := 0 - switch value := call.Arguments[0].Export().(type) { - case int64: - sleepMs = int(value) - case int32: - sleepMs = int(value) - case int: - sleepMs = value - case float64: - sleepMs = int(value) - default: - sleepMs = 0 - } - - if sleepMs <= 0 { - return r.vm.ToValue(true) - } - if sleepMs > 5*60*1000 { - sleepMs = 5 * 60 * 1000 - } - - ctx := r.activeOperationContext(context.Background()) - timer := time.NewTimer(time.Duration(sleepMs) * time.Millisecond) - defer timer.Stop() - // A pending cancellation sentinel can precede context initialization. - // Preserve its visibility for utility callers outside DownloadPrepared. - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() - itemID := r.getActiveDownloadItemID() - requestID := r.getActiveRequestID() - for { - if (itemID != "" && isDownloadCancelled(itemID)) || - (requestID != "" && isExtensionRequestCancelled(requestID)) { - return r.vm.ToValue(false) - } - select { - case <-ctx.Done(): - return r.vm.ToValue(false) - case <-timer.C: - return r.vm.ToValue(true) - case <-ticker.C: - } - } -} - -func (r *extensionRuntime) isDownloadCancelled(call goja.FunctionCall) goja.Value { - itemID := r.getActiveDownloadItemID() - if itemID == "" { - return r.vm.ToValue(false) - } - return r.vm.ToValue(isDownloadCancelled(itemID)) -} - -func (r *extensionRuntime) isRequestCancelled(call goja.FunctionCall) goja.Value { - requestID := r.getActiveRequestID() - if requestID == "" { - return r.vm.ToValue(false) - } - return r.vm.ToValue(isExtensionRequestCancelled(requestID)) -} - -func (r *extensionRuntime) setDownloadStatus(call goja.FunctionCall) goja.Value { - itemID := r.getActiveDownloadItemID() - if itemID == "" || len(call.Arguments) < 1 { - return goja.Undefined() - } - - status := strings.ToLower(strings.TrimSpace(call.Arguments[0].String())) - switch status { - case itemProgressStatusPreparing: - SetItemPreparing(itemID) - case itemProgressStatusDownloading: - SetItemDownloading(itemID) - case itemProgressStatusFinalizing: - SetItemFinalizing(itemID) - } - - return goja.Undefined() -} - -func (r *extensionRuntime) logDebug(call goja.FunctionCall) goja.Value { - msg := r.formatLogArgs(call.Arguments) - GoLog("[Extension:%s:DEBUG] %s\n", r.extensionID, msg) - return goja.Undefined() -} - -func (r *extensionRuntime) logInfo(call goja.FunctionCall) goja.Value { - msg := r.formatLogArgs(call.Arguments) - GoLog("[Extension:%s:INFO] %s\n", r.extensionID, msg) - return goja.Undefined() -} - -func (r *extensionRuntime) logWarn(call goja.FunctionCall) goja.Value { - msg := r.formatLogArgs(call.Arguments) - GoLog("[Extension:%s:WARN] %s\n", r.extensionID, msg) - return goja.Undefined() -} - -func (r *extensionRuntime) logError(call goja.FunctionCall) goja.Value { - msg := r.formatLogArgs(call.Arguments) - GoLog("[Extension:%s:ERROR] %s\n", r.extensionID, msg) - return goja.Undefined() -} - -func (r *extensionRuntime) formatLogArgs(args []goja.Value) string { - return formatExtensionLogArgs(args) -} - -const ( - maxExtensionLogArgs = 8 - maxExtensionLogArgLength = 512 -) - -func formatExtensionLogArgs(args []goja.Value) string { - limit := len(args) - if limit > maxExtensionLogArgs { - limit = maxExtensionLogArgs - } - parts := make([]string, 0, limit+1) - for _, arg := range args[:limit] { - value := "<value>" - if exportType := arg.ExportType(); exportType != nil { - switch exportType.Kind() { - case reflect.Bool, - reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64, - reflect.String: - value = arg.String() - default: - value = "<" + exportType.String() + ">" - } - } - if len(value) > maxExtensionLogArgLength { - value = value[:maxExtensionLogArgLength] + "...[truncated]" - } - parts = append(parts, value) - } - if len(args) > limit { - parts = append(parts, fmt.Sprintf("...[%d more args]", len(args)-limit)) - } - return truncateLogMessage(sanitizeSensitiveLogText(strings.Join(parts, " "))) -} - -func (r *extensionRuntime) RegisterGoBackendAPIs(vm *goja.Runtime) { - gobackendObj := vm.Get("gobackend") - if gobackendObj == nil || goja.IsUndefined(gobackendObj) { - gobackendObj = vm.NewObject() - vm.Set("gobackend", gobackendObj) - } - - obj := gobackendObj.(*goja.Object) - - obj.Set("sanitizeFilename", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return vm.ToValue("") - } - return vm.ToValue(sanitizeFilename(call.Arguments[0].String())) - }) - - obj.Set("getAudioQuality", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 1 { - return vm.ToValue(map[string]any{ - "error": "file path is required", - }) - } - - filePath := call.Arguments[0].String() - quality, err := GetAudioQuality(filePath) - if err != nil { - return vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - - return vm.ToValue(map[string]any{ - "bitDepth": quality.BitDepth, - "sampleRate": quality.SampleRate, - "totalSamples": quality.TotalSamples, - "duration": quality.Duration, - "codec": quality.Codec, - }) - }) - - obj.Set("getLyricsLRC", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 3 { - return vm.ToValue(map[string]any{ - "error": "spotifyID, trackName, and artistName are required", - }) - } - - spotifyID := strings.TrimSpace(call.Arguments[0].String()) - trackName := strings.TrimSpace(call.Arguments[1].String()) - artistName := strings.TrimSpace(call.Arguments[2].String()) - filePath := "" - if len(call.Arguments) > 3 && !goja.IsUndefined(call.Arguments[3]) && !goja.IsNull(call.Arguments[3]) { - filePath = strings.TrimSpace(call.Arguments[3].String()) - } - var durationMs int64 - if len(call.Arguments) > 4 && !goja.IsUndefined(call.Arguments[4]) && !goja.IsNull(call.Arguments[4]) { - durationMs = call.Arguments[4].ToInteger() - } - - lyrics, err := GetLyricsLRC(spotifyID, trackName, artistName, filePath, durationMs) - if err != nil { - return vm.ToValue(map[string]any{ - "error": err.Error(), - }) - } - - return vm.ToValue(map[string]any{ - "lyrics": lyrics, - }) - }) - - obj.Set("checkISRCExists", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return vm.ToValue(map[string]any{ - "error": "outputDir and isrc are required", - }) - } - - outputDir := strings.TrimSpace(call.Arguments[0].String()) - isrc := strings.TrimSpace(call.Arguments[1].String()) - if outputDir == "" || isrc == "" { - return vm.ToValue(map[string]any{ - "error": "outputDir and isrc are required", - }) - } - - filePath, exists := checkISRCExistsInternal(outputDir, isrc) - return vm.ToValue(map[string]any{ - "exists": exists, - "filePath": filePath, - }) - }) - - obj.Set("addToISRCIndex", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 3 { - return vm.ToValue(map[string]any{ - "error": "outputDir, isrc, and filePath are required", - }) - } - - outputDir := strings.TrimSpace(call.Arguments[0].String()) - isrc := strings.TrimSpace(call.Arguments[1].String()) - filePath := strings.TrimSpace(call.Arguments[2].String()) - if outputDir == "" || isrc == "" || filePath == "" { - return vm.ToValue(map[string]any{ - "error": "outputDir, isrc, and filePath are required", - }) - } - - AddToISRCIndex(outputDir, isrc, filePath) - return vm.ToValue(map[string]any{ - "success": true, - }) - }) - - obj.Set("buildFilename", func(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return vm.ToValue("") - } - - template := call.Arguments[0].String() - metadataObj := call.Arguments[1].Export() - - metadata, ok := metadataObj.(map[string]any) - if !ok { - return vm.ToValue("") - } - - return vm.ToValue(buildFilenameFromTemplate(template, metadata)) - }) - - obj.Set("getLocalTime", func(call goja.FunctionCall) goja.Value { - now := time.Now() - _, offsetSeconds := now.Zone() - offsetMinutes := offsetSeconds / 60 - - return vm.ToValue(map[string]any{ - "year": now.Year(), - "month": int(now.Month()), - "day": now.Day(), - "hour": now.Hour(), - "minute": now.Minute(), - "second": now.Second(), - "weekday": int(now.Weekday()), - "offsetMinutes": -offsetMinutes, // JS convention: negative for east of UTC - "timezone": now.Location().String(), - "timestamp": now.Unix(), - }) - }) -} diff --git a/go_backend/extension_secret_key.go b/go_backend/extension_secret_key.go deleted file mode 100644 index 721be5a4..00000000 --- a/go_backend/extension_secret_key.go +++ /dev/null @@ -1,54 +0,0 @@ -package gobackend - -import ( - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "fmt" - "sync" -) - -const extensionStorageMasterKeyBytes = 32 - -var extensionStorageKeyState struct { - sync.RWMutex - key []byte -} - -// SetExtensionStorageMasterKey installs the platform-keystore-backed key used -// to encrypt extension settings and credentials. The key itself is never -// persisted by Go; Flutter stores it in Keychain/Android Keystore. -func SetExtensionStorageMasterKey(encodedKey string) error { - key, err := base64.StdEncoding.DecodeString(encodedKey) - if err != nil || len(key) != extensionStorageMasterKeyBytes { - return fmt.Errorf("extension storage master key must be 32 base64-encoded bytes") - } - extensionStorageKeyState.Lock() - extensionStorageKeyState.key = append(extensionStorageKeyState.key[:0], key...) - extensionStorageKeyState.Unlock() - return nil -} - -func extensionStorageKeyConfigured() bool { - extensionStorageKeyState.RLock() - configured := len(extensionStorageKeyState.key) == extensionStorageMasterKeyBytes - extensionStorageKeyState.RUnlock() - return configured -} - -func deriveExtensionStorageKey(extensionID, purpose string) ([]byte, error) { - extensionStorageKeyState.RLock() - if len(extensionStorageKeyState.key) != extensionStorageMasterKeyBytes { - extensionStorageKeyState.RUnlock() - return nil, fmt.Errorf("extension storage master key is not configured") - } - masterKey := append([]byte(nil), extensionStorageKeyState.key...) - extensionStorageKeyState.RUnlock() - - mac := hmac.New(sha256.New, masterKey) - _, _ = mac.Write([]byte("SpotiFLAC Mobile extension storage v2\x00")) - _, _ = mac.Write([]byte(purpose)) - _, _ = mac.Write([]byte{0}) - _, _ = mac.Write([]byte(extensionID)) - return mac.Sum(nil), nil -} diff --git a/go_backend/extension_settings.go b/go_backend/extension_settings.go deleted file mode 100644 index a1ba1cd7..00000000 --- a/go_backend/extension_settings.go +++ /dev/null @@ -1,249 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "sync" -) - -type ExtensionSettingsStore struct { - mu sync.RWMutex - dataDir string - settings map[string]map[string]any // extensionID -> settings -} - -var ( - globalSettingsStore *ExtensionSettingsStore - globalSettingsStoreOnce sync.Once -) - -func GetExtensionSettingsStore() *ExtensionSettingsStore { - globalSettingsStoreOnce.Do(func() { - globalSettingsStore = &ExtensionSettingsStore{ - settings: make(map[string]map[string]any), - } - }) - return globalSettingsStore -} - -func (s *ExtensionSettingsStore) SetDataDir(dataDir string) error { - s.mu.Lock() - defer s.mu.Unlock() - if !extensionStorageKeyConfigured() { - return fmt.Errorf("extension storage master key is not configured") - } - - s.dataDir = dataDir - if err := os.MkdirAll(dataDir, 0700); err != nil { - return fmt.Errorf("failed to create settings directory: %w", err) - } - - return s.loadAllSettings() -} - -func (s *ExtensionSettingsStore) getSettingsPath(extensionID string) string { - return filepath.Join(s.dataDir, extensionID, "settings.enc") -} - -func (s *ExtensionSettingsStore) getLegacySettingsPath(extensionID string) string { - return filepath.Join(s.dataDir, extensionID, "settings.json") -} - -func (s *ExtensionSettingsStore) loadAllSettings() error { - entries, err := os.ReadDir(s.dataDir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - - for _, entry := range entries { - if entry.IsDir() { - extensionID := entry.Name() - settings, err := s.loadSettings(extensionID) - if err != nil { - GoLog("[ExtensionSettings] Failed to load settings for %s: %v\n", extensionID, err) - continue - } - s.settings[extensionID] = settings - } - } - - return nil -} - -func (s *ExtensionSettingsStore) loadSettings(extensionID string) (map[string]any, error) { - settingsPath := s.getSettingsPath(extensionID) - data, err := os.ReadFile(settingsPath) - if err != nil { - if os.IsNotExist(err) { - return s.migrateLegacySettings(extensionID) - } - return nil, err - } - key, err := deriveExtensionStorageKey(extensionID, "settings") - if err != nil { - return nil, err - } - data, err = decryptAES(data, key) - if err != nil { - return nil, fmt.Errorf("failed to decrypt settings: %w", err) - } - - var settings map[string]any - if err := json.Unmarshal(data, &settings); err != nil { - return nil, err - } - - return settings, nil -} - -func (s *ExtensionSettingsStore) migrateLegacySettings(extensionID string) (map[string]any, error) { - legacyPath := s.getLegacySettingsPath(extensionID) - data, err := os.ReadFile(legacyPath) - if err != nil { - if os.IsNotExist(err) { - return make(map[string]any), nil - } - return nil, err - } - settings := make(map[string]any) - if err := json.Unmarshal(data, &settings); err != nil { - return nil, fmt.Errorf("failed to read legacy settings: %w", err) - } - if err := s.saveSettings(extensionID, settings); err != nil { - return nil, fmt.Errorf("failed to encrypt legacy settings: %w", err) - } - if err := os.Remove(legacyPath); err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("failed to remove legacy settings: %w", err) - } - return settings, nil -} - -func (s *ExtensionSettingsStore) saveSettings(extensionID string, settings map[string]any) error { - settingsPath := s.getSettingsPath(extensionID) - - dir := filepath.Dir(settingsPath) - if err := os.MkdirAll(dir, 0700); err != nil { - return err - } - - data, err := json.Marshal(settings) - if err != nil { - return err - } - key, err := deriveExtensionStorageKey(extensionID, "settings") - if err != nil { - return err - } - data, err = encryptAES(data, key) - if err != nil { - return err - } - fileMu := extensionFileMu(settingsPath) - fileMu.Lock() - err = writeExtensionFileLocked(settingsPath, data) - fileMu.Unlock() - return err -} - -func (s *ExtensionSettingsStore) Get(extensionID, key string) (any, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - extSettings, exists := s.settings[extensionID] - if !exists { - return nil, fmt.Errorf("extension '%s' settings not found", extensionID) - } - - value, exists := extSettings[key] - if !exists { - return nil, fmt.Errorf("setting '%s' not found for extension '%s'", key, extensionID) - } - return value, nil -} - -func (s *ExtensionSettingsStore) GetAll(extensionID string) map[string]any { - s.mu.RLock() - defer s.mu.RUnlock() - - extSettings, exists := s.settings[extensionID] - if !exists { - return make(map[string]any) - } - - result := make(map[string]any) - for k, v := range extSettings { - result[k] = v - } - return result -} - -func (s *ExtensionSettingsStore) Set(extensionID, key string, value any) error { - s.mu.Lock() - defer s.mu.Unlock() - - if _, exists := s.settings[extensionID]; !exists { - s.settings[extensionID] = make(map[string]any) - } - - s.settings[extensionID][key] = value - - return s.saveSettings(extensionID, s.settings[extensionID]) -} - -func (s *ExtensionSettingsStore) SetAll(extensionID string, settings map[string]any) error { - s.mu.Lock() - defer s.mu.Unlock() - - s.settings[extensionID] = settings - - return s.saveSettings(extensionID, settings) -} - -func (s *ExtensionSettingsStore) Remove(extensionID, key string) error { - s.mu.Lock() - defer s.mu.Unlock() - - extSettings, exists := s.settings[extensionID] - if !exists { - return nil - } - - delete(extSettings, key) - - return s.saveSettings(extensionID, extSettings) -} - -func (s *ExtensionSettingsStore) RemoveAll(extensionID string) error { - s.mu.Lock() - defer s.mu.Unlock() - - delete(s.settings, extensionID) - - settingsPath := s.getSettingsPath(extensionID) - if err := os.Remove(settingsPath); err != nil && !os.IsNotExist(err) { - return err - } - legacyPath := s.getLegacySettingsPath(extensionID) - if err := os.Remove(legacyPath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to remove legacy extension settings: %w", err) - } - - return nil -} - -func (s *ExtensionSettingsStore) GetAllExtensionSettingsJSON() (string, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - data, err := json.Marshal(s.settings) - if err != nil { - return "", err - } - - return string(data), nil -} diff --git a/go_backend/extension_signed_session.go b/go_backend/extension_signed_session.go deleted file mode 100644 index bf84c42b..00000000 --- a/go_backend/extension_signed_session.go +++ /dev/null @@ -1,1659 +0,0 @@ -package gobackend - -import ( - "bytes" - "context" - "crypto/hmac" - "crypto/rand" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "sync" - "time" - - "github.com/dop251/goja" -) - -const signedSessionRefreshSkew = time.Hour - -const ( - signedSessionExchangeMaxAttempts = 3 - signedSessionMaxRetryAfter = 5 * time.Minute - signedSessionMaxSessionRetries = 1 - signedSessionMaxProviderRetries = 2 - signedSessionProviderRetryDelay = time.Second -) - -var ( - pendingSignedSessionGrants = make(map[string]string) - pendingSignedSessionGrantsMu sync.Mutex - signedSessionCoordinators sync.Map - signedSessionRetryWaitContext = sleepRetry - signedSessionProviderWait = sleepRetry - signedSessionRequestNow = time.Now -) - -const signedSessionExchangeTimeout = DefaultJSTimeout - -var sessionHintPattern = regexp.MustCompile(`^[0-9a-f]{32}$`) - -type signedSessionHints struct { - Default string `json:"d"` - Values map[string]string `json:"s"` -} - -var signedSessionHintState = struct { - sync.RWMutex - state signedSessionHints -}{ - state: signedSessionHints{Values: map[string]string{}}, -} - -func SetRuntimeState(raw string) { - next := signedSessionHints{Values: map[string]string{}} - if err := json.Unmarshal([]byte(raw), &next); err != nil { - next = signedSessionHints{Values: map[string]string{}} - } - next.Default = normalizeSessionHint(next.Default) - values := make(map[string]string, len(next.Values)) - for key, value := range next.Values { - key = filepath.Base(strings.TrimSpace(key)) - value = normalizeSessionHint(value) - if key != "" && key != "." && value != "" { - values[key] = value - } - } - next.Values = values - - signedSessionHintState.Lock() - signedSessionHintState.state = next - signedSessionHintState.Unlock() -} - -func normalizeSessionHint(value string) string { - value = strings.ToLower(strings.TrimSpace(value)) - if !sessionHintPattern.MatchString(value) { - return "" - } - return value -} - -func signedSessionHintFor(path string) string { - key := filepath.Base(strings.TrimSpace(path)) - signedSessionHintState.RLock() - defer signedSessionHintState.RUnlock() - if value := signedSessionHintState.state.Values[key]; value != "" { - return value - } - return signedSessionHintState.state.Default -} - -// signedSessionCoordinator serializes authentication state for every runtime -// that shares one persisted signed-session file. Parallel downloads use -// isolated extension runtimes, so a runtime-local mutex cannot prevent two -// bootstraps or an old 401 response from overwriting a newly exchanged session. -type signedSessionCoordinator struct { - mu sync.Mutex - - authURL string - callbackURL string - callbackState string - challengeCreatedAt time.Time - pendingExtensionIDs map[string]struct{} - completedGrantHash string - blockedGeneration string - clearGeneration uint64 - - // exchangeInFlight serializes grant exchanges without keeping mu held over - // HTTP or Retry-After backoff. Waiters observe the completion channel and - // retry their state check after the owner commits or fails. - exchangeInFlight bool - exchangeDone chan struct{} - bootstrapInFlight bool - bootstrapDone chan struct{} - bootstrapErr error - refreshInFlight bool - refreshDone chan struct{} - refreshErr error -} - -func (c *signedSessionCoordinator) beginExchange(ctx context.Context) (func(), error) { - if ctx == nil { - ctx = context.Background() - } - for { - c.mu.Lock() - if !c.exchangeInFlight { - c.exchangeInFlight = true - c.exchangeDone = make(chan struct{}) - done := c.exchangeDone - c.mu.Unlock() - return func() { - c.mu.Lock() - if c.exchangeInFlight && c.exchangeDone == done { - c.exchangeInFlight = false - c.exchangeDone = nil - close(done) - } - c.mu.Unlock() - }, nil - } - done := c.exchangeDone - c.mu.Unlock() - - select { - case <-done: - case <-ctx.Done(): - return nil, ctx.Err() - } - } -} - -func (r *extensionRuntime) signedSessionCoordinator(config SignedSessionConfig) (*signedSessionCoordinator, error) { - path, err := r.signedSessionFilePath(config) - if err != nil { - return nil, err - } - value, _ := signedSessionCoordinators.LoadOrStore(path, &signedSessionCoordinator{}) - return value.(*signedSessionCoordinator), nil -} - -func (c *signedSessionCoordinator) clearChallenge() { - for extensionID := range c.pendingExtensionIDs { - ClearPendingAuthRequest(extensionID) - } - c.authURL = "" - c.callbackURL = "" - c.callbackState = "" - c.challengeCreatedAt = time.Time{} - c.pendingExtensionIDs = nil -} - -func (c *signedSessionCoordinator) rememberChallenge(request *PendingAuthRequest) { - if c.pendingExtensionIDs == nil { - c.pendingExtensionIDs = make(map[string]struct{}) - } - c.authURL = request.AuthURL - c.callbackURL = request.CallbackURL - c.callbackState = request.State - // Reusers must register the exact same challenge identity, including its - // original timestamp. A fresh time both breaks nonce sharing and extends TTL. - c.challengeCreatedAt = request.CreatedAt - c.pendingExtensionIDs[request.ExtensionID] = struct{}{} -} - -func (c *signedSessionCoordinator) activeChallenge() bool { - return strings.TrimSpace(c.authURL) != "" && - !c.challengeCreatedAt.IsZero() && - time.Since(c.challengeCreatedAt) < pendingAuthRequestTTL -} - -func signedSessionGeneration(record *signedSessionRecord) string { - if record == nil || record.SessionID == "" || record.SessionSecret == "" { - return "" - } - sum := sha256.Sum256([]byte(record.SessionID + "\n" + record.SessionSecret)) - return hex.EncodeToString(sum[:]) -} - -func (c *signedSessionCoordinator) blockGeneration(record *signedSessionRecord) { - c.blockedGeneration = signedSessionGeneration(record) -} - -func (c *signedSessionCoordinator) generationIsBlocked(record *signedSessionRecord) bool { - generation := signedSessionGeneration(record) - return generation != "" && generation == c.blockedGeneration -} - -func (c *signedSessionCoordinator) clearBlockedGeneration() { - c.blockedGeneration = "" -} - -type signedSessionRecord struct { - InstallID string `json:"install_id"` - SessionID string `json:"session_id,omitempty"` - SessionSecret string `json:"session_secret,omitempty"` - ExpiresAt string `json:"expires_at,omitempty"` - Namespace string `json:"namespace,omitempty"` - BaseURL string `json:"base_url,omitempty"` - AppVersion string `json:"app_version,omitempty"` - Platform string `json:"platform,omitempty"` -} - -type signedSessionExchangeResponse struct { - SessionID string `json:"session_id,omitempty"` - SessionSecret string `json:"session_secret,omitempty"` - ExpiresAt string `json:"expires_at,omitempty"` - ChallengeID string `json:"challenge_id,omitempty"` - ChallengeURL string `json:"challenge_url,omitempty"` - AuthURL string `json:"auth_url,omitempty"` -} - -// signedSessionErrorContract is the gateway-owned error envelope. Decisions -// that mutate authentication state must use these fields together with the -// HTTP status; a provider response must never be able to masquerade as a -// gateway session failure merely by returning 401 or 403 upstream. -type signedSessionErrorContract struct { - Error string `json:"error,omitempty"` - Code string `json:"code,omitempty"` - Origin string `json:"origin,omitempty"` - Action string `json:"action,omitempty"` - Retryable bool `json:"retryable,omitempty"` - RetryMode string `json:"retry_mode,omitempty"` - RetryAfterSeconds int `json:"retry_after_seconds,omitempty"` -} - -func signedSessionConfigWithDefaults(config *SignedSessionConfig) SignedSessionConfig { - if config == nil { - return SignedSessionConfig{} - } - resolved := *config - if resolved.AppVersion == "" { - resolved.AppVersion = "ext-1.0" - } - if resolved.Platform == "" { - resolved.Platform = "extension" - } - if resolved.CallbackURL == "" { - resolved.CallbackURL = "spotiflac://session-grant" - } - if resolved.SchemeLabel == "" { - resolved.SchemeLabel = "SPOTIFLAC-HMAC-V1" - } - if resolved.HeaderPrefix == "" { - resolved.HeaderPrefix = "X-Sig-" - } - if resolved.TimeWindowSeconds <= 0 { - resolved.TimeWindowSeconds = 300 - } - if resolved.Endpoints.Bootstrap == "" { - resolved.Endpoints.Bootstrap = "/bootstrap" - } - if resolved.Endpoints.Challenge == "" { - resolved.Endpoints.Challenge = "/challenge" - } - if resolved.Endpoints.Exchange == "" { - resolved.Endpoints.Exchange = "/session/exchange" - } - return resolved -} - -func (r *extensionRuntime) signedSessionFilePath(config SignedSessionConfig) (string, error) { - namespace := sanitizeSignedSessionNamespace(config.Namespace) - if namespace == "" { - return "", fmt.Errorf("signed session namespace is empty") - } - baseDir := filepath.Dir(r.dataDir) - if baseDir == "." || baseDir == "" { - baseDir = r.dataDir - } - dir := filepath.Join(baseDir, "signed_sessions") - scope := strings.Join([]string{ - namespace, - strings.TrimSpace(strings.ToLower(config.BaseURL)), - strings.TrimSpace(strings.ToLower(config.AppVersion)), - strings.TrimSpace(strings.ToLower(config.Platform)), - }, "\n") - sum := sha256.Sum256([]byte(scope)) - return filepath.Join(dir, namespace+"-"+hex.EncodeToString(sum[:])[:16]+".json"), nil -} - -func sanitizeSignedSessionNamespace(namespace string) string { - namespace = strings.TrimSpace(strings.ToLower(namespace)) - var b strings.Builder - for _, ch := range namespace { - if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '-' || ch == '_' || ch == '.' { - b.WriteRune(ch) - } - } - return strings.Trim(b.String(), ".-_") -} - -func (r *extensionRuntime) loadSignedSession(config SignedSessionConfig) (*signedSessionRecord, error) { - path, err := r.signedSessionFilePath(config) - if err != nil { - return nil, err - } - record := &signedSessionRecord{} - if data, err := os.ReadFile(path); err == nil { - _ = json.Unmarshal(data, record) - } - changed := false - if strings.TrimSpace(record.InstallID) == "" { - record.InstallID = signedSessionHintFor(path) - if record.InstallID == "" { - record.InstallID = randomHex(16) - } - changed = true - } - if normalizeSignedSessionRecordScope(config, record) { - changed = true - } - // Only rewrite the file when the record actually changed; loads happen on - // every signed request and preflight. - if changed { - if err := r.saveSignedSession(config, record); err != nil { - return nil, err - } - } - return record, nil -} - -// normalizeSignedSessionRecordScope stamps the config scope onto the record, -// resetting the session when the scope changed. Returns whether the record -// was modified. -func normalizeSignedSessionRecordScope(config SignedSessionConfig, record *signedSessionRecord) bool { - namespace := sanitizeSignedSessionNamespace(config.Namespace) - baseURL := strings.TrimSpace(config.BaseURL) - appVersion := strings.TrimSpace(config.AppVersion) - platform := strings.TrimSpace(config.Platform) - if record.Namespace == namespace && - record.BaseURL == baseURL && - record.AppVersion == appVersion && - record.Platform == platform { - return false - } - blankScope := record.Namespace == "" && record.BaseURL == "" && - record.AppVersion == "" && record.Platform == "" - if !blankScope { - record.SessionID = "" - record.SessionSecret = "" - record.ExpiresAt = "" - } - record.Namespace = namespace - record.BaseURL = baseURL - record.AppVersion = appVersion - record.Platform = platform - return true -} - -func (r *extensionRuntime) saveSignedSession(config SignedSessionConfig, record *signedSessionRecord) error { - path, err := r.signedSessionFilePath(config) - if err != nil { - return err - } - if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { - return err - } - data, err := json.MarshalIndent(record, "", " ") - if err != nil { - return err - } - return os.WriteFile(path, data, 0600) -} - -func randomHex(bytesLen int) string { - buf := make([]byte, bytesLen) - if _, err := rand.Read(buf); err != nil { - return fmt.Sprintf("%d", time.Now().UnixNano()) - } - return hex.EncodeToString(buf) -} - -func parseSignedSessionTime(value string) (time.Time, bool) { - value = strings.TrimSpace(value) - if value == "" { - return time.Time{}, false - } - layouts := []string{ - time.RFC3339Nano, - time.RFC3339, - "2006-01-02T15:04:05.000Z", - } - for _, layout := range layouts { - if parsed, err := time.Parse(layout, value); err == nil { - return parsed, true - } - } - return time.Time{}, false -} - -func signedSessionRecordIsUsable(record *signedSessionRecord) bool { - if record == nil || strings.TrimSpace(record.SessionID) == "" || strings.TrimSpace(record.SessionSecret) == "" { - return false - } - if expiresAt, ok := parseSignedSessionTime(record.ExpiresAt); ok { - return time.Now().Before(expiresAt) - } - return true -} - -func sameSignedSession(a, b *signedSessionRecord) bool { - return a != nil && b != nil && - a.SessionID != "" && - a.SessionID == b.SessionID && - a.SessionSecret == b.SessionSecret -} - -func parseSignedSessionErrorContract(body []byte) (signedSessionErrorContract, bool) { - var contract signedSessionErrorContract - if len(body) == 0 || json.Unmarshal(body, &contract) != nil { - return signedSessionErrorContract{}, false - } - contract.Error = strings.TrimSpace(contract.Error) - contract.Code = strings.ToUpper(strings.TrimSpace(contract.Code)) - contract.Origin = strings.ToLower(strings.TrimSpace(contract.Origin)) - contract.Action = strings.ToLower(strings.TrimSpace(contract.Action)) - contract.RetryMode = strings.ToLower(strings.TrimSpace(contract.RetryMode)) - if contract.RetryAfterSeconds < 0 { - contract.RetryAfterSeconds = 0 - } - return contract, contract.Code != "" || contract.Origin != "" || contract.Action != "" -} - -func signedSessionGatewayAction(statusCode int, contract signedSessionErrorContract) string { - if contract.Origin != "gateway" { - return "" - } - switch { - case statusCode == http.StatusUnauthorized && - contract.Code == "SESSION_INVALID" && - contract.Action == "bootstrap_session": - return "bootstrap_session" - case statusCode == http.StatusPreconditionRequired && - contract.Code == "VERIFY_REQUIRED" && - contract.Action == "verify": - return "verify" - default: - return "" - } -} - -func signedSessionSameOperationRetry(statusCode int, contract signedSessionErrorContract) bool { - return statusCode == http.StatusServiceUnavailable && - contract.Origin == "provider" && - contract.Code == "PROVIDER_UNAVAILABLE" && - contract.Retryable && - contract.RetryMode == "same_operation" -} - -func signedSessionRequestAuthInvalid(statusCode int, contract signedSessionErrorContract) bool { - return statusCode == http.StatusForbidden && - contract.Origin == "gateway" && - contract.Code == "REQUEST_AUTH_INVALID" && - contract.Action == "" -} - -func signedSessionProviderRetryDuration(resp *http.Response, contract signedSessionErrorContract) time.Duration { - if retryAfter := getRetryAfterDuration(resp); retryAfter > 0 { - return retryAfter - } - if contract.RetryAfterSeconds > 0 { - maxSeconds := int(maxRetryAfterDelay / time.Second) - return time.Duration(min(contract.RetryAfterSeconds, maxSeconds)) * time.Second - } - return signedSessionProviderRetryDelay -} - -// preflightSignedSession prepares a signed session before download metadata -// enrichment starts. A fresh pending challenge is reused, while bootstrap -// responses that can issue a session silently are accepted without prompting -// the user. Bootstrap failures are returned so callers do not continue into -// the provider and accidentally issue the same failing bootstrap repeatedly. -func (r *extensionRuntime) preflightSignedSession() (bool, error) { - if r == nil || r.manifest == nil || r.manifest.SignedSession == nil { - return false, nil - } - - config := signedSessionConfigWithDefaults(r.manifest.SignedSession) - if config.Namespace == "" || config.BaseURL == "" { - return false, fmt.Errorf("signedSession is not configured") - } - coordinator, err := r.signedSessionCoordinator(config) - if err != nil { - return false, err - } - coordinator.mu.Lock() - defer coordinator.mu.Unlock() - - record, err := r.loadSignedSession(config) - if err != nil { - return false, err - } - if signedSessionRecordIsUsable(record) && - !coordinator.generationIsBlocked(record) { - return false, nil - } - - if authURL, err := r.startSignedSessionVerificationLocked(config, coordinator, "download-preflight"); err != nil { - return false, err - } else if authURL != "" { - return true, nil - } - - // Bootstrap may provision a session directly instead of returning a - // challenge. Reload the record before treating the empty URL as a failure. - record, err = r.loadSignedSession(config) - if err != nil { - return false, err - } - if signedSessionRecordIsUsable(record) { - return false, nil - } - - return false, fmt.Errorf("signed-session bootstrap did not return a session or verification challenge") -} - -func (r *extensionRuntime) signedSessionStatus(call goja.FunctionCall) goja.Value { - config := signedSessionConfigWithDefaults(r.manifest.SignedSession) - if config.Namespace == "" || config.BaseURL == "" { - return r.vm.ToValue(map[string]any{"authenticated": false, "error": "signedSession is not configured"}) - } - coordinator, err := r.signedSessionCoordinator(config) - if err != nil { - return r.vm.ToValue(map[string]any{"authenticated": false, "error": err.Error()}) - } - coordinator.mu.Lock() - defer coordinator.mu.Unlock() - record, err := r.loadSignedSession(config) - if err != nil { - return r.vm.ToValue(map[string]any{"authenticated": false, "error": err.Error()}) - } - blocked := coordinator.generationIsBlocked(record) - authenticated := signedSessionRecordIsUsable(record) && !blocked - return r.vm.ToValue(map[string]any{ - "authenticated": authenticated, - "verification_required": blocked, - "expires_at": record.ExpiresAt, - "install_id": record.InstallID, - "session_id": record.SessionID, - "app_version": config.AppVersion, - "platform": config.Platform, - }) -} - -func (r *extensionRuntime) signedSessionClear(call goja.FunctionCall) goja.Value { - config := signedSessionConfigWithDefaults(r.manifest.SignedSession) - coordinator, err := r.signedSessionCoordinator(config) - if err != nil { - return r.vm.ToValue(map[string]any{"success": false, "error": err.Error()}) - } - coordinator.mu.Lock() - defer coordinator.mu.Unlock() - record, err := r.loadSignedSession(config) - if err != nil { - return r.vm.ToValue(map[string]any{"success": false, "error": err.Error()}) - } - record.SessionID = "" - record.SessionSecret = "" - record.ExpiresAt = "" - if err := r.saveSignedSession(config, record); err != nil { - return r.vm.ToValue(map[string]any{"success": false, "error": err.Error()}) - } - // Invalidate exchanges that released the coordinator lock while performing - // HTTP. A response that started before this explicit clear must never - // resurrect the just-cleared credentials. - coordinator.clearGeneration++ - coordinator.completedGrantHash = "" - coordinator.clearBlockedGeneration() - coordinator.clearChallenge() - ClearPendingAuthRequest(r.extensionID) - return r.vm.ToValue(map[string]any{"success": true}) -} - -func (r *extensionRuntime) signedSessionCompleteGrant(call goja.FunctionCall) goja.Value { - grant := "" - if len(call.Arguments) > 0 { - grant = strings.TrimSpace(call.Arguments[0].String()) - } - if grant != "" { - setPendingSignedSessionGrant(r.extensionID, grant) - } - if grant == "" { - pendingSignedSessionGrantsMu.Lock() - grant = pendingSignedSessionGrants[r.extensionID] - pendingSignedSessionGrantsMu.Unlock() - } - if grant == "" { - return r.vm.ToValue(map[string]any{"success": false, "error": "no pending grant"}) - } - ctx, cancel := r.signedSessionExchangeContext() - defer cancel() - if err := r.exchangeSignedSessionGrantContext(ctx, grant); err != nil { - return r.vm.ToValue(map[string]any{"success": false, "error": err.Error()}) - } - pendingSignedSessionGrantsMu.Lock() - delete(pendingSignedSessionGrants, r.extensionID) - pendingSignedSessionGrantsMu.Unlock() - ClearPendingAuthRequest(r.extensionID) - return r.vm.ToValue(map[string]any{"success": true}) -} - -func (r *extensionRuntime) exchangeSignedSessionGrant(grant string) error { - ctx, cancel := r.signedSessionExchangeContext() - defer cancel() - return r.exchangeSignedSessionGrantContext(ctx, grant) -} - -func (r *extensionRuntime) signedSessionExchangeContext() (context.Context, context.CancelFunc) { - parent := context.Background() - if r != nil { - parent = r.activeOperationContext(parent) - } - return context.WithTimeout(parent, signedSessionExchangeTimeout) -} - -func waitSignedSessionRetry(ctx context.Context, delay time.Duration) error { - if ctx == nil { - ctx = context.Background() - } - return signedSessionRetryWaitContext(ctx, delay) -} - -func (r *extensionRuntime) exchangeSignedSessionGrantContext(ctx context.Context, grant string) error { - if r == nil || r.manifest == nil || r.manifest.SignedSession == nil { - return fmt.Errorf("signedSession is not configured") - } - if r.httpClient == nil { - return fmt.Errorf("signed-session exchange HTTP client is unavailable") - } - config := signedSessionConfigWithDefaults(r.manifest.SignedSession) - coordinator, err := r.signedSessionCoordinator(config) - if err != nil { - return err - } - if ctx == nil { - ctx = context.Background() - } - coordinator.mu.Lock() - clearGeneration := coordinator.clearGeneration - coordinator.mu.Unlock() - release, err := coordinator.beginExchange(ctx) - if err != nil { - return err - } - defer release() - return r.exchangeSignedSessionGrantLocked(ctx, config, coordinator, clearGeneration, grant) -} - -func (r *extensionRuntime) exchangeSignedSessionGrantLocked( - ctx context.Context, - config SignedSessionConfig, - coordinator *signedSessionCoordinator, - clearGeneration uint64, - grant string, -) error { - coordinator.mu.Lock() - if coordinator.clearGeneration != clearGeneration { - coordinator.mu.Unlock() - return fmt.Errorf("signed-session exchange was superseded by session clear") - } - record, err := r.loadSignedSession(config) - if err != nil { - coordinator.mu.Unlock() - return err - } - grantHashBytes := sha256.Sum256([]byte(grant)) - grantHash := hex.EncodeToString(grantHashBytes[:]) - // A duplicated callback may arrive after another runtime already exchanged - // the same one-time grant. Treat that exact shared result as completed - // instead of consuming the grant again. - if coordinator.completedGrantHash == grantHash && - signedSessionRecordIsUsable(record) { - coordinator.clearBlockedGeneration() - coordinator.clearChallenge() - coordinator.mu.Unlock() - return nil - } - endpoint, err := signedSessionURL(config, config.Endpoints.Exchange) - if err != nil { - coordinator.mu.Unlock() - return err - } - payload := map[string]any{ - "grant": grant, - "install_id": record.InstallID, - "app_version": config.AppVersion, - "platform": config.Platform, - } - body, _ := json.Marshal(payload) - coordinator.mu.Unlock() - - var respBody []byte - for attempt := 1; attempt <= signedSessionExchangeMaxAttempts; attempt++ { - req, requestErr := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if requestErr != nil { - return requestErr - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "SpotiFLAC-Mobile/"+config.AppVersion) - resp, requestErr := r.httpClient.Do(req) - if requestErr != nil { - return requestErr - } - respBody, requestErr = readExtensionHTTPResponseBody(resp) - resp.Body.Close() - if requestErr != nil { - return requestErr - } - if resp.StatusCode == http.StatusTooManyRequests && attempt < signedSessionExchangeMaxAttempts { - retryAfter := time.Duration(signedSessionRetryAfterSeconds(resp)) * time.Second - if retryAfter <= 0 { - retryAfter = time.Second - } - if retryAfter > signedSessionMaxRetryAfter { - retryAfter = signedSessionMaxRetryAfter - } - LogWarn( - "SignedSession", - "Grant exchange rate limited for extension %s; retrying in %s (attempt %d/%d)", - r.extensionID, - retryAfter, - attempt+1, - signedSessionExchangeMaxAttempts, - ) - if waitErr := waitSignedSessionRetry(ctx, retryAfter); waitErr != nil { - return waitErr - } - continue - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - message := fmt.Sprintf("session exchange failed: HTTP %d", resp.StatusCode) - if retryAfter := signedSessionRetryAfterSeconds(resp); retryAfter > 0 { - message += fmt.Sprintf("; retry-after seconds: %d", retryAfter) - } - return errors.New(message) - } - break - } - var exchanged signedSessionExchangeResponse - if err := json.Unmarshal(respBody, &exchanged); err != nil { - return fmt.Errorf("invalid session exchange response: %w", err) - } - if exchanged.SessionID == "" || exchanged.SessionSecret == "" || exchanged.ExpiresAt == "" { - return fmt.Errorf("session exchange response missing session fields") - } - - coordinator.mu.Lock() - defer coordinator.mu.Unlock() - if coordinator.clearGeneration != clearGeneration { - return fmt.Errorf("signed-session exchange was superseded by session clear") - } - latest, err := r.loadSignedSession(config) - if err != nil { - return err - } - // Another exchange may have completed while this request was in flight. - // Never overwrite that newer shared session with a stale response. - if coordinator.completedGrantHash == grantHash && - signedSessionRecordIsUsable(latest) { - coordinator.clearBlockedGeneration() - coordinator.clearChallenge() - return nil - } - if signedSessionRecordIsUsable(latest) && !sameSignedSession(latest, record) { - coordinator.clearBlockedGeneration() - coordinator.clearChallenge() - return nil - } - record = latest - record.SessionID = exchanged.SessionID - record.SessionSecret = exchanged.SessionSecret - record.ExpiresAt = exchanged.ExpiresAt - if err := r.saveSignedSession(config, record); err != nil { - return err - } - coordinator.completedGrantHash = grantHash - coordinator.clearBlockedGeneration() - coordinator.clearChallenge() - return nil -} - -func (r *extensionRuntime) signedSessionFetch(call goja.FunctionCall) goja.Value { - if len(call.Arguments) < 2 { - return r.vm.ToValue(map[string]any{"ok": false, "error": "method and path are required"}) - } - config := signedSessionConfigWithDefaults(r.manifest.SignedSession) - if config.Namespace == "" || config.BaseURL == "" { - return r.vm.ToValue(map[string]any{"ok": false, "error": "signedSession is not configured"}) - } - method := strings.ToUpper(strings.TrimSpace(call.Arguments[0].String())) - requestPath := call.Arguments[1].String() - body := []byte{} - if len(call.Arguments) > 2 && !goja.IsUndefined(call.Arguments[2]) && !goja.IsNull(call.Arguments[2]) { - switch v := call.Arguments[2].Export().(type) { - case string: - body = []byte(v) - case map[string]any, []any: - encoded, err := json.Marshal(v) - if err != nil { - return r.vm.ToValue(map[string]any{"ok": false, "error": err.Error()}) - } - body = encoded - default: - body = []byte(call.Arguments[2].String()) - } - } - extraHeaders := parseGojaHeaders(call.Argument(3).Export()) - - coordinator, err := r.signedSessionCoordinator(config) - if err != nil { - return r.vm.ToValue(map[string]any{"ok": false, "error": err.Error()}) - } - coordinator.mu.Lock() - record, err := r.ensureSignedSession(config) - if err != nil { - authURL, verificationErr := r.startSignedSessionVerificationLocked(config, coordinator, "signed-fetch") - coordinator.mu.Unlock() - if authURL != "" { - return r.signedSessionVerificationRequiredValue(authURL) - } else if verificationErr != nil { - return r.vm.ToValue(map[string]any{"ok": false, "error": verificationErr.Error()}) - } - return r.vm.ToValue(map[string]any{"ok": false, "error": err.Error()}) - } - if coordinator.generationIsBlocked(record) { - authURL, verificationErr := r.startSignedSessionVerificationLocked( - config, - coordinator, - "signed-fetch-blocked-generation", - ) - if authURL != "" { - coordinator.mu.Unlock() - return r.signedSessionVerificationRequiredValue(authURL) - } - if verificationErr != nil { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{"ok": false, "error": verificationErr.Error()}) - } - record, err = r.loadSignedSession(config) - if err != nil { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{"ok": false, "error": err.Error()}) - } - if !signedSessionRecordIsUsable(record) || coordinator.generationIsBlocked(record) { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{ - "ok": false, - "error": "verification_required: signed-session generation is blocked", - }) - } - } - coordinator.mu.Unlock() - - // Refresh can involve a slow HTTP request. Coalesce it across parallel - // extension runtimes without keeping the shared coordinator mutex locked, - // then reload the committed generation before signing the request. - if signedSessionRefreshDue(config, record) { - if _, refreshErr := r.refreshSignedSessionCoalesced(config, coordinator); refreshErr != nil { - LogWarn("SignedSession", "Session refresh failed for extension %s: %v", r.extensionID, refreshErr) - } - coordinator.mu.Lock() - latest, loadErr := r.loadSignedSession(config) - if loadErr != nil { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{"ok": false, "error": loadErr.Error()}) - } - if !signedSessionRecordIsUsable(latest) || coordinator.generationIsBlocked(latest) { - authURL, verificationErr := r.startSignedSessionVerificationLocked( - config, - coordinator, - "signed-fetch-refresh", - ) - coordinator.mu.Unlock() - if authURL != "" { - return r.signedSessionVerificationRequiredValue(authURL) - } - if verificationErr != nil { - return r.vm.ToValue(map[string]any{"ok": false, "error": verificationErr.Error()}) - } - return r.vm.ToValue(map[string]any{"ok": false, "error": "signed session is not authenticated"}) - } - record = latest - coordinator.mu.Unlock() - } - - // A request that loses a race with a successful grant exchange may return a - // canonical SESSION_INVALID response for the old secret. Reload and retry - // with the newer shared session; never let that stale response erase its - // replacement. Provider retries have an independent, bounded budget. - sessionRetries := 0 - providerRetries := 0 - requestAuthRetryUsed := false - providerRetryCtx := r.activeOperationContext(context.Background()) - for { - resp, respBody, respHeaders, requestErr := r.doSignedSessionRequest( - config, - record, - method, - requestPath, - body, - extraHeaders, - ) - if requestErr != nil { - return r.vm.ToValue(map[string]any{"ok": false, "error": requestErr.Error()}) - } - contract, _ := parseSignedSessionErrorContract(respBody) - - if signedSessionSameOperationRetry(resp.StatusCode, contract) { - if providerRetries >= signedSessionMaxProviderRetries { - return r.signedSessionResponseValue(resp, respBody, respHeaders) - } - providerRetries++ - delay := signedSessionProviderRetryDuration(resp, contract) - LogWarn( - "SignedSession", - "Provider temporarily unavailable for extension %s; retrying in %s (attempt %d/%d)", - r.extensionID, - delay, - providerRetries+1, - signedSessionMaxProviderRetries+1, - ) - if waitErr := signedSessionProviderWait(providerRetryCtx, delay); waitErr != nil { - return r.vm.ToValue(map[string]any{"ok": false, "error": waitErr.Error()}) - } - continue - } - - if signedSessionRequestAuthInvalid(resp.StatusCode, contract) { - coordinator.mu.Lock() - latest, loadErr := r.loadSignedSession(config) - if loadErr != nil { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{"ok": false, "error": loadErr.Error()}) - } - if !requestAuthRetryUsed && - signedSessionRecordIsUsable(latest) && - !sameSignedSession(latest, record) { - requestAuthRetryUsed = true - record = latest - coordinator.mu.Unlock() - LogDebug( - "SignedSession", - "Retrying stale REQUEST_AUTH_INVALID for extension %s with the current session generation", - r.extensionID, - ) - continue - } - coordinator.mu.Unlock() - LogWarn( - "SignedSession", - "REQUEST_AUTH_INVALID for extension %s on the current session generation; preserving session state", - r.extensionID, - ) - return r.signedSessionResponseValue(resp, respBody, respHeaders) - } - - gatewayAction := signedSessionGatewayAction(resp.StatusCode, contract) - if gatewayAction == "" { - return r.signedSessionResponseValue(resp, respBody, respHeaders) - } - - coordinator.mu.Lock() - latest, loadErr := r.loadSignedSession(config) - if loadErr != nil { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{"ok": false, "error": loadErr.Error()}) - } - if signedSessionRecordIsUsable(latest) && - !sameSignedSession(latest, record) { - if sessionRetries >= signedSessionMaxSessionRetries { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{"ok": false, "error": "signed-session retry limit reached"}) - } - sessionRetries++ - record = latest - coordinator.mu.Unlock() - LogDebug( - "SignedSession", - "Discarding stale %s response for extension %s and retrying with the exchanged session", - contract.Code, - r.extensionID, - ) - continue - } - // VERIFY_REQUIRED is an explicit challenge request, not a revocation. - // Only SESSION_INVALID is allowed to clear the current gateway session. - if gatewayAction == "bootstrap_session" && sameSignedSession(latest, record) { - coordinator.clearBlockedGeneration() - latest.SessionID = "" - latest.SessionSecret = "" - latest.ExpiresAt = "" - if saveErr := r.saveSignedSession(config, latest); saveErr != nil { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{"ok": false, "error": saveErr.Error()}) - } - } else if gatewayAction == "verify" && sameSignedSession(latest, record) { - // Stop subsequent requests from repeatedly hitting the gateway with - // a generation that is known to require human verification. - coordinator.blockGeneration(record) - } - authURL, verificationErr := r.startSignedSessionVerificationLocked( - config, - coordinator, - "signed-fetch-"+gatewayAction, - ) - if authURL != "" { - coordinator.mu.Unlock() - return r.signedSessionVerificationRequiredValue(authURL) - } else if verificationErr != nil { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{"ok": false, "error": verificationErr.Error()}) - } - - // Bootstrap may silently issue a replacement session instead of a - // challenge. Retry the original operation with that generation. - bootstrapped, loadErr := r.loadSignedSession(config) - if loadErr != nil { - coordinator.mu.Unlock() - return r.vm.ToValue(map[string]any{"ok": false, "error": loadErr.Error()}) - } - if signedSessionRecordIsUsable(bootstrapped) && - !sameSignedSession(bootstrapped, record) && - sessionRetries < signedSessionMaxSessionRetries { - sessionRetries++ - record = bootstrapped - coordinator.mu.Unlock() - continue - } - coordinator.mu.Unlock() - return r.signedSessionResponseValue(resp, respBody, respHeaders) - } -} - -func (r *extensionRuntime) signedSessionResponseValue( - resp *http.Response, - respBody []byte, - respHeaders map[string]any, -) goja.Value { - contract, hasContract := parseSignedSessionErrorContract(respBody) - retryAfterSeconds := signedSessionRetryAfterSeconds(resp) - if retryAfterSeconds <= 0 && contract.RetryAfterSeconds > 0 { - retryAfterSeconds = contract.RetryAfterSeconds - } - result := map[string]any{ - "statusCode": resp.StatusCode, - "status": resp.StatusCode, - "ok": resp.StatusCode >= 200 && resp.StatusCode < 300, - "url": resp.Request.URL.String(), - "body": string(respBody), - "headers": respHeaders, - "retryAfterSeconds": retryAfterSeconds, - } - if hasContract { - result["error"] = contract.Error - result["code"] = contract.Code - result["origin"] = contract.Origin - result["action"] = contract.Action - result["retryable"] = contract.Retryable - result["retryMode"] = contract.RetryMode - } - return r.vm.ToValue(result) -} - -func (r *extensionRuntime) signedSessionVerificationRequiredValue(authURL string) goja.Value { - r.noteVerificationRequired(authURL) - return r.vm.ToValue(map[string]any{ - "ok": false, - "needsVerification": true, - "error": "VERIFY_REQUIRED", - "open_auth_url": authURL, - "auth_url": authURL, - }) -} - -func (r *extensionRuntime) ensureSignedSession(config SignedSessionConfig) (*signedSessionRecord, error) { - record, err := r.loadSignedSession(config) - if err != nil { - return nil, err - } - if record.SessionID == "" || record.SessionSecret == "" { - return nil, fmt.Errorf("signed session is not authenticated") - } - if expiresAt, ok := parseSignedSessionTime(record.ExpiresAt); ok { - if time.Now().After(expiresAt) { - record.SessionID = "" - record.SessionSecret = "" - record.ExpiresAt = "" - _ = r.saveSignedSession(config, record) - return nil, fmt.Errorf("signed session expired") - } - } - return record, nil -} - -func signedSessionRefreshDue(config SignedSessionConfig, record *signedSessionRecord) bool { - if config.Endpoints.Refresh == "" || !signedSessionRecordIsUsable(record) { - return false - } - expiresAt, ok := parseSignedSessionTime(record.ExpiresAt) - return ok && time.Now().Before(expiresAt) && time.Until(expiresAt) <= signedSessionRefreshSkew -} - -func (r *extensionRuntime) fetchSignedSessionRefresh( - config SignedSessionConfig, - record *signedSessionRecord, -) (signedSessionExchangeResponse, error) { - var refreshed signedSessionExchangeResponse - body, _ := json.Marshal(map[string]string{"install_id": record.InstallID}) - resp, respBody, _, err := r.doSignedSessionRequest(config, record, http.MethodPost, config.Endpoints.Refresh, body, nil) - if err != nil { - return refreshed, err - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return refreshed, fmt.Errorf("session refresh failed: HTTP %d", resp.StatusCode) - } - if err := json.Unmarshal(respBody, &refreshed); err != nil { - return refreshed, err - } - return refreshed, nil -} - -func applySignedSessionRefresh(record *signedSessionRecord, refreshed signedSessionExchangeResponse) bool { - changed := false - if refreshed.SessionID != "" && refreshed.SessionID != record.SessionID { - record.SessionID = refreshed.SessionID - changed = true - } - if refreshed.SessionSecret != "" && refreshed.SessionSecret != record.SessionSecret { - record.SessionSecret = refreshed.SessionSecret - changed = true - } - if refreshed.ExpiresAt != "" && refreshed.ExpiresAt != record.ExpiresAt { - record.ExpiresAt = refreshed.ExpiresAt - changed = true - } - return changed -} - -func (r *extensionRuntime) refreshSignedSessionCoalesced( - config SignedSessionConfig, - coordinator *signedSessionCoordinator, -) (*signedSessionRecord, error) { - ctx := r.activeOperationContext(context.Background()) - for { - coordinator.mu.Lock() - latest, err := r.loadSignedSession(config) - if err != nil { - coordinator.mu.Unlock() - return nil, err - } - if !signedSessionRefreshDue(config, latest) { - coordinator.mu.Unlock() - return latest, nil - } - if coordinator.refreshInFlight { - done := coordinator.refreshDone - coordinator.mu.Unlock() - select { - case <-done: - coordinator.mu.Lock() - sharedErr := coordinator.refreshErr - sameOperation := coordinator.refreshDone == done - coordinator.mu.Unlock() - if sameOperation && sharedErr != nil { - return nil, sharedErr - } - continue - case <-ctx.Done(): - return nil, ctx.Err() - } - } - - coordinator.refreshInFlight = true - coordinator.refreshDone = make(chan struct{}) - coordinator.refreshErr = nil - done := coordinator.refreshDone - clearGeneration := coordinator.clearGeneration - refreshGeneration := *latest - coordinator.mu.Unlock() - - refreshed, refreshErr := r.fetchSignedSessionRefresh(config, &refreshGeneration) - - coordinator.mu.Lock() - var current *signedSessionRecord - finalErr := refreshErr - if finalErr == nil && coordinator.clearGeneration != clearGeneration { - finalErr = fmt.Errorf("signed-session refresh was superseded by session clear") - } - if finalErr == nil { - var loadErr error - current, loadErr = r.loadSignedSession(config) - finalErr = loadErr - } - if finalErr == nil && sameSignedSession(current, &refreshGeneration) { - if applySignedSessionRefresh(current, refreshed) { - finalErr = r.saveSignedSession(config, current) - } - } - if coordinator.refreshInFlight && coordinator.refreshDone == done { - coordinator.refreshInFlight = false - coordinator.refreshErr = finalErr - close(done) - } - coordinator.mu.Unlock() - return current, finalErr - } -} - -func (r *extensionRuntime) refreshSignedSession(config SignedSessionConfig, record *signedSessionRecord) error { - refreshed, err := r.fetchSignedSessionRefresh(config, record) - if err != nil { - return err - } - if applySignedSessionRefresh(record, refreshed) { - return r.saveSignedSession(config, record) - } - return nil -} - -func (r *extensionRuntime) startSignedSessionVerificationLocked( - config SignedSessionConfig, - coordinator *signedSessionCoordinator, - reason string, -) (string, error) { - if coordinator.activeChallenge() { - if err := registerPendingAuthRequest(&PendingAuthRequest{ - ExtensionID: r.extensionID, - AuthURL: coordinator.authURL, - CallbackURL: coordinator.callbackURL, - State: coordinator.callbackState, - CreatedAt: coordinator.challengeCreatedAt, - }); err != nil { - return "", err - } - coordinator.pendingExtensionIDs[r.extensionID] = struct{}{} - return coordinator.authURL, nil - } - if coordinator.authURL != "" { - coordinator.clearChallenge() - } - if pending := GetPendingAuthRequest(r.extensionID); pending != nil { - if time.Since(pending.CreatedAt) < pendingAuthRequestTTL && - strings.TrimSpace(pending.AuthURL) != "" { - coordinator.rememberChallenge(pending) - return pending.AuthURL, nil - } - ClearPendingAuthRequest(r.extensionID) - } - - record, err := r.loadSignedSession(config) - if err != nil { - return "", fmt.Errorf("load signed-session bootstrap state: %w", err) - } - if signedSessionRecordIsUsable(record) && - !coordinator.generationIsBlocked(record) { - return "", nil - } - if coordinator.bootstrapInFlight { - done := coordinator.bootstrapDone - ctx := r.activeOperationContext(context.Background()) - coordinator.mu.Unlock() - select { - case <-done: - coordinator.mu.Lock() - sharedErr := coordinator.bootstrapErr - sameOperation := coordinator.bootstrapDone == done - if sameOperation && sharedErr != nil { - return "", sharedErr - } - return r.startSignedSessionVerificationLocked(config, coordinator, reason) - case <-ctx.Done(): - coordinator.mu.Lock() - return "", ctx.Err() - } - } - - coordinator.bootstrapInFlight = true - coordinator.bootstrapDone = make(chan struct{}) - coordinator.bootstrapErr = nil - bootstrapDone := coordinator.bootstrapDone - clearGeneration := coordinator.clearGeneration - ctx := r.activeOperationContext(context.Background()) - coordinator.mu.Unlock() - bootstrap, bootstrapErr := r.performSignedSessionBootstrap(ctx, config, record, reason) - coordinator.mu.Lock() - finalErr := bootstrapErr - authURL := "" - if finalErr == nil && coordinator.clearGeneration != clearGeneration { - finalErr = fmt.Errorf("signed-session bootstrap was superseded by session clear") - } - - var latest *signedSessionRecord - if finalErr == nil { - latest, finalErr = r.loadSignedSession(config) - } - if finalErr == nil && signedSessionRecordIsUsable(latest) && - !sameSignedSession(latest, record) && - !coordinator.generationIsBlocked(latest) { - coordinator.clearChallenge() - } else if finalErr == nil && bootstrap.SessionID != "" { - latest.SessionID = bootstrap.SessionID - latest.SessionSecret = bootstrap.SessionSecret - latest.ExpiresAt = bootstrap.ExpiresAt - if saveErr := r.saveSignedSession(config, latest); saveErr != nil { - finalErr = fmt.Errorf("save bootstrapped signed session: %w", saveErr) - } else { - coordinator.clearBlockedGeneration() - coordinator.clearChallenge() - } - } else if finalErr == nil { - request := &PendingAuthRequest{ - ExtensionID: r.extensionID, - AuthURL: bootstrap.AuthURL, - CallbackURL: bootstrap.CallbackURL, - State: bootstrap.CallbackState, - CreatedAt: time.Now(), - } - if registerErr := registerPendingAuthRequest(request); registerErr != nil { - finalErr = registerErr - } else { - coordinator.rememberChallenge(request) - authURL = bootstrap.AuthURL - } - } - if coordinator.bootstrapInFlight && coordinator.bootstrapDone == bootstrapDone { - coordinator.bootstrapInFlight = false - coordinator.bootstrapErr = finalErr - close(bootstrapDone) - } - return authURL, finalErr -} - -type signedSessionBootstrapResult struct { - SessionID string - SessionSecret string - ExpiresAt string - AuthURL string - CallbackURL string - CallbackState string -} - -func (r *extensionRuntime) performSignedSessionBootstrap( - ctx context.Context, - config SignedSessionConfig, - record *signedSessionRecord, - reason string, -) (signedSessionBootstrapResult, error) { - var result signedSessionBootstrapResult - bootstrapURL, err := signedSessionURL(config, config.Endpoints.Bootstrap) - if err != nil { - return result, fmt.Errorf("build signed-session bootstrap URL: %w", err) - } - parsed, err := url.Parse(bootstrapURL) - if err != nil { - return result, fmt.Errorf("parse signed-session bootstrap URL: %w", err) - } - query := parsed.Query() - query.Set("app_version", config.AppVersion) - query.Set("install_id", record.InstallID) - parsed.RawQuery = query.Encode() - if r.httpClient == nil { - return result, fmt.Errorf("signed-session bootstrap HTTP client is unavailable") - } - - var resp *http.Response - for attempt := 0; attempt < 2; attempt++ { - req, requestErr := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) - if requestErr != nil { - return result, fmt.Errorf("build signed-session bootstrap request: %w", requestErr) - } - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "SpotiFLAC-Mobile/"+config.AppVersion) - resp, err = r.httpClient.Do(req) - if err == nil { - break - } - if resp != nil && resp.Body != nil { - resp.Body.Close() - resp = nil - } - if attempt == 0 { - // Android can retain pooled connections across a Wi-Fi/cellular - // transition. Rebuild the GET once after dropping those sockets. - r.httpClient.CloseIdleConnections() - } - } - if err != nil { - var urlErr *url.Error - if errors.As(err, &urlErr) && urlErr.Err != nil { - err = urlErr.Err - } - bootstrapErr := fmt.Errorf( - "signed-session bootstrap network request to %s failed: %v", - parsed.Host, - err, - ) - LogWarn("SignedSession", "Bootstrap failed for extension %s (%s): %v", r.extensionID, reason, bootstrapErr) - return result, bootstrapErr - } - defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - // Drain a bounded error response so the transport can reuse the - // connection without exposing response bodies that may contain secrets. - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) - message := fmt.Sprintf("signed-session bootstrap returned HTTP %d", resp.StatusCode) - if resp.StatusCode >= 500 { - message = fmt.Sprintf("signed-session bootstrap network request returned HTTP %d", resp.StatusCode) - } - if retryAfter := signedSessionRetryAfterSeconds(resp); retryAfter > 0 { - message += fmt.Sprintf("; retry-after seconds: %d", retryAfter) - } - bootstrapErr := errors.New(message) - LogWarn("SignedSession", "Bootstrap failed for extension %s (%s): %v", r.extensionID, reason, bootstrapErr) - return result, bootstrapErr - } - body, err := readExtensionHTTPResponseBody(resp) - if err != nil { - return result, fmt.Errorf("read signed-session bootstrap response: %w", err) - } - var boot signedSessionExchangeResponse - if err := json.Unmarshal(body, &boot); err != nil { - return result, fmt.Errorf("decode signed-session bootstrap response: %w", err) - } - if boot.SessionID != "" && boot.SessionSecret != "" && boot.ExpiresAt != "" { - result.SessionID = boot.SessionID - result.SessionSecret = boot.SessionSecret - result.ExpiresAt = boot.ExpiresAt - return result, nil - } - authURL := boot.AuthURL - if authURL == "" && boot.ChallengeURL != "" { - authURL = boot.ChallengeURL - } - // Preserve a server-provided state when present. Otherwise add a fresh - // host-generated nonce so the callback is bound to this one challenge. - callbackState, err := newExtensionCallbackState() - if err != nil { - return result, fmt.Errorf("prepare signed-session callback state: %w", err) - } - if parsedAuthURL, parseErr := url.Parse(authURL); parseErr == nil { - if serverState := strings.TrimSpace(parsedAuthURL.Query().Get("state")); serverState != "" { - callbackState = serverState - } else if authURL != "" { - authURL, err = setOAuthState(authURL, callbackState) - if err != nil { - return result, fmt.Errorf("prepare signed-session verification URL: %w", err) - } - } - } - callbackURL, err := setOAuthState(config.CallbackURL, callbackState) - if err != nil { - return result, fmt.Errorf("prepare signed-session callback: %w", err) - } - if authURL == "" && boot.ChallengeID != "" { - authURL = r.buildSignedSessionChallengeURL(config, boot.ChallengeID, callbackState) - } - if authURL == "" { - return result, fmt.Errorf("signed-session bootstrap did not return a session or verification challenge") - } - result.AuthURL = authURL - result.CallbackURL = callbackURL - result.CallbackState = callbackState - return result, nil -} - -func (r *extensionRuntime) buildSignedSessionChallengeURL(config SignedSessionConfig, challengeID, callbackState string) string { - challengeURL, err := signedSessionURL(config, config.Endpoints.Challenge) - if err != nil { - return "" - } - parsed, err := url.Parse(challengeURL) - if err != nil { - return "" - } - callback, err := url.Parse(config.CallbackURL) - if err != nil { - return "" - } - q := callback.Query() - q.Set("cb_version", "v2grant") - q.Set("state", callbackState) - callback.RawQuery = q.Encode() - - query := parsed.Query() - query.Set("id", challengeID) - query.Set("cb", callback.String()) - parsed.RawQuery = query.Encode() - return parsed.String() -} - -func signedSessionURL(config SignedSessionConfig, endpoint string) (string, error) { - base, err := url.Parse(strings.TrimRight(config.BaseURL, "/") + "/") - if err != nil || base.Scheme != "https" || base.Host == "" { - return "", fmt.Errorf("invalid signed session baseUrl") - } - endpoint = strings.TrimSpace(endpoint) - if endpoint == "" { - return "", fmt.Errorf("signed session endpoint is empty") - } - if strings.HasPrefix(endpoint, "https://") { - return endpoint, nil - } - endpoint = strings.TrimLeft(endpoint, "/") - ref, _ := url.Parse(endpoint) - return base.ResolveReference(ref).String(), nil -} - -func (r *extensionRuntime) doSignedSessionRequest( - config SignedSessionConfig, - record *signedSessionRecord, - method string, - requestPath string, - body []byte, - extraHeaders map[string]string, -) (*http.Response, []byte, map[string]any, error) { - fullURL, err := signedSessionURL(config, requestPath) - if err != nil { - return nil, nil, nil, err - } - parsed, err := url.Parse(fullURL) - if err != nil { - return nil, nil, nil, err - } - ts := signedSessionRequestNow().UTC().Format("2006-01-02T15:04:05.000Z") - nonce := randomHex(12) - bodyHashBytes := sha256.Sum256(body) - bodyHash := hex.EncodeToString(bodyHashBytes[:]) - parsedTs, _ := time.Parse("2006-01-02T15:04:05.000Z", ts) - window := parsedTs.Unix() / int64(config.TimeWindowSeconds) - rollingInput := fmt.Sprintf("%d:%s", window, record.SessionID) - rk := base64.RawURLEncoding.EncodeToString(hmacSHA256Bytes([]byte(record.SessionSecret), []byte(rollingInput))) - signingInput := strings.Join([]string{ - config.SchemeLabel, - method, - parsed.EscapedPath(), - "", - bodyHash, - ts, - nonce, - record.SessionID, - config.AppVersion, - config.Platform, - }, "\n") - sig := base64.RawURLEncoding.EncodeToString(hmacSHA256Bytes([]byte(rk), []byte(signingInput))) - - req, err := http.NewRequest(method, fullURL, bytes.NewReader(body)) - if err != nil { - return nil, nil, nil, err - } - req = r.bindDownloadCancelContext(req) - req.Header.Set("Accept", "application/json") - if len(body) > 0 { - req.Header.Set("Content-Type", "application/json") - } - req.Header.Set("User-Agent", "SpotiFLAC-Mobile/"+config.AppVersion) - prefix := config.HeaderPrefix - req.Header.Set(prefix+"Session", record.SessionID) - req.Header.Set(prefix+"Timestamp", ts) - req.Header.Set(prefix+"Nonce", nonce) - req.Header.Set(prefix+"Body-SHA256", bodyHash) - req.Header.Set(prefix+"Signature", sig) - req.Header.Set(prefix+"App-Version", config.AppVersion) - req.Header.Set(prefix+"Platform", config.Platform) - for k, v := range extraHeaders { - req.Header.Set(k, v) - } - - resp, err := r.httpClient.Do(req) - if err != nil { - return nil, nil, nil, err - } - defer resp.Body.Close() - respBody, err := readExtensionHTTPResponseBody(resp) - if err != nil { - return nil, nil, nil, err - } - headers := make(map[string]any) - for k, v := range resp.Header { - if len(v) == 1 { - headers[k] = v[0] - } else { - headers[k] = v - } - } - return resp, respBody, headers, nil -} - -func signedSessionRetryAfterSeconds(resp *http.Response) int { - if resp == nil { - return 0 - } - value := strings.TrimSpace(resp.Header.Get("Retry-After")) - if value == "" { - return 0 - } - if seconds, err := strconv.Atoi(value); err == nil { - if seconds < 0 { - return 0 - } - return seconds - } - if retryAt, err := http.ParseTime(value); err == nil { - seconds := int(time.Until(retryAt).Seconds()) - if seconds < 0 { - return 0 - } - return seconds - } - return 0 -} - -func hmacSHA256Bytes(key, message []byte) []byte { - mac := hmac.New(sha256.New, key) - mac.Write(message) - return mac.Sum(nil) -} - -func setPendingSignedSessionGrant(extensionID, grant string) { - extensionID = strings.TrimSpace(extensionID) - grant = strings.TrimSpace(grant) - if extensionID == "" || grant == "" { - return - } - pendingSignedSessionGrantsMu.Lock() - pendingSignedSessionGrants[extensionID] = grant - pendingSignedSessionGrantsMu.Unlock() -} diff --git a/go_backend/extension_signed_session_test.go b/go_backend/extension_signed_session_test.go deleted file mode 100644 index 8826211a..00000000 --- a/go_backend/extension_signed_session_test.go +++ /dev/null @@ -1,2293 +0,0 @@ -package gobackend - -import ( - "context" - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - goruntime "runtime" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/dop251/goja" -) - -func TestSanitizeSignedSessionNamespace(t *testing.T) { - cases := []struct { - name string - in string - want string - }{ - {"lowercases", "MyExt", "myext"}, - {"trims whitespace", " my-ext ", "my-ext"}, - {"keeps allowed punctuation", "my-ext_v1.2", "my-ext_v1.2"}, - {"strips spaces and slashes but keeps dots", "my ext/../v1", "myext..v1"}, - {"strips leading and trailing punctuation", "..--my-ext__..", "my-ext"}, - {"empty stays empty", "", ""}, - {"only punctuation collapses to empty", "...", ""}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := sanitizeSignedSessionNamespace(tc.in); got != tc.want { - t.Errorf("sanitizeSignedSessionNamespace(%q) = %q, want %q", tc.in, got, tc.want) - } - }) - } -} - -func TestSignedSessionConfigWithDefaults(t *testing.T) { - t.Run("nil config yields zero value", func(t *testing.T) { - got := signedSessionConfigWithDefaults(nil) - if got != (SignedSessionConfig{}) { - t.Errorf("expected zero value config, got %+v", got) - } - }) - - t.Run("fills in defaults without a namespace or baseUrl", func(t *testing.T) { - got := signedSessionConfigWithDefaults(&SignedSessionConfig{}) - if got.AppVersion != "ext-1.0" { - t.Errorf("AppVersion = %q, want ext-1.0", got.AppVersion) - } - if got.Platform != "extension" { - t.Errorf("Platform = %q, want extension", got.Platform) - } - if got.CallbackURL != "spotiflac://session-grant" { - t.Errorf("CallbackURL = %q", got.CallbackURL) - } - if got.SchemeLabel != "SPOTIFLAC-HMAC-V1" { - t.Errorf("SchemeLabel = %q", got.SchemeLabel) - } - if got.HeaderPrefix != "X-Sig-" { - t.Errorf("HeaderPrefix = %q", got.HeaderPrefix) - } - if got.TimeWindowSeconds != 300 { - t.Errorf("TimeWindowSeconds = %d, want 300", got.TimeWindowSeconds) - } - if got.Endpoints.Bootstrap != "/bootstrap" || got.Endpoints.Challenge != "/challenge" || got.Endpoints.Exchange != "/session/exchange" { - t.Errorf("Endpoints defaults = %+v", got.Endpoints) - } - }) - - t.Run("preserves values the manifest already set", func(t *testing.T) { - custom := &SignedSessionConfig{ - Namespace: "provider", - BaseURL: "https://auth.example.com", - AppVersion: "5.0", - Platform: "mobile", - TimeWindowSeconds: 60, - Endpoints: SignedSessionEndpoints{Exchange: "/custom/exchange"}, - } - got := signedSessionConfigWithDefaults(custom) - if got.Namespace != "provider" || got.BaseURL != "https://auth.example.com" { - t.Errorf("namespace/baseUrl were overwritten: %+v", got) - } - if got.AppVersion != "5.0" || got.Platform != "mobile" || got.TimeWindowSeconds != 60 { - t.Errorf("existing scalars were overwritten: %+v", got) - } - if got.Endpoints.Exchange != "/custom/exchange" { - t.Errorf("Endpoints.Exchange overwritten: %q", got.Endpoints.Exchange) - } - // Untouched endpoints still get their defaults filled in. - if got.Endpoints.Bootstrap != "/bootstrap" { - t.Errorf("Endpoints.Bootstrap = %q, want default", got.Endpoints.Bootstrap) - } - }) -} - -func TestParseSignedSessionTime(t *testing.T) { - cases := []struct { - name string - in string - wantOK bool - wantUTC string - }{ - {"RFC3339Nano", "2026-05-04T10:00:00.123456789Z", true, "2026-05-04T10:00:00Z"}, - {"RFC3339", "2026-05-04T10:00:00Z", true, "2026-05-04T10:00:00Z"}, - {"millisecond layout", "2026-05-04T10:00:00.000Z", true, "2026-05-04T10:00:00Z"}, - {"empty", "", false, ""}, - {"garbage", "not-a-time", false, ""}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, ok := parseSignedSessionTime(tc.in) - if ok != tc.wantOK { - t.Fatalf("parseSignedSessionTime(%q) ok = %v, want %v", tc.in, ok, tc.wantOK) - } - if ok && got.UTC().Format(time.RFC3339) != tc.wantUTC { - t.Errorf("parseSignedSessionTime(%q) = %v, want %v", tc.in, got.UTC().Format(time.RFC3339), tc.wantUTC) - } - }) - } -} - -func TestPreflightSignedSession(t *testing.T) { - t.Run("reuses a valid session without network", func(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return nil, fmt.Errorf("unexpected request: %s", req.URL) - }) - runtime := newSignedSessionTestRuntime(t, "preflight-valid", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "preflight-valid", - BaseURL: "https://auth.example.com", - } - config := signedSessionConfigWithDefaults(runtime.manifest.SignedSession) - record, err := runtime.loadSignedSession(config) - if err != nil { - t.Fatalf("load session: %v", err) - } - record.SessionID = "session" - record.SessionSecret = "secret" - record.ExpiresAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339) - if err := runtime.saveSignedSession(config, record); err != nil { - t.Fatalf("save session: %v", err) - } - - verificationRequired, err := runtime.preflightSignedSession() - if err != nil || verificationRequired { - t.Fatalf("preflight = verification:%v error:%v", verificationRequired, err) - } - if calls != 0 { - t.Fatalf("valid session made %d network request(s)", calls) - } - }) - - t.Run("reuses a fresh pending challenge", func(t *testing.T) { - runtime := newSignedSessionTestRuntime(t, "preflight-pending", roundTripFunc(func(req *http.Request) (*http.Response, error) { - return nil, fmt.Errorf("unexpected request: %s", req.URL) - })) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "preflight-pending", - BaseURL: "https://auth.example.com", - } - pendingAuthRequestsMu.Lock() - pendingAuthRequests[runtime.extensionID] = &PendingAuthRequest{ - ExtensionID: runtime.extensionID, - AuthURL: "https://auth.example.com/challenge", - CreatedAt: time.Now(), - } - pendingAuthRequestsMu.Unlock() - t.Cleanup(func() { ClearPendingAuthRequest(runtime.extensionID) }) - - verificationRequired, err := runtime.preflightSignedSession() - if err != nil || !verificationRequired { - t.Fatalf("preflight = verification:%v error:%v", verificationRequired, err) - } - }) - - t.Run("bootstraps a challenge for an unauthenticated session", func(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - payload, _ := json.Marshal(signedSessionExchangeResponse{ - ChallengeURL: "https://auth.example.com/challenge", - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "preflight-challenge", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "preflight-challenge", - BaseURL: "https://auth.example.com", - } - t.Cleanup(func() { ClearPendingAuthRequest(runtime.extensionID) }) - - verificationRequired, err := runtime.preflightSignedSession() - if err != nil || !verificationRequired { - t.Fatalf("preflight = verification:%v error:%v", verificationRequired, err) - } - if calls != 1 { - t.Fatalf("bootstrap calls = %d, want 1", calls) - } - if pending := GetPendingAuthRequest(runtime.extensionID); pending == nil || pending.AuthURL == "" { - t.Fatalf("pending challenge = %#v", pending) - } - }) - - t.Run("accepts a session issued directly by bootstrap", func(t *testing.T) { - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - payload, _ := json.Marshal(signedSessionExchangeResponse{ - SessionID: "boot-session", - SessionSecret: "boot-secret", - ExpiresAt: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "preflight-direct", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "preflight-direct", - BaseURL: "https://auth.example.com", - } - - verificationRequired, err := runtime.preflightSignedSession() - if err != nil || verificationRequired { - t.Fatalf("preflight = verification:%v error:%v", verificationRequired, err) - } - config := signedSessionConfigWithDefaults(runtime.manifest.SignedSession) - record, err := runtime.loadSignedSession(config) - if err != nil || record.SessionID != "boot-session" { - t.Fatalf("bootstrapped session = %#v error:%v", record, err) - } - }) - - t.Run("retries a transport failure once and surfaces the cause", func(t *testing.T) { - calls := 0 - runtime := newSignedSessionTestRuntime(t, "preflight-network", roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return nil, fmt.Errorf("dial tcp: Wi-Fi route unavailable") - })) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "preflight-network", - BaseURL: "https://auth.example.com", - } - - verificationRequired, err := runtime.preflightSignedSession() - if err == nil || verificationRequired { - t.Fatalf("preflight = verification:%v error:%v", verificationRequired, err) - } - if calls != 2 { - t.Fatalf("bootstrap calls = %d, want one initial attempt and one retry", calls) - } - if !strings.Contains(err.Error(), "network request") || !strings.Contains(err.Error(), "Wi-Fi route unavailable") { - t.Fatalf("bootstrap error did not preserve the transport cause: %v", err) - } - if strings.Contains(err.Error(), "install_id=") { - t.Fatalf("bootstrap error leaked the install identifier: %v", err) - } - }) - - t.Run("surfaces HTTP status and retry-after without inventing a challenge", func(t *testing.T) { - calls := 0 - runtime := newSignedSessionTestRuntime(t, "preflight-rate-limit", roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return &http.Response{ - StatusCode: http.StatusTooManyRequests, - Header: http.Header{"Retry-After": []string{"17"}}, - Body: io.NopCloser(strings.NewReader(`{"error":"limited"}`)), - Request: req, - }, nil - })) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "preflight-rate-limit", - BaseURL: "https://auth.example.com", - } - - verificationRequired, err := runtime.preflightSignedSession() - if err == nil || verificationRequired { - t.Fatalf("preflight = verification:%v error:%v", verificationRequired, err) - } - if calls != 1 { - t.Fatalf("rate-limited bootstrap calls = %d, want 1", calls) - } - if !strings.Contains(err.Error(), "HTTP 429") || !strings.Contains(err.Error(), "retry-after seconds: 17") { - t.Fatalf("rate-limit details missing from bootstrap error: %v", err) - } - }) -} - -func TestDownloadWithExtensionsPreflightsBeforeMetadataEnrichment(t *testing.T) { - extensionID := "preflight-download" - itemID := "preflight-item" - RemoveItemProgress(itemID) - t.Cleanup(func() { RemoveItemProgress(itemID) }) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - payload, _ := json.Marshal(signedSessionExchangeResponse{ - ChallengeURL: "https://auth.example.com/challenge", - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, extensionID, transport) - manifest := &ExtensionManifest{ - Name: extensionID, - Types: []ExtensionType{ExtensionTypeDownloadProvider}, - SignedSession: &SignedSessionConfig{ - Namespace: extensionID, - BaseURL: "https://auth.example.com", - }, - } - runtime.manifest = manifest - ext := &loadedExtension{ - ID: extensionID, - Manifest: manifest, - VM: runtime.vm, - runtime: runtime, - initialized: true, - Enabled: true, - DataDir: runtime.dataDir, - } - - manager := getExtensionManager() - manager.mu.Lock() - previous, hadPrevious := manager.extensions[extensionID] - manager.extensions[extensionID] = ext - manager.mu.Unlock() - t.Cleanup(func() { - ClearPendingAuthRequest(extensionID) - manager.mu.Lock() - if hadPrevious { - manager.extensions[extensionID] = previous - } else { - delete(manager.extensions, extensionID) - } - manager.mu.Unlock() - }) - - // Metadata may originate from another extension, but the explicitly chosen - // download provider owns the signed-session namespace and verification. - requestJSON := `{"source":"source-metadata","service":"preflight-download","item_id":"preflight-item","isrc":"USRC17607839"}` - responseJSON, err := DownloadWithExtensionsJSON(requestJSON) - if err != nil { - t.Fatalf("DownloadWithExtensionsJSON: %v", err) - } - var response DownloadResponse - if err := json.Unmarshal([]byte(responseJSON), &response); err != nil { - t.Fatalf("decode response: %v", err) - } - if response.ErrorType != "verification_required" || response.Service != extensionID { - t.Fatalf("response = %#v", response) - } - if got := GetItemProgress(itemID); got != "{}" { - t.Fatalf("verification response left stale progress: %s", got) - } -} - -func TestParallelSignedSessionPreflightSharesOneBootstrap(t *testing.T) { - var calls atomic.Int32 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls.Add(1) - // Keep the first bootstrap in flight long enough for the second runtime - // to contend on the shared session coordinator. - time.Sleep(20 * time.Millisecond) - payload, _ := json.Marshal(signedSessionExchangeResponse{ - ChallengeURL: "https://auth.example.com/challenge", - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - }) - - root := t.TempDir() - config := &SignedSessionConfig{ - Namespace: "shared-download-session", - BaseURL: "https://auth.example.com", - } - newRuntime := func(extensionID string) *extensionRuntime { - return &extensionRuntime{ - extensionID: extensionID, - manifest: &ExtensionManifest{ - Name: extensionID, - SignedSession: config, - }, - dataDir: filepath.Join(root, extensionID), - vm: goja.New(), - httpClient: &http.Client{Transport: transport}, - } - } - runtimeA := newRuntime("provider-a") - runtimeB := newRuntime("provider-b") - t.Cleanup(func() { - ClearPendingAuthRequest("provider-a") - ClearPendingAuthRequest("provider-b") - }) - - results := make(chan error, 2) - for _, runtime := range []*extensionRuntime{runtimeA, runtimeB} { - go func(runtime *extensionRuntime) { - verificationRequired, err := runtime.preflightSignedSession() - if err == nil && !verificationRequired { - err = fmt.Errorf("verification was not requested") - } - results <- err - }(runtime) - } - for range 2 { - if err := <-results; err != nil { - t.Fatal(err) - } - } - if got := calls.Load(); got != 1 { - t.Fatalf("parallel preflight bootstrap calls = %d, want 1", got) - } - pendingA := GetPendingAuthRequest("provider-a") - pendingB := GetPendingAuthRequest("provider-b") - if pendingA == nil || pendingB == nil || pendingA.State != pendingB.State || - !pendingA.CreatedAt.Equal(pendingB.CreatedAt) { - t.Fatal("shared bootstrap must preserve the same nonce and creation time") - } - if _, err := ConsumeExtensionCallbackState(pendingA.State); err != nil { - t.Fatalf("shared callback cannot be consumed: %v", err) - } - if GetPendingAuthRequest("provider-a") != nil || GetPendingAuthRequest("provider-b") != nil { - t.Fatal("consuming a shared callback must clear every alias") - } - if _, err := ConsumeExtensionCallbackState(pendingA.State); err == nil { - t.Fatal("shared callback replay was accepted") - } -} - -func TestRememberSignedSessionChallengePreservesOriginalLifetime(t *testing.T) { - request := &PendingAuthRequest{ - ExtensionID: "remember-original", AuthURL: "https://auth.example.com/challenge", - CallbackURL: "spotiflac://callback", State: "remember-original-state", - CreatedAt: time.Now().Add(-2 * time.Minute), - } - if err := registerPendingAuthRequest(request); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - ClearPendingAuthRequest("remember-original") - ClearPendingAuthRequest("remember-shared") - }) - coordinator := &signedSessionCoordinator{} - coordinator.rememberChallenge(request) - if !coordinator.challengeCreatedAt.Equal(request.CreatedAt) { - t.Fatal("remembering an existing challenge reset its TTL") - } - runtime := &extensionRuntime{extensionID: "remember-shared"} - coordinator.mu.Lock() - _, err := runtime.startSignedSessionVerificationLocked(SignedSessionConfig{}, coordinator, "test") - coordinator.mu.Unlock() - if err != nil { - t.Fatalf("reusing a remembered challenge: %v", err) - } - shared := GetPendingAuthRequest("remember-shared") - if shared == nil || !shared.CreatedAt.Equal(request.CreatedAt) { - t.Fatal("shared request lost the original challenge timestamp") - } - expired := *request - expired.CreatedAt = time.Now().Add(-pendingAuthRequestTTL - time.Second) - coordinator.rememberChallenge(&expired) - if coordinator.activeChallenge() { - t.Fatal("remembering an expired challenge made it active again") - } -} - -func TestParallelSignedSessionPreflightSharesBootstrapFailure(t *testing.T) { - var calls atomic.Int32 - requestStarted := make(chan struct{}) - releaseRequest := make(chan struct{}) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - if calls.Add(1) == 1 { - close(requestStarted) - } - <-releaseRequest - return &http.Response{ - StatusCode: http.StatusServiceUnavailable, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(`{"error":"busy"}`)), - Request: req, - }, nil - }) - - root := t.TempDir() - config := &SignedSessionConfig{ - Namespace: "shared-failed-session", - BaseURL: "https://auth.example.com", - } - newRuntime := func(extensionID string) *extensionRuntime { - return &extensionRuntime{ - extensionID: extensionID, - manifest: &ExtensionManifest{ - Name: extensionID, - SignedSession: config, - }, - dataDir: filepath.Join(root, extensionID), - vm: goja.New(), - httpClient: &http.Client{Transport: transport}, - } - } - - const workers = 12 - errors := make(chan error, workers) - for worker := range workers { - go func(worker int) { - _, err := newRuntime(fmt.Sprintf("failed-provider-%d", worker)).preflightSignedSession() - errors <- err - }(worker) - } - select { - case <-requestStarted: - case <-time.After(time.Second): - t.Fatal("bootstrap request did not start") - } - // Give every parallel caller time to join the in-flight generation. The - // request remains blocked, so no caller can observe a completed operation. - time.Sleep(50 * time.Millisecond) - close(releaseRequest) - for range workers { - if err := <-errors; err == nil || !strings.Contains(err.Error(), "HTTP 503") { - t.Fatalf("unexpected coalesced bootstrap error: %v", err) - } - } - if got := calls.Load(); got != 1 { - t.Fatalf("parallel failed bootstrap calls = %d, want 1", got) - } -} - -func TestDownloadWithExtensionsStopsAfterFailedSignedSessionPreflight(t *testing.T) { - extensionID := "preflight-network-failure" - itemID := "preflight-network-item" - RemoveItemProgress(itemID) - t.Cleanup(func() { RemoveItemProgress(itemID) }) - - calls := 0 - runtime := newSignedSessionTestRuntime(t, extensionID, roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return nil, fmt.Errorf("dial tcp: Wi-Fi route unavailable") - })) - manifest := &ExtensionManifest{ - Name: extensionID, - Types: []ExtensionType{ExtensionTypeDownloadProvider}, - SignedSession: &SignedSessionConfig{ - Namespace: extensionID, - BaseURL: "https://auth.example.com", - }, - } - runtime.manifest = manifest - ext := &loadedExtension{ - ID: extensionID, - Manifest: manifest, - VM: runtime.vm, - runtime: runtime, - initialized: true, - Enabled: true, - DataDir: runtime.dataDir, - } - - manager := getExtensionManager() - manager.mu.Lock() - previous, hadPrevious := manager.extensions[extensionID] - manager.extensions[extensionID] = ext - manager.mu.Unlock() - t.Cleanup(func() { - manager.mu.Lock() - if hadPrevious { - manager.extensions[extensionID] = previous - } else { - delete(manager.extensions, extensionID) - } - manager.mu.Unlock() - }) - - requestJSON := `{"service":"preflight-network-failure","item_id":"preflight-network-item","isrc":"USRC17607839"}` - responseJSON, err := DownloadWithExtensionsJSON(requestJSON) - if err != nil { - t.Fatalf("DownloadWithExtensionsJSON: %v", err) - } - var response DownloadResponse - if err := json.Unmarshal([]byte(responseJSON), &response); err != nil { - t.Fatalf("decode response: %v", err) - } - if response.ErrorType != "network" || response.Service != extensionID || !strings.Contains(response.Error, "Could not start verification") { - t.Fatalf("response = %#v", response) - } - if calls != 2 { - t.Fatalf("bootstrap calls = %d, want only the initial attempt and its transport retry", calls) - } - if got := GetItemProgress(itemID); got != "{}" { - t.Fatalf("failed preflight left stale progress: %s", got) - } -} - -func TestSignedSessionURL(t *testing.T) { - base := SignedSessionConfig{BaseURL: "https://auth.example.com/api"} - - t.Run("joins a relative endpoint onto the base", func(t *testing.T) { - got, err := signedSessionURL(base, "/session/exchange") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - want := "https://auth.example.com/api/session/exchange" - if got != want { - t.Errorf("signedSessionURL = %q, want %q", got, want) - } - }) - - t.Run("passes an absolute https endpoint through unchanged", func(t *testing.T) { - got, err := signedSessionURL(base, "https://other.example.com/challenge") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got != "https://other.example.com/challenge" { - t.Errorf("signedSessionURL = %q", got) - } - }) - - t.Run("rejects an empty endpoint", func(t *testing.T) { - if _, err := signedSessionURL(base, ""); err == nil { - t.Error("expected error for empty endpoint") - } - }) - - t.Run("rejects a non-https base URL", func(t *testing.T) { - if _, err := signedSessionURL(SignedSessionConfig{BaseURL: "http://auth.example.com"}, "/x"); err == nil { - t.Error("expected error for http:// base URL") - } - }) - - t.Run("rejects a base URL with no host", func(t *testing.T) { - if _, err := signedSessionURL(SignedSessionConfig{BaseURL: "https:///no-host"}, "/x"); err == nil { - t.Error("expected error for base URL without a host") - } - }) -} - -func TestSignedSessionRetryAfterSeconds(t *testing.T) { - t.Run("nil response", func(t *testing.T) { - if got := signedSessionRetryAfterSeconds(nil); got != 0 { - t.Errorf("got %d, want 0", got) - } - }) - - t.Run("numeric seconds", func(t *testing.T) { - resp := &http.Response{Header: http.Header{"Retry-After": []string{"30"}}} - if got := signedSessionRetryAfterSeconds(resp); got != 30 { - t.Errorf("got %d, want 30", got) - } - }) - - t.Run("negative numeric seconds clamp to zero", func(t *testing.T) { - resp := &http.Response{Header: http.Header{"Retry-After": []string{"-5"}}} - if got := signedSessionRetryAfterSeconds(resp); got != 0 { - t.Errorf("got %d, want 0", got) - } - }) - - t.Run("HTTP-date in the future", func(t *testing.T) { - future := time.Now().Add(2 * time.Minute).UTC() - resp := &http.Response{Header: http.Header{"Retry-After": []string{future.Format(http.TimeFormat)}}} - got := signedSessionRetryAfterSeconds(resp) - if got <= 0 || got > 120 { - t.Errorf("got %d, want roughly 120", got) - } - }) - - t.Run("missing header", func(t *testing.T) { - resp := &http.Response{Header: http.Header{}} - if got := signedSessionRetryAfterSeconds(resp); got != 0 { - t.Errorf("got %d, want 0", got) - } - }) -} - -func TestSignedSessionProviderRetryDuration(t *testing.T) { - t.Run("header is authoritative", func(t *testing.T) { - resp := &http.Response{Header: http.Header{"Retry-After": []string{"10"}}} - contract := signedSessionErrorContract{RetryAfterSeconds: 30} - if got := signedSessionProviderRetryDuration(resp, contract); got != 10*time.Second { - t.Fatalf("retry duration = %s, want 10s", got) - } - }) - - t.Run("body fallback is bounded", func(t *testing.T) { - resp := &http.Response{Header: make(http.Header)} - contract := signedSessionErrorContract{RetryAfterSeconds: int(^uint(0) >> 1)} - if got := signedSessionProviderRetryDuration(resp, contract); got != maxRetryAfterDelay { - t.Fatalf("retry duration = %s, want cap %s", got, maxRetryAfterDelay) - } - }) -} - -func TestNormalizeSignedSessionRecordScope(t *testing.T) { - config := SignedSessionConfig{Namespace: "Provider", BaseURL: "https://a.example.com", AppVersion: "1.0", Platform: "mobile"} - - t.Run("first save just stamps the scope", func(t *testing.T) { - record := &signedSessionRecord{SessionID: "s1", SessionSecret: "secret"} - normalizeSignedSessionRecordScope(config, record) - if record.Namespace != "provider" || record.BaseURL != config.BaseURL { - t.Errorf("scope not stamped: %+v", record) - } - if record.SessionID != "s1" || record.SessionSecret != "secret" { - t.Errorf("session fields should survive first stamp: %+v", record) - } - }) - - t.Run("same scope preserves the session", func(t *testing.T) { - record := &signedSessionRecord{ - Namespace: "provider", BaseURL: config.BaseURL, AppVersion: config.AppVersion, Platform: config.Platform, - SessionID: "s1", SessionSecret: "secret", ExpiresAt: "later", - } - normalizeSignedSessionRecordScope(config, record) - if record.SessionID != "s1" || record.SessionSecret != "secret" || record.ExpiresAt != "later" { - t.Errorf("unexpected wipe on matching scope: %+v", record) - } - }) - - t.Run("changed scope wipes the session secret", func(t *testing.T) { - record := &signedSessionRecord{ - Namespace: "provider", BaseURL: "https://old.example.com", AppVersion: config.AppVersion, Platform: config.Platform, - SessionID: "s1", SessionSecret: "secret", ExpiresAt: "later", - } - normalizeSignedSessionRecordScope(config, record) - if record.SessionID != "" || record.SessionSecret != "" || record.ExpiresAt != "" { - t.Errorf("expected session fields to be wiped after scope change: %+v", record) - } - if record.BaseURL != config.BaseURL { - t.Errorf("BaseURL not updated to new scope: %q", record.BaseURL) - } - }) -} - -func newSignedSessionTestRuntime(t *testing.T, extensionID string, transport roundTripFunc) *extensionRuntime { - t.Helper() - dataDir := t.TempDir() - return &extensionRuntime{ - extensionID: extensionID, - manifest: &ExtensionManifest{Name: extensionID}, - dataDir: dataDir, - vm: goja.New(), - httpClient: &http.Client{Transport: transport}, - } -} - -func saveUsableSignedSession( - t *testing.T, - runtime *extensionRuntime, - config SignedSessionConfig, - sessionID string, -) SignedSessionConfig { - t.Helper() - resolved := signedSessionConfigWithDefaults(&config) - record, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - record.SessionID = sessionID - record.SessionSecret = "secret-" + sessionID - record.ExpiresAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339) - if err := runtime.saveSignedSession(resolved, record); err != nil { - t.Fatal(err) - } - return resolved -} - -func TestSignedSessionFilePathDeterminism(t *testing.T) { - runtime := newSignedSessionTestRuntime(t, "provider-ext", nil) - - configA := SignedSessionConfig{Namespace: "provider", BaseURL: "https://a.example.com"} - configB := SignedSessionConfig{Namespace: "provider", BaseURL: "https://b.example.com"} - - pathA1, err := runtime.signedSessionFilePath(configA) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - pathA2, err := runtime.signedSessionFilePath(configA) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if pathA1 != pathA2 { - t.Errorf("same config produced different paths: %q vs %q", pathA1, pathA2) - } - - pathB, err := runtime.signedSessionFilePath(configB) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if pathA1 == pathB { - t.Errorf("different baseUrl scopes collided on the same file: %q", pathA1) - } - - if _, err := runtime.signedSessionFilePath(SignedSessionConfig{Namespace: ""}); err == nil { - t.Error("expected error for empty namespace") - } -} - -func TestLoadAndSaveSignedSessionRoundTrip(t *testing.T) { - runtime := newSignedSessionTestRuntime(t, "provider-ext", nil) - config := SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"} - - record, err := runtime.loadSignedSession(config) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if record.InstallID == "" { - t.Fatal("expected a generated install_id on first load") - } - - path, err := runtime.signedSessionFilePath(config) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - info, err := os.Stat(path) - if err != nil { - t.Fatalf("expected session file to be persisted: %v", err) - } - // Windows does not preserve Unix permission bits, so only assert on POSIX. - if goruntime.GOOS != "windows" { - if perm := info.Mode().Perm(); perm != 0o600 { - t.Errorf("session file perm = %o, want 0600", perm) - } - } - - record.SessionID = "sess-1" - record.SessionSecret = "top-secret" - record.ExpiresAt = "2030-01-01T00:00:00Z" - if err := runtime.saveSignedSession(config, record); err != nil { - t.Fatalf("unexpected error saving: %v", err) - } - - reloaded, err := runtime.loadSignedSession(config) - if err != nil { - t.Fatalf("unexpected error reloading: %v", err) - } - if reloaded.InstallID != record.InstallID { - t.Errorf("install_id changed across reload: %q vs %q", reloaded.InstallID, record.InstallID) - } - if reloaded.SessionID != "sess-1" || reloaded.SessionSecret != "top-secret" || reloaded.ExpiresAt != "2030-01-01T00:00:00Z" { - t.Errorf("session fields did not round-trip: %+v", reloaded) - } -} - -func TestSignedSessionStatusAndClear(t *testing.T) { - runtime := newSignedSessionTestRuntime(t, "provider-ext", nil) - runtime.manifest.SignedSession = &SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"} - - readStatus := func() map[string]any { - v := runtime.signedSessionStatus(goja.FunctionCall{}) - return v.Export().(map[string]any) - } - - if status := readStatus(); status["authenticated"] != false { - t.Fatalf("expected unauthenticated before any grant, got %+v", status) - } - - config := signedSessionConfigWithDefaults(runtime.manifest.SignedSession) - record, err := runtime.loadSignedSession(config) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - record.SessionID = "sess-1" - record.SessionSecret = "secret" - record.ExpiresAt = time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") - if err := runtime.saveSignedSession(config, record); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if status := readStatus(); status["authenticated"] != true { - t.Fatalf("expected authenticated after saving a live session, got %+v", status) - } - - record.ExpiresAt = time.Now().Add(-time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") - if err := runtime.saveSignedSession(config, record); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if status := readStatus(); status["authenticated"] != false { - t.Fatalf("expected expired session to report unauthenticated, got %+v", status) - } - - // Restore a live session, then confirm clear() wipes it. - record.ExpiresAt = time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") - if err := runtime.saveSignedSession(config, record); err != nil { - t.Fatalf("unexpected error: %v", err) - } - clearResult := runtime.signedSessionClear(goja.FunctionCall{}).Export().(map[string]any) - if clearResult["success"] != true { - t.Fatalf("expected clear to succeed, got %+v", clearResult) - } - if status := readStatus(); status["authenticated"] != false { - t.Fatalf("expected unauthenticated after clear, got %+v", status) - } -} - -// TestDoSignedSessionRequestSignature is the highest-value test in this file: -// it recomputes the HMAC-SHA256 rolling-key signature server-side from the -// headers the client actually sent, the same way a real backend would, to -// guard against silent regressions in the signing scheme (field order, -// rolling-key derivation, or header names). -func TestDoSignedSessionRequestSignature(t *testing.T) { - const sessionSecret = "shhh-its-a-secret" - const sessionID = "sess-42" - - var capturedErr string - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - config := signedSessionConfigWithDefaults(&SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"}) - prefix := config.HeaderPrefix - - ts := req.Header.Get(prefix + "Timestamp") - nonce := req.Header.Get(prefix + "Nonce") - bodyHash := req.Header.Get(prefix + "Body-SHA256") - gotSig := req.Header.Get(prefix + "Signature") - gotSession := req.Header.Get(prefix + "Session") - - if gotSession != sessionID { - capturedErr = "unexpected session id header: " + gotSession - } - - bodyBytes, _ := io.ReadAll(req.Body) - wantBodyHashBytes := sha256.Sum256(bodyBytes) - wantBodyHash := hex.EncodeToString(wantBodyHashBytes[:]) - if bodyHash != wantBodyHash { - capturedErr = "body hash mismatch" - } - - parsedTs, err := time.Parse("2006-01-02T15:04:05.000Z", ts) - if err != nil { - capturedErr = "bad timestamp: " + err.Error() - } - window := parsedTs.Unix() / int64(config.TimeWindowSeconds) - rollingInput := fmt.Sprintf("%d:%s", window, sessionID) - rk := base64.RawURLEncoding.EncodeToString(hmacSHA256Bytes([]byte(sessionSecret), []byte(rollingInput))) - signingInput := strings.Join([]string{ - config.SchemeLabel, - req.Method, - req.URL.EscapedPath(), - "", - bodyHash, - ts, - nonce, - sessionID, - config.AppVersion, - config.Platform, - }, "\n") - wantSig := base64.RawURLEncoding.EncodeToString(hmacSHA256Bytes([]byte(rk), []byte(signingInput))) - - if !hmac.Equal([]byte(gotSig), []byte(wantSig)) { - capturedErr = "signature mismatch: got " + gotSig + " want " + wantSig - } - - return &http.Response{ - StatusCode: 200, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(`{"ok":true}`)), - Request: req, - }, nil - }) - - runtime := newSignedSessionTestRuntime(t, "provider-ext", transport) - config := signedSessionConfigWithDefaults(&SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"}) - record := &signedSessionRecord{InstallID: "install-1", SessionID: sessionID, SessionSecret: sessionSecret} - - resp, body, _, err := runtime.doSignedSessionRequest(config, record, http.MethodPost, "/tracks/search", []byte(`{"q":"test"}`), nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if capturedErr != "" { - t.Fatalf("signature verification failed: %s", capturedErr) - } - if resp.StatusCode != 200 { - t.Errorf("status = %d, want 200", resp.StatusCode) - } - if string(body) != `{"ok":true}` { - t.Errorf("body = %q", body) - } -} - -func TestSignedSessionFetchUnauthenticatedTriggersVerification(t *testing.T) { - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - if strings.Contains(req.URL.Path, "/bootstrap") { - payload := signedSessionExchangeResponse{AuthURL: "https://auth.example.com/login?state=abc"} - body, _ := json.Marshal(payload) - return &http.Response{ - StatusCode: 200, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(body))), - Request: req, - }, nil - } - t.Fatalf("unexpected request to %s", req.URL.String()) - return nil, nil - }) - - runtime := newSignedSessionTestRuntime(t, "provider-ext", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"} - - call := goja.FunctionCall{Arguments: []goja.Value{runtime.vm.ToValue("GET"), runtime.vm.ToValue("/tracks/search")}} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - - if result["ok"] != false { - t.Fatalf("expected ok=false when unauthenticated, got %+v", result) - } - if result["needsVerification"] != true { - t.Fatalf("expected needsVerification=true, got %+v", result) - } - authURL, err := url.Parse(result["auth_url"].(string)) - if err != nil || authURL.Scheme != "https" || authURL.Host != "auth.example.com" || authURL.Path != "/login" || authURL.Query().Get("state") == "" { - t.Fatalf("unexpected auth_url: %+v", result) - } -} - -func TestSignedSessionFetchRevokesSessionOnCanonicalSessionInvalid(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - switch { - case strings.Contains(req.URL.Path, "/bootstrap"): - payload := signedSessionExchangeResponse{AuthURL: "https://auth.example.com/login"} - body, _ := json.Marshal(payload) - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(body))), Request: req}, nil - default: - return &http.Response{ - StatusCode: http.StatusUnauthorized, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"error":"Unauthorized","code":"SESSION_INVALID","origin":"gateway","action":"bootstrap_session"}`, - )), - Request: req, - }, nil - } - }) - - runtime := newSignedSessionTestRuntime(t, "provider-ext", transport) - config := SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - - resolved := signedSessionConfigWithDefaults(&config) - record, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - record.SessionID = "sess-1" - record.SessionSecret = "secret" - record.ExpiresAt = time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") - if err := runtime.saveSignedSession(resolved, record); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - call := goja.FunctionCall{Arguments: []goja.Value{runtime.vm.ToValue("GET"), runtime.vm.ToValue("/tracks/search")}} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - if result["ok"] != false || result["needsVerification"] != true { - t.Fatalf("expected verification after canonical SESSION_INVALID, got %+v", result) - } - - reloaded, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if reloaded.SessionID != "" || reloaded.SessionSecret != "" { - t.Fatalf("expected session to be wiped after SESSION_INVALID, got %+v", reloaded) - } -} - -func TestSignedSessionFetchRetriesAfterSilentSessionBootstrap(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - switch { - case strings.Contains(req.URL.Path, "/bootstrap"): - payload, _ := json.Marshal(signedSessionExchangeResponse{ - SessionID: "sess-replacement", - SessionSecret: "secret-replacement", - ExpiresAt: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - case req.Header.Get("X-Sig-Session") == "sess-replacement": - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(`{"ticket_id":"fresh"}`)), - Request: req, - }, nil - default: - return &http.Response{ - StatusCode: http.StatusUnauthorized, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"error":"Unauthorized","code":"SESSION_INVALID","origin":"gateway","action":"bootstrap_session"}`, - )), - Request: req, - }, nil - } - }) - runtime := newSignedSessionTestRuntime(t, "silent-bootstrap", transport) - config := SignedSessionConfig{Namespace: "silent-bootstrap", BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - resolved := saveUsableSignedSession(t, runtime, config, "sess-old") - - call := goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("POST"), - runtime.vm.ToValue("/tickets"), - }} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - if result["ok"] != true || calls != 3 { - t.Fatalf("signed request was not retried after silent bootstrap: calls=%d result=%+v", calls, result) - } - reloaded, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-replacement" || reloaded.SessionSecret != "secret-replacement" { - t.Fatalf("replacement session was not persisted: %+v", reloaded) - } -} - -func TestSignedSessionFetchDoesNotMutateSessionForNonCanonicalAuthStatus(t *testing.T) { - tests := []struct { - name string - statusCode int - body string - }{ - { - name: "bare 401", - statusCode: http.StatusUnauthorized, - body: `{}`, - }, - { - name: "provider-origin 401", - statusCode: http.StatusUnauthorized, - body: `{"error":"Unauthorized","code":"PROVIDER_AUTH_FAILED","origin":"provider"}`, - }, - { - name: "gateway 401 missing action", - statusCode: http.StatusUnauthorized, - body: `{"error":"Unauthorized","code":"SESSION_INVALID","origin":"gateway"}`, - }, - { - name: "bare 428", - statusCode: http.StatusPreconditionRequired, - body: `{"error":"VERIFY_REQUIRED"}`, - }, - { - name: "request auth 403 with forbidden action", - statusCode: http.StatusForbidden, - body: `{"error":"Forbidden","code":"REQUEST_AUTH_INVALID","origin":"gateway","action":"bootstrap_session"}`, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return &http.Response{ - StatusCode: tc.statusCode, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(tc.body)), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "strict-contract-"+strings.ReplaceAll(tc.name, " ", "-"), transport) - config := SignedSessionConfig{Namespace: runtime.extensionID, BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - resolved := saveUsableSignedSession(t, runtime, config, "sess-safe") - - call := goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("GET"), - runtime.vm.ToValue("/tracks/search"), - }} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - if result["needsVerification"] == true { - t.Fatalf("non-canonical response triggered verification: %+v", result) - } - if calls != 1 { - t.Fatalf("network calls = %d, want no bootstrap or retry", calls) - } - reloaded, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-safe" || reloaded.SessionSecret != "secret-sess-safe" { - t.Fatalf("non-canonical response mutated the session: %+v", reloaded) - } - }) - } -} - -func TestSignedSessionFetchCanonicalVerifyDoesNotClearSession(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - switch { - case strings.Contains(req.URL.Path, "/bootstrap"): - payload, _ := json.Marshal(signedSessionExchangeResponse{ - AuthURL: "https://auth.example.com/verify", - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - case strings.Contains(req.URL.Path, "/session/exchange"): - payload, _ := json.Marshal(signedSessionExchangeResponse{ - SessionID: "sess-after-verify", - SessionSecret: "secret-after-verify", - ExpiresAt: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - case req.Header.Get("X-Sig-Session") == "sess-after-verify": - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(`{"ok":true}`)), - Request: req, - }, nil - } - return &http.Response{ - StatusCode: http.StatusPreconditionRequired, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"error":"VERIFY_REQUIRED","code":"VERIFY_REQUIRED","origin":"gateway","action":"verify"}`, - )), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "canonical-verify", transport) - config := SignedSessionConfig{Namespace: "canonical-verify", BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - resolved := saveUsableSignedSession(t, runtime, config, "sess-verified") - - call := goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("GET"), - runtime.vm.ToValue("/tracks/search"), - }} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - authURL, err := url.Parse(result["auth_url"].(string)) - if result["needsVerification"] != true || err != nil || authURL.Path != "/verify" || authURL.Query().Get("state") == "" { - t.Fatalf("canonical VERIFY_REQUIRED did not open verification: %+v", result) - } - if calls != 2 { - t.Fatalf("network calls = %d, want signed request plus bootstrap", calls) - } - reloaded, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-verified" || reloaded.SessionSecret != "secret-sess-verified" { - t.Fatalf("VERIFY_REQUIRED cleared the valid gateway session: %+v", reloaded) - } - - blockedResult := runtime.signedSessionFetch(call).Export().(map[string]any) - if blockedResult["needsVerification"] != true { - t.Fatalf("blocked generation did not join verification: %+v", blockedResult) - } - if calls != 2 { - t.Fatalf("blocked generation reached the gateway again: calls=%d", calls) - } - verificationRequired, err := runtime.preflightSignedSession() - if err != nil || !verificationRequired { - t.Fatalf("preflight did not preserve the blocked verification state: required=%v err=%v", verificationRequired, err) - } - if calls != 2 { - t.Fatalf("blocked preflight reached the gateway again: calls=%d", calls) - } - status := runtime.signedSessionStatus(goja.FunctionCall{}).Export().(map[string]any) - if status["authenticated"] != false || status["verification_required"] != true { - t.Fatalf("blocked generation status is incorrect: %+v", status) - } - - if err := runtime.exchangeSignedSessionGrant("grant-after-verify"); err != nil { - t.Fatalf("exchange grant: %v", err) - } - afterVerification := runtime.signedSessionFetch(call).Export().(map[string]any) - if afterVerification["ok"] != true { - t.Fatalf("new verified generation remained blocked: %+v", afterVerification) - } - if calls != 4 { - t.Fatalf("network calls = %d, want 428, bootstrap, exchange, success", calls) - } -} - -func TestSignedSessionFetchRequestAuthInvalidPreservesSession(t *testing.T) { - t.Run("current generation returns the error without bootstrap", func(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return &http.Response{ - StatusCode: http.StatusForbidden, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"error":"Forbidden","code":"REQUEST_AUTH_INVALID","origin":"gateway","retryable":false}`, - )), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "request-auth-current", transport) - config := SignedSessionConfig{Namespace: "request-auth-current", BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - resolved := saveUsableSignedSession(t, runtime, config, "sess-request-auth") - - call := goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("POST"), - runtime.vm.ToValue("/tickets"), - }} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - if result["code"] != "REQUEST_AUTH_INVALID" || result["needsVerification"] == true { - t.Fatalf("REQUEST_AUTH_INVALID was not returned as a non-verification error: %+v", result) - } - if calls != 1 { - t.Fatalf("current generation request was retried or bootstrapped: calls=%d", calls) - } - reloaded, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-request-auth" || reloaded.SessionSecret != "secret-sess-request-auth" { - t.Fatalf("REQUEST_AUTH_INVALID mutated the current session: %+v", reloaded) - } - }) - - t.Run("stale generation retries once with the current session", func(t *testing.T) { - oldRequestStarted := make(chan struct{}) - releaseOldRequest := make(chan struct{}) - var requestCalls atomic.Int32 - - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch { - case strings.HasSuffix(req.URL.Path, "/session/exchange"): - payload, _ := json.Marshal(signedSessionExchangeResponse{ - SessionID: "sess-request-new", - SessionSecret: "secret-request-new", - ExpiresAt: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - case strings.HasSuffix(req.URL.Path, "/tickets"): - requestCalls.Add(1) - if req.Header.Get("X-Sig-Session") == "sess-request-old" { - close(oldRequestStarted) - <-releaseOldRequest - return &http.Response{ - StatusCode: http.StatusForbidden, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"error":"Forbidden","code":"REQUEST_AUTH_INVALID","origin":"gateway","retryable":false}`, - )), - Request: req, - }, nil - } - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(`{"ticket_id":"fresh"}`)), - Request: req, - }, nil - default: - t.Fatalf("unexpected request: %s", req.URL.String()) - return nil, nil - } - }) - - root := t.TempDir() - config := &SignedSessionConfig{Namespace: "request-auth-shared", BaseURL: "https://auth.example.com"} - newRuntime := func(extensionID string) *extensionRuntime { - return &extensionRuntime{ - extensionID: extensionID, - manifest: &ExtensionManifest{ - Name: extensionID, - SignedSession: config, - }, - dataDir: filepath.Join(root, extensionID), - vm: goja.New(), - httpClient: &http.Client{Transport: transport}, - } - } - runtimeA := newRuntime("request-auth-a") - runtimeB := newRuntime("request-auth-b") - resolved := saveUsableSignedSession(t, runtimeA, *config, "sess-request-old") - - resultCh := make(chan map[string]any, 1) - go func() { - call := goja.FunctionCall{Arguments: []goja.Value{ - runtimeA.vm.ToValue("POST"), - runtimeA.vm.ToValue("/tickets"), - }} - resultCh <- runtimeA.signedSessionFetch(call).Export().(map[string]any) - }() - <-oldRequestStarted - if err := runtimeB.exchangeSignedSessionGrant("grant-request-auth"); err != nil { - t.Fatalf("exchange grant: %v", err) - } - close(releaseOldRequest) - - result := <-resultCh - if result["ok"] != true || requestCalls.Load() != 2 { - t.Fatalf("stale request was not rebuilt once: calls=%d result=%+v", requestCalls.Load(), result) - } - reloaded, err := runtimeA.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-request-new" || reloaded.SessionSecret != "secret-request-new" { - t.Fatalf("stale REQUEST_AUTH_INVALID changed the new session: %+v", reloaded) - } - }) -} - -func TestSignedSessionFetchProviderContractsNeverClearSession(t *testing.T) { - t.Run("provider authentication failure passes through without retry", func(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return &http.Response{ - StatusCode: http.StatusBadGateway, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"error":"Provider authentication failed","code":"PROVIDER_AUTH_FAILED","origin":"provider","retryable":false,"retry_mode":"none"}`, - )), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-auth-failed", transport) - config := SignedSessionConfig{Namespace: "provider-auth-failed", BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - resolved := saveUsableSignedSession(t, runtime, config, "sess-provider-auth") - - call := goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("POST"), - runtime.vm.ToValue("/tickets"), - }} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - if result["code"] != "PROVIDER_AUTH_FAILED" || - result["origin"] != "provider" || - result["retryMode"] != "none" { - t.Fatalf("provider contract was not exposed to the extension: %+v", result) - } - if result["needsVerification"] == true || calls != 1 { - t.Fatalf("provider auth failure triggered verification or retry: calls=%d result=%+v", calls, result) - } - reloaded, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-provider-auth" { - t.Fatalf("provider auth failure cleared the gateway session: %+v", reloaded) - } - }) - - t.Run("non-replay retry modes fail closed", func(t *testing.T) { - previousWait := signedSessionProviderWait - waitCalls := 0 - signedSessionProviderWait = func(context.Context, time.Duration) error { - waitCalls++ - return nil - } - t.Cleanup(func() { signedSessionProviderWait = previousWait }) - - tests := []struct { - name string - modeJSON string - wantMode string - }{ - {name: "missing mode"}, - {name: "none", modeJSON: `,"retry_mode":"none"`, wantMode: "none"}, - {name: "new ticket", modeJSON: `,"retry_mode":"new_ticket"`, wantMode: "new_ticket"}, - {name: "poll existing", modeJSON: `,"retry_mode":"poll_existing"`, wantMode: "poll_existing"}, - {name: "unknown", modeJSON: `,"retry_mode":"future_mode"`, wantMode: "future_mode"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - body := fmt.Sprintf( - `{"error":"Provider temporarily unavailable","code":"PROVIDER_UNAVAILABLE","origin":"provider","retryable":true%s,"retry_after_seconds":10}`, - tc.modeJSON, - ) - return &http.Response{ - StatusCode: http.StatusServiceUnavailable, - Header: http.Header{"Retry-After": []string{"10"}}, - Body: io.NopCloser(strings.NewReader(body)), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "retry-mode-"+strings.ReplaceAll(tc.name, " ", "-"), transport) - config := SignedSessionConfig{Namespace: runtime.extensionID, BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - resolved := saveUsableSignedSession(t, runtime, config, "sess-retry-mode") - waitsBefore := waitCalls - - call := goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("POST"), - runtime.vm.ToValue("/tickets"), - }} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - if calls != 1 || waitCalls != waitsBefore { - t.Fatalf("mode %q was replayed: calls=%d waits=%d", tc.wantMode, calls, waitCalls-waitsBefore) - } - if result["retryMode"] != tc.wantMode { - t.Fatalf("retryMode = %v, want %q: %+v", result["retryMode"], tc.wantMode, result) - } - reloaded, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-retry-mode" { - t.Fatalf("mode %q changed gateway session: %+v", tc.wantMode, reloaded) - } - }) - } - }) - - t.Run("temporary provider failure retries with Retry-After", func(t *testing.T) { - previousWait := signedSessionProviderWait - previousNow := signedSessionRequestNow - var waits []time.Duration - signedSessionProviderWait = func(ctx context.Context, delay time.Duration) error { - if err := ctx.Err(); err != nil { - return fmt.Errorf("retry context was already done after the response closed: %w", err) - } - waits = append(waits, delay) - return nil - } - nextRequestTime := time.Date(2026, time.July, 30, 12, 0, 0, 0, time.UTC) - signedSessionRequestNow = func() time.Time { - current := nextRequestTime - nextRequestTime = nextRequestTime.Add(time.Second) - return current - } - t.Cleanup(func() { - signedSessionProviderWait = previousWait - signedSessionRequestNow = previousNow - }) - - calls := 0 - nonces := make(map[string]struct{}) - timestamps := make(map[string]struct{}) - signatures := make(map[string]struct{}) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - nonces[req.Header.Get("X-Sig-Nonce")] = struct{}{} - timestamps[req.Header.Get("X-Sig-Timestamp")] = struct{}{} - signatures[req.Header.Get("X-Sig-Signature")] = struct{}{} - if calls <= signedSessionMaxProviderRetries { - return &http.Response{ - StatusCode: http.StatusServiceUnavailable, - Header: http.Header{"Retry-After": []string{"10"}}, - Body: io.NopCloser(strings.NewReader( - `{"error":"Provider temporarily unavailable","code":"PROVIDER_UNAVAILABLE","origin":"provider","retryable":true,"retry_mode":"same_operation","retry_after_seconds":30}`, - )), - Request: req, - }, nil - } - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(`{"ok":true}`)), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-unavailable", transport) - // Production extension API clients have a finite timeout. net/http - // cancels that per-request context after the response body is closed; - // provider retry waits must outlive the completed request. - runtime.httpClient.Timeout = 15 * time.Second - config := SignedSessionConfig{Namespace: "provider-unavailable", BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - resolved := saveUsableSignedSession(t, runtime, config, "sess-provider-retry") - - call := goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("POST"), - runtime.vm.ToValue("/tickets"), - }} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - if result["ok"] != true || calls != signedSessionMaxProviderRetries+1 { - t.Fatalf("provider retry did not recover: calls=%d result=%+v", calls, result) - } - if len(waits) != signedSessionMaxProviderRetries { - t.Fatalf("provider retry waits = %v", waits) - } - for _, delay := range waits { - if delay != 10*time.Second { - t.Fatalf("Retry-After header was not authoritative: waits=%v", waits) - } - } - wantSignedAttempts := signedSessionMaxProviderRetries + 1 - if len(nonces) != wantSignedAttempts || - len(timestamps) != wantSignedAttempts || - len(signatures) != wantSignedAttempts { - t.Fatalf( - "provider retries reused signed request headers: nonces=%d timestamps=%d signatures=%d want=%d", - len(nonces), - len(timestamps), - len(signatures), - wantSignedAttempts, - ) - } - reloaded, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-provider-retry" { - t.Fatalf("provider retry changed the gateway session: %+v", reloaded) - } - }) - - t.Run("temporary provider retry still honors user cancellation", func(t *testing.T) { - previousWait := signedSessionProviderWait - const itemID = "provider-retry-user-cancel" - signedSessionProviderWait = func(ctx context.Context, _ time.Duration) error { - cancelDownload(itemID) - return sleepRetry(ctx, time.Hour) - } - t.Cleanup(func() { signedSessionProviderWait = previousWait }) - - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return &http.Response{ - StatusCode: http.StatusServiceUnavailable, - Header: http.Header{"Retry-After": []string{"10"}}, - Body: io.NopCloser(strings.NewReader( - `{"error":"Provider temporarily unavailable","code":"PROVIDER_UNAVAILABLE","origin":"provider","retryable":true,"retry_mode":"same_operation","retry_after_seconds":10}`, - )), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-retry-user-cancel", transport) - runtime.httpClient.Timeout = 15 * time.Second - runtime.setActiveDownloadItemID(itemID) - initDownloadCancel(itemID) - t.Cleanup(func() { - clearDownloadCancel(itemID) - runtime.clearActiveDownloadItemID() - }) - config := SignedSessionConfig{Namespace: runtime.extensionID, BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - saveUsableSignedSession(t, runtime, config, "sess-provider-cancel") - - call := goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("POST"), - runtime.vm.ToValue("/tickets"), - }} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - if calls != 1 || !strings.Contains(fmt.Sprint(result["error"]), context.Canceled.Error()) { - t.Fatalf("user cancellation did not stop provider retry: calls=%d result=%+v", calls, result) - } - }) - - t.Run("temporary provider retry budget is bounded", func(t *testing.T) { - previousWait := signedSessionProviderWait - signedSessionProviderWait = func(context.Context, time.Duration) error { return nil } - t.Cleanup(func() { signedSessionProviderWait = previousWait }) - - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return &http.Response{ - StatusCode: http.StatusServiceUnavailable, - Header: http.Header{"Retry-After": []string{"1"}}, - Body: io.NopCloser(strings.NewReader( - `{"error":"Provider temporarily unavailable","code":"PROVIDER_UNAVAILABLE","origin":"provider","retryable":true,"retry_mode":"same_operation","retry_after_seconds":1}`, - )), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-unavailable-bounded", transport) - config := SignedSessionConfig{Namespace: "provider-unavailable-bounded", BaseURL: "https://auth.example.com"} - runtime.manifest.SignedSession = &config - resolved := saveUsableSignedSession(t, runtime, config, "sess-provider-bounded") - - call := goja.FunctionCall{Arguments: []goja.Value{ - runtime.vm.ToValue("POST"), - runtime.vm.ToValue("/tickets"), - }} - result := runtime.signedSessionFetch(call).Export().(map[string]any) - if calls != signedSessionMaxProviderRetries+1 { - t.Fatalf("provider attempts = %d, want %d", calls, signedSessionMaxProviderRetries+1) - } - if result["code"] != "PROVIDER_UNAVAILABLE" || result["retryable"] != true { - t.Fatalf("final provider failure was not returned intact: %+v", result) - } - reloaded, err := runtime.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-provider-bounded" { - t.Fatalf("exhausted provider retries changed the gateway session: %+v", reloaded) - } - }) -} - -func TestStaleSignedSession401RetriesWithoutClearingExchangedSession(t *testing.T) { - oldRequestStarted := make(chan struct{}) - releaseOldRequest := make(chan struct{}) - var searchCalls atomic.Int32 - - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch { - case strings.HasSuffix(req.URL.Path, "/session/exchange"): - payload, _ := json.Marshal(signedSessionExchangeResponse{ - SessionID: "sess-new", - SessionSecret: "secret-new", - ExpiresAt: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - case strings.HasSuffix(req.URL.Path, "/tracks/search"): - searchCalls.Add(1) - if req.Header.Get("X-Sig-Session") == "sess-old" { - close(oldRequestStarted) - <-releaseOldRequest - return &http.Response{ - StatusCode: http.StatusUnauthorized, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"error":"Unauthorized","code":"SESSION_INVALID","origin":"gateway","action":"bootstrap_session"}`, - )), - Request: req, - }, nil - } - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(`{"ok":true}`)), - Request: req, - }, nil - default: - t.Fatalf("unexpected request: %s", req.URL.String()) - return nil, nil - } - }) - - root := t.TempDir() - config := &SignedSessionConfig{ - Namespace: "shared-stale-session", - BaseURL: "https://auth.example.com", - } - newRuntime := func(extensionID string) *extensionRuntime { - return &extensionRuntime{ - extensionID: extensionID, - manifest: &ExtensionManifest{ - Name: extensionID, - SignedSession: config, - }, - dataDir: filepath.Join(root, extensionID), - vm: goja.New(), - httpClient: &http.Client{Transport: transport}, - } - } - runtimeA := newRuntime("download-a") - runtimeB := newRuntime("download-b") - resolved := signedSessionConfigWithDefaults(config) - record, err := runtimeA.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - record.SessionID = "sess-old" - record.SessionSecret = "secret-old" - record.ExpiresAt = time.Now().Add(time.Hour).UTC().Format(time.RFC3339) - if err := runtimeA.saveSignedSession(resolved, record); err != nil { - t.Fatal(err) - } - - resultCh := make(chan map[string]any, 1) - go func() { - call := goja.FunctionCall{Arguments: []goja.Value{ - runtimeA.vm.ToValue("GET"), - runtimeA.vm.ToValue("/tracks/search"), - }} - resultCh <- runtimeA.signedSessionFetch(call).Export().(map[string]any) - }() - <-oldRequestStarted - if err := runtimeB.exchangeSignedSessionGrant("grant-new"); err != nil { - t.Fatalf("exchange grant: %v", err) - } - close(releaseOldRequest) - - result := <-resultCh - if result["ok"] != true { - t.Fatalf("stale request was not retried with the new session: %+v", result) - } - if got := searchCalls.Load(); got != 2 { - t.Fatalf("signed search calls = %d, want stale request plus one retry", got) - } - reloaded, err := runtimeA.loadSignedSession(resolved) - if err != nil { - t.Fatal(err) - } - if reloaded.SessionID != "sess-new" || reloaded.SessionSecret != "secret-new" { - t.Fatalf("stale 401 overwrote exchanged session: %+v", reloaded) - } -} - -func TestExchangeSignedSessionGrant(t *testing.T) { - t.Run("success stores the exchanged session", func(t *testing.T) { - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - if !strings.HasSuffix(req.URL.Path, "/session/exchange") { - t.Fatalf("unexpected path: %s", req.URL.Path) - } - payload := signedSessionExchangeResponse{SessionID: "sess-9", SessionSecret: "secret-9", ExpiresAt: "2030-01-01T00:00:00Z"} - body, _ := json.Marshal(payload) - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(body))), Request: req}, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-ext", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"} - - if err := runtime.exchangeSignedSessionGrant("grant-token"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := runtime.exchangeSignedSessionGrant("grant-token"); err != nil { - t.Fatalf("duplicate callback was not idempotent: %v", err) - } - if calls != 1 { - t.Fatalf("duplicate grant exchange calls = %d, want 1", calls) - } - - config := signedSessionConfigWithDefaults(runtime.manifest.SignedSession) - record, err := runtime.loadSignedSession(config) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if record.SessionID != "sess-9" || record.SessionSecret != "secret-9" { - t.Fatalf("session was not persisted after exchange: %+v", record) - } - }) - - t.Run("non-2xx response is surfaced as an error", func(t *testing.T) { - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: 400, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-ext", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"} - - if err := runtime.exchangeSignedSessionGrant("bad-grant"); err == nil { - t.Fatal("expected an error for a non-2xx exchange response") - } - }) - - t.Run("429 preserves the grant and retries after the server delay", func(t *testing.T) { - pendingSignedSessionGrantsMu.Lock() - pendingSignedSessionGrants = make(map[string]string) - pendingSignedSessionGrantsMu.Unlock() - previousWait := signedSessionRetryWaitContext - var waits []time.Duration - signedSessionRetryWaitContext = func(_ context.Context, delay time.Duration) error { - waits = append(waits, delay) - return nil - } - t.Cleanup(func() { signedSessionRetryWaitContext = previousWait }) - - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - if calls == 1 { - return &http.Response{ - StatusCode: http.StatusTooManyRequests, - Header: http.Header{"Retry-After": []string{"7"}}, - Body: io.NopCloser(strings.NewReader(`{}`)), - Request: req, - }, nil - } - payload, _ := json.Marshal(signedSessionExchangeResponse{ - SessionID: "sess-after-429", - SessionSecret: "secret-after-429", - ExpiresAt: "2030-01-01T00:00:00Z", - }) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(payload))), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-rate-limit", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "provider-rate-limit", - BaseURL: "https://auth.example.com", - } - setPendingSignedSessionGrant(runtime.extensionID, "grant-preserved") - - result := runtime.signedSessionCompleteGrant(goja.FunctionCall{}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("grant exchange did not recover from 429: %+v", result) - } - if calls != 2 { - t.Fatalf("exchange calls = %d, want 2", calls) - } - if len(waits) != 1 || waits[0] != 7*time.Second { - t.Fatalf("exchange retry waits = %v, want [7s]", waits) - } - pendingSignedSessionGrantsMu.Lock() - _, stillPending := pendingSignedSessionGrants[runtime.extensionID] - pendingSignedSessionGrantsMu.Unlock() - if stillPending { - t.Fatal("grant was not cleared after successful retry") - } - }) - - t.Run("exhausted 429 retries keep the grant for a later retry", func(t *testing.T) { - pendingSignedSessionGrantsMu.Lock() - pendingSignedSessionGrants = make(map[string]string) - pendingSignedSessionGrantsMu.Unlock() - previousWait := signedSessionRetryWaitContext - signedSessionRetryWaitContext = func(context.Context, time.Duration) error { - return nil - } - t.Cleanup(func() { signedSessionRetryWaitContext = previousWait }) - - calls := 0 - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return &http.Response{ - StatusCode: http.StatusTooManyRequests, - Header: http.Header{"Retry-After": []string{"3"}}, - Body: io.NopCloser(strings.NewReader(`{}`)), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-rate-limit-exhausted", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{ - Namespace: "provider-rate-limit-exhausted", - BaseURL: "https://auth.example.com", - } - setPendingSignedSessionGrant(runtime.extensionID, "grant-retry-later") - - result := runtime.signedSessionCompleteGrant(goja.FunctionCall{}).Export().(map[string]any) - if result["success"] != false { - t.Fatalf("exhausted rate limit unexpectedly succeeded: %+v", result) - } - if calls != signedSessionExchangeMaxAttempts { - t.Fatalf("exchange calls = %d, want %d", calls, signedSessionExchangeMaxAttempts) - } - pendingSignedSessionGrantsMu.Lock() - pendingGrant := pendingSignedSessionGrants[runtime.extensionID] - pendingSignedSessionGrantsMu.Unlock() - if pendingGrant != "grant-retry-later" { - t.Fatalf("pending grant = %q, want it preserved for retry", pendingGrant) - } - }) -} - -func TestSetPendingSignedSessionGrant(t *testing.T) { - pendingSignedSessionGrantsMu.Lock() - pendingSignedSessionGrants = make(map[string]string) - pendingSignedSessionGrantsMu.Unlock() - - setPendingSignedSessionGrant(" ext-a ", " grant-1 ") - - pendingSignedSessionGrantsMu.Lock() - got := pendingSignedSessionGrants["ext-a"] - pendingSignedSessionGrantsMu.Unlock() - - if got != "grant-1" { - t.Fatalf("expected trimmed grant to be stored, got %q", got) - } - - setPendingSignedSessionGrant("", "grant-2") - setPendingSignedSessionGrant("ext-b", "") - - pendingSignedSessionGrantsMu.Lock() - _, hasEmptyExt := pendingSignedSessionGrants[""] - _, hasEmptyGrant := pendingSignedSessionGrants["ext-b"] - pendingSignedSessionGrantsMu.Unlock() - - if hasEmptyExt || hasEmptyGrant { - t.Fatal("expected empty extensionID/grant pairs to be ignored") - } -} - -func TestRefreshSignedSession(t *testing.T) { - t.Run("updates changed fields and persists them", func(t *testing.T) { - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - if !strings.HasSuffix(req.URL.Path, "/session/refresh") { - t.Fatalf("unexpected path: %s", req.URL.Path) - } - payload := signedSessionExchangeResponse{SessionSecret: "rotated-secret", ExpiresAt: "2031-01-01T00:00:00Z"} - body, _ := json.Marshal(payload) - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(body))), Request: req}, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-ext", transport) - config := signedSessionConfigWithDefaults(&SignedSessionConfig{ - Namespace: "provider", BaseURL: "https://auth.example.com", - Endpoints: SignedSessionEndpoints{Refresh: "/session/refresh"}, - }) - record := &signedSessionRecord{InstallID: "install-1", SessionID: "sess-1", SessionSecret: "old-secret", ExpiresAt: "2030-01-01T00:00:00Z"} - - if err := runtime.refreshSignedSession(config, record); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if record.SessionSecret != "rotated-secret" || record.ExpiresAt != "2031-01-01T00:00:00Z" { - t.Fatalf("refreshed fields not applied in-memory: %+v", record) - } - if record.SessionID != "sess-1" { - t.Fatalf("session id should be untouched when the response omits it: %q", record.SessionID) - } - - reloaded, err := runtime.loadSignedSession(config) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if reloaded.SessionSecret != "rotated-secret" { - t.Fatalf("refresh was not persisted to disk: %+v", reloaded) - } - }) - - t.Run("non-2xx response is surfaced as an error", func(t *testing.T) { - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: 500, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-ext", transport) - config := signedSessionConfigWithDefaults(&SignedSessionConfig{ - Namespace: "provider", BaseURL: "https://auth.example.com", - Endpoints: SignedSessionEndpoints{Refresh: "/session/refresh"}, - }) - record := &signedSessionRecord{InstallID: "install-1", SessionID: "sess-1", SessionSecret: "old-secret"} - - if err := runtime.refreshSignedSession(config, record); err == nil { - t.Fatal("expected an error for a non-2xx refresh response") - } - }) -} - -func TestRefreshSignedSessionCoalescesWithoutHoldingCoordinatorMutex(t *testing.T) { - var calls int32 - requestStarted := make(chan struct{}) - releaseRequest := make(chan struct{}) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - if atomic.AddInt32(&calls, 1) == 1 { - close(requestStarted) - } - <-releaseRequest - payload := signedSessionExchangeResponse{ - SessionSecret: "rotated-secret", - ExpiresAt: time.Now().Add(2 * time.Hour).UTC().Format(time.RFC3339), - } - body, _ := json.Marshal(payload) - return &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(string(body))), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "refresh-coalesced", transport) - config := signedSessionConfigWithDefaults(&SignedSessionConfig{ - Namespace: "refresh-coalesced", - BaseURL: "https://auth.example.com", - Endpoints: SignedSessionEndpoints{Refresh: "/session/refresh"}, - }) - record, err := runtime.loadSignedSession(config) - if err != nil { - t.Fatal(err) - } - record.SessionID = "session-1" - record.SessionSecret = "old-secret" - record.ExpiresAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339) - if err := runtime.saveSignedSession(config, record); err != nil { - t.Fatal(err) - } - coordinator, err := runtime.signedSessionCoordinator(config) - if err != nil { - t.Fatal(err) - } - - const workers = 12 - results := make(chan *signedSessionRecord, workers) - errors := make(chan error, workers) - var wg sync.WaitGroup - wg.Add(workers) - for range workers { - go func() { - defer wg.Done() - refreshed, refreshErr := runtime.refreshSignedSessionCoalesced(config, coordinator) - results <- refreshed - errors <- refreshErr - }() - } - select { - case <-requestStarted: - case <-time.After(time.Second): - t.Fatal("refresh request did not start") - } - - mutexAvailable := make(chan struct{}) - go func() { - coordinator.mu.Lock() - defer coordinator.mu.Unlock() - close(mutexAvailable) - }() - select { - case <-mutexAvailable: - case <-time.After(250 * time.Millisecond): - t.Fatal("coordinator mutex was held during refresh HTTP") - } - - close(releaseRequest) - wg.Wait() - close(results) - close(errors) - for refreshErr := range errors { - if refreshErr != nil { - t.Fatalf("coalesced refresh failed: %v", refreshErr) - } - } - for refreshed := range results { - if refreshed == nil || refreshed.SessionSecret != "rotated-secret" { - t.Fatalf("unexpected refreshed record: %+v", refreshed) - } - } - if got := atomic.LoadInt32(&calls); got != 1 { - t.Fatalf("refresh requests = %d, want 1", got) - } -} - -func TestRefreshSignedSessionSharesFailureWithWaiters(t *testing.T) { - var calls atomic.Int32 - requestStarted := make(chan struct{}) - releaseRequest := make(chan struct{}) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - if calls.Add(1) == 1 { - close(requestStarted) - } - <-releaseRequest - return &http.Response{ - StatusCode: http.StatusServiceUnavailable, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(`{}`)), - Request: req, - }, nil - }) - runtime := newSignedSessionTestRuntime(t, "refresh-failure-coalesced", transport) - config := signedSessionConfigWithDefaults(&SignedSessionConfig{ - Namespace: "refresh-failure-coalesced", - BaseURL: "https://auth.example.com", - Endpoints: SignedSessionEndpoints{Refresh: "/session/refresh"}, - }) - record, err := runtime.loadSignedSession(config) - if err != nil { - t.Fatal(err) - } - record.SessionID = "session-1" - record.SessionSecret = "old-secret" - record.ExpiresAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339) - if err := runtime.saveSignedSession(config, record); err != nil { - t.Fatal(err) - } - coordinator, err := runtime.signedSessionCoordinator(config) - if err != nil { - t.Fatal(err) - } - - const workers = 12 - errors := make(chan error, workers) - for range workers { - go func() { - _, refreshErr := runtime.refreshSignedSessionCoalesced(config, coordinator) - errors <- refreshErr - }() - } - select { - case <-requestStarted: - case <-time.After(time.Second): - t.Fatal("refresh request did not start") - } - time.Sleep(50 * time.Millisecond) - close(releaseRequest) - for range workers { - if refreshErr := <-errors; refreshErr == nil || !strings.Contains(refreshErr.Error(), "HTTP 503") { - t.Fatalf("unexpected coalesced refresh error: %v", refreshErr) - } - } - if got := calls.Load(); got != 1 { - t.Fatalf("parallel failed refresh calls = %d, want 1", got) - } -} - -func TestSignedSessionCompleteGrant(t *testing.T) { - t.Run("uses the grant argument when provided", func(t *testing.T) { - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - payload := signedSessionExchangeResponse{SessionID: "sess-arg", SessionSecret: "secret-arg", ExpiresAt: "2030-01-01T00:00:00Z"} - body, _ := json.Marshal(payload) - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(body))), Request: req}, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-ext", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"} - - call := goja.FunctionCall{Arguments: []goja.Value{runtime.vm.ToValue("grant-from-arg")}} - result := runtime.signedSessionCompleteGrant(call).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("expected success, got %+v", result) - } - }) - - t.Run("falls back to a pending grant registered out of band", func(t *testing.T) { - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - payload := signedSessionExchangeResponse{SessionID: "sess-pending", SessionSecret: "secret-pending", ExpiresAt: "2030-01-01T00:00:00Z"} - body, _ := json.Marshal(payload) - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(body))), Request: req}, nil - }) - runtime := newSignedSessionTestRuntime(t, "provider-ext-pending", transport) - runtime.manifest.SignedSession = &SignedSessionConfig{Namespace: "provider", BaseURL: "https://auth.example.com"} - setPendingSignedSessionGrant(runtime.extensionID, "pending-grant") - - result := runtime.signedSessionCompleteGrant(goja.FunctionCall{}).Export().(map[string]any) - if result["success"] != true { - t.Fatalf("expected success, got %+v", result) - } - - pendingSignedSessionGrantsMu.Lock() - _, stillPending := pendingSignedSessionGrants[runtime.extensionID] - pendingSignedSessionGrantsMu.Unlock() - if stillPending { - t.Fatal("expected the pending grant to be consumed after use") - } - }) - - t.Run("no grant available reports failure", func(t *testing.T) { - runtime := newSignedSessionTestRuntime(t, "provider-ext-none", nil) - result := runtime.signedSessionCompleteGrant(goja.FunctionCall{}).Export().(map[string]any) - if result["success"] != false { - t.Fatalf("expected failure without a grant, got %+v", result) - } - }) -} - -func TestBuildSignedSessionChallengeURL(t *testing.T) { - config := signedSessionConfigWithDefaults(&SignedSessionConfig{ - Namespace: "provider", - BaseURL: "https://auth.example.com", - CallbackURL: "spotiflac://session-grant", - }) - runtime := newSignedSessionTestRuntime(t, "provider-ext", nil) - - got := runtime.buildSignedSessionChallengeURL(config, "chal-123", "state-123") - - if !strings.HasPrefix(got, "https://auth.example.com/challenge?") { - t.Fatalf("unexpected base URL: %q", got) - } - if !strings.Contains(got, "id=chal-123") { - t.Fatalf("expected challenge id in query: %q", got) - } - if !strings.Contains(got, "cb=spotiflac%3A%2F%2Fsession-grant") { - t.Fatalf("expected encoded callback URL in query: %q", got) - } -} diff --git a/go_backend/extension_test.go b/go_backend/extension_test.go deleted file mode 100644 index 0c873a3f..00000000 --- a/go_backend/extension_test.go +++ /dev/null @@ -1,610 +0,0 @@ -package gobackend - -import ( - "context" - "errors" - "net/http" - "path/filepath" - "strconv" - "testing" - "time" - - "github.com/dop251/goja" -) - -func TestParseManifest_Valid(t *testing.T) { - validManifest := `{ - "name": "test-provider", - "displayName": "Test Provider", - "version": "1.0.0", - "description": "A test extension", - "type": ["metadata_provider"], - "permissions": { - "network": ["api.test.com"], - "storage": true - } - }` - - manifest, err := ParseManifest([]byte(validManifest)) - if err != nil { - t.Fatalf("Expected valid manifest to parse, got error: %v", err) - } - - if manifest.Name != "test-provider" { - t.Errorf("Expected name 'test-provider', got '%s'", manifest.Name) - } - - if manifest.Version != "1.0.0" { - t.Errorf("Expected version '1.0.0', got '%s'", manifest.Version) - } - - if !manifest.IsMetadataProvider() { - t.Error("Expected IsMetadataProvider() to return true") - } - - if manifest.IsDownloadProvider() { - t.Error("Expected IsDownloadProvider() to return false") - } -} - -func TestExtensionManifestStopsProviderFallback(t *testing.T) { - modernManifest := &ExtensionManifest{StopProviderFallback: true} - if !modernManifest.StopsProviderFallback() { - t.Fatal("expected stopProviderFallback to stop provider fallback") - } - - legacyManifest := &ExtensionManifest{SkipBuiltInFallback: true} - if !legacyManifest.StopsProviderFallback() { - t.Fatal("expected legacy skipBuiltInFallback to stop provider fallback") - } - - defaultManifest := &ExtensionManifest{} - if defaultManifest.StopsProviderFallback() { - t.Fatal("expected default manifest to allow provider fallback") - } -} - -func TestParseManifest_MissingName(t *testing.T) { - invalidManifest := `{ - "version": "1.0.0", - "description": "A test extension", - "type": ["metadata_provider"] - }` - - _, err := ParseManifest([]byte(invalidManifest)) - if err == nil { - t.Fatal("Expected error for missing name") - } -} - -func TestParseManifestRejectsUnsafeExtensionIDs(t *testing.T) { - for _, name := range []string{"../escape", `..\\escape`, "/absolute", "UpperCase", "."} { - manifest := `{"name":` + strconv.Quote(name) + `,"version":"1.0.0","description":"test","type":["metadata_provider"]}` - if _, err := ParseManifest([]byte(manifest)); err == nil { - t.Fatalf("expected unsafe extension ID %q to be rejected", name) - } - } -} - -func TestManifestPrivilegedCapabilitiesRequirePermissions(t *testing.T) { - rawFFmpeg := &ExtensionManifest{ - Name: "raw-ffmpeg", - Version: "1.0.0", - Description: "test", - Types: []ExtensionType{ExtensionTypeDownloadProvider}, - Capabilities: map[string]any{"rawFfmpeg": true}, - } - if err := rawFFmpeg.Validate(); err == nil { - t.Fatal("expected rawFfmpeg without file permission to be rejected") - } - - signedSession := &ExtensionManifest{ - Name: "signed-session", - Version: "1.0.0", - Description: "test", - Types: []ExtensionType{ExtensionTypeDownloadProvider}, - Permissions: ExtensionPermissions{Network: []string{"api.example.com"}}, - SignedSession: &SignedSessionConfig{ - Namespace: "test", - BaseURL: "https://api.example.com", - }, - } - if err := signedSession.Validate(); err == nil { - t.Fatal("expected signedSession without storage permission to be rejected") - } -} - -func TestParseManifest_MissingType(t *testing.T) { - invalidManifest := `{ - "name": "test-provider", - "version": "1.0.0", - "description": "A test extension" - }` - - _, err := ParseManifest([]byte(invalidManifest)) - if err == nil { - t.Fatal("Expected error for missing type") - } -} - -func TestIsDomainAllowed(t *testing.T) { - manifest := &ExtensionManifest{ - Permissions: ExtensionPermissions{ - Network: []string{"api.test.com", "*.example.com"}, - }, - } - - tests := []struct { - domain string - expected bool - }{ - {"api.test.com", true}, - {"api.example.com", true}, - {"sub.example.com", true}, - {"notallowed.com", false}, - {"test.com", false}, - } - - for _, tt := range tests { - result := manifest.IsDomainAllowed(tt.domain) - if result != tt.expected { - t.Errorf("IsDomainAllowed(%s) = %v, expected %v", tt.domain, result, tt.expected) - } - } -} - -func TestExtensionRuntime_NetworkSandbox(t *testing.T) { - ext := &loadedExtension{ - ID: "test-ext", - Manifest: &ExtensionManifest{ - Name: "test-ext", - Permissions: ExtensionPermissions{ - Network: []string{"api.allowed.com", "*.wildcard.com"}, - }, - }, - DataDir: t.TempDir(), - } - - runtime := newExtensionRuntime(ext) - - if err := runtime.validateDomain("https://api.allowed.com/path"); err != nil { - t.Errorf("Expected api.allowed.com to be allowed, got error: %v", err) - } - - if err := runtime.validateDomain("https://sub.wildcard.com/path"); err != nil { - t.Errorf("Expected sub.wildcard.com to be allowed (wildcard), got error: %v", err) - } - - if err := runtime.validateDomain("https://blocked.com/path"); err == nil { - t.Error("Expected blocked.com to be denied") - } - - if err := runtime.validateDomain("https://notallowed.com/path"); err == nil { - t.Error("Expected notallowed.com to be denied") - } - - if err := runtime.validateDomain("http://api.allowed.com/path"); err == nil { - t.Error("Expected http URL to be denied without allowHttp") - } - - ext.Manifest.Permissions.AllowHTTP = true - if err := runtime.validateDomain("http://api.allowed.com/path"); err != nil { - t.Errorf("Expected http URL to be allowed with allowHttp, got error: %v", err) - } -} - -func TestExtensionRuntime_FileSandbox(t *testing.T) { - tempDir := t.TempDir() - - ext := &loadedExtension{ - ID: "test-ext", - Manifest: &ExtensionManifest{ - Name: "test-ext", - Permissions: ExtensionPermissions{ - File: true, - }, - }, - DataDir: tempDir, - } - - runtime := newExtensionRuntime(ext) - - validPath, err := runtime.validatePath("test.txt") - if err != nil { - t.Errorf("Expected relative path to be valid, got error: %v", err) - } - if validPath == "" { - t.Error("Expected non-empty path") - } - - _, err = runtime.validatePath("../../../etc/passwd") - if err == nil { - t.Error("Expected path traversal to be blocked") - } - - nestedPath, err := runtime.validatePath("subdir/file.txt") - if err != nil { - t.Errorf("Expected nested path to be valid, got error: %v", err) - } - if nestedPath == "" { - t.Error("Expected non-empty nested path") - } - - var absPath string - if filepath.IsAbs("C:\\Windows\\System32") { - absPath = "C:\\Windows\\System32\\test.txt" - } else { - absPath = "/etc/passwd" - } - _, err = runtime.validatePath(absPath) - if err == nil { - t.Error("Expected absolute path to be blocked") - } - - extNoFile := &loadedExtension{ - ID: "test-ext-no-file", - Manifest: &ExtensionManifest{ - Name: "test-ext-no-file", - Permissions: ExtensionPermissions{ - File: false, - }, - }, - DataDir: tempDir, - } - runtimeNoFile := newExtensionRuntime(extNoFile) - _, err = runtimeNoFile.validatePath("test.txt") - if err == nil { - t.Error("Expected file access to be denied without file permission") - } -} - -func TestExtensionRuntime_UtilityFunctions(t *testing.T) { - ext := &loadedExtension{ - ID: "test-ext", - Manifest: &ExtensionManifest{ - Name: "test-ext", - }, - DataDir: t.TempDir(), - } - - runtime := newExtensionRuntime(ext) - vm := goja.New() - runtime.RegisterAPIs(vm) - - result, err := vm.RunString(`utils.base64Encode("hello")`) - if err != nil { - t.Fatalf("base64Encode failed: %v", err) - } - if result.String() != "aGVsbG8=" { - t.Errorf("Expected 'aGVsbG8=', got '%s'", result.String()) - } - - result, err = vm.RunString(`utils.base64Decode("aGVsbG8=")`) - if err != nil { - t.Fatalf("base64Decode failed: %v", err) - } - if result.String() != "hello" { - t.Errorf("Expected 'hello', got '%s'", result.String()) - } - - result, err = vm.RunString(`utils.md5("hello")`) - if err != nil { - t.Fatalf("md5 failed: %v", err) - } - if result.String() != "5d41402abc4b2a76b9719d911017c592" { - t.Errorf("Expected '5d41402abc4b2a76b9719d911017c592', got '%s'", result.String()) - } - - result, err = vm.RunString(`utils.stringifyJSON({name: "test", value: 123})`) - if err != nil { - t.Fatalf("stringifyJSON failed: %v", err) - } - if result.String() == "" { - t.Error("Expected non-empty JSON string") - } - - result, err = vm.RunString(`utils.sleep(1)`) - if err != nil { - t.Fatalf("sleep failed: %v", err) - } - if !result.ToBoolean() { - t.Error("Expected sleep to complete successfully") - } - - runtime.setActiveDownloadItemID("test-item") - cancelDownload("test-item") - t.Cleanup(func() { - clearDownloadCancel("test-item") - runtime.clearActiveDownloadItemID() - }) - - result, err = vm.RunString(`utils.isDownloadCancelled()`) - if err != nil { - t.Fatalf("isDownloadCancelled failed: %v", err) - } - if !result.ToBoolean() { - t.Error("Expected active download cancellation to be visible to JS") - } - - SetAppVersion("4.2.2") - t.Cleanup(func() { - SetAppVersion("") - }) - - result, err = vm.RunString(`utils.appVersion()`) - if err != nil { - t.Fatalf("appVersion failed: %v", err) - } - if got := result.String(); got != "4.2.2" { - t.Fatalf("Expected appVersion 4.2.2, got %q", got) - } - - result, err = vm.RunString(`utils.appUserAgent()`) - if err != nil { - t.Fatalf("appUserAgent failed: %v", err) - } - if got := result.String(); got != "SpotiFLAC-Mobile/4.2.2" { - t.Fatalf("Expected appUserAgent SpotiFLAC-Mobile/4.2.2, got %q", got) - } - - result, err = vm.RunString(`utils.sleep(50)`) - if err != nil { - t.Fatalf("cancel-aware sleep failed: %v", err) - } - if result.ToBoolean() { - t.Error("Expected sleep to abort when download is cancelled") - } -} - -func TestExtensionRuntime_BindDownloadCancelContext(t *testing.T) { - ext := &loadedExtension{ - ID: "test-ext", - Manifest: &ExtensionManifest{ - Name: "test-ext", - }, - DataDir: t.TempDir(), - } - - runtime := newExtensionRuntime(ext) - runtime.setActiveDownloadItemID("test-item") - initDownloadCancel("test-item") - t.Cleanup(func() { - clearDownloadCancel("test-item") - runtime.clearActiveDownloadItemID() - }) - - req, err := http.NewRequest("GET", "https://api.example.com/test", nil) - if err != nil { - t.Fatalf("NewRequest failed: %v", err) - } - - req = runtime.bindDownloadCancelContext(req) - downloadCancels.mu.Lock() - refs := downloadCancels.entries["test-item"].refs - downloadCancels.mu.Unlock() - if refs != 1 { - t.Fatalf("binding a request leaked a cancellation reference: %d", refs) - } - cancelDownload("test-item") - - select { - case <-req.Context().Done(): - case <-time.After(500 * time.Millisecond): - t.Fatal("Expected bound request context to be cancelled") - } - - if req.Context().Err() == nil { - t.Fatal("Expected request context error after cancellation") - } -} - -func TestExtensionRuntime_BindDownloadCancelContextPreservesPreCancelledState(t *testing.T) { - ext := &loadedExtension{ - ID: "test-ext", - Manifest: &ExtensionManifest{ - Name: "test-ext", - }, - DataDir: t.TempDir(), - } - - runtime := newExtensionRuntime(ext) - runtime.setActiveDownloadItemID("test-item") - cancelDownload("test-item") - initDownloadCancel("test-item") - t.Cleanup(func() { - clearDownloadCancel("test-item") - runtime.clearActiveDownloadItemID() - }) - - req, err := http.NewRequest("GET", "https://api.example.com/test", nil) - if err != nil { - t.Fatalf("NewRequest failed: %v", err) - } - - req = runtime.bindDownloadCancelContext(req) - - select { - case <-req.Context().Done(): - case <-time.After(500 * time.Millisecond): - t.Fatal("Expected pre-cancelled request context to stay cancelled") - } - - if req.Context().Err() == nil { - t.Fatal("Expected request context error for pre-cancelled item") - } -} - -func TestRunWithTimeoutContextCancelsExecution(t *testing.T) { - vm := goja.New() - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - _, err := RunWithTimeoutContextAndRecover(ctx, vm, `while (true) {}`, 5*time.Second) - if !errors.Is(err, ErrExtensionRequestCancelled) { - t.Fatalf("expected extension request cancellation, got %v", err) - } -} - -func TestExtensionRuntime_BindExtensionRequestCancelContext(t *testing.T) { - ext := &loadedExtension{ - ID: "test-ext", - Manifest: &ExtensionManifest{ - Name: "test-ext", - }, - DataDir: t.TempDir(), - } - runtime := newExtensionRuntime(ext) - - const requestID = "test-extension-request" - clearExtensionRequestCancel(requestID) - defer clearExtensionRequestCancel(requestID) - - runtime.setActiveRequestID(requestID) - defer runtime.clearActiveRequestID() - initExtensionRequestCancel(requestID) - - req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) - if err != nil { - t.Fatalf("new request: %v", err) - } - req = runtime.bindDownloadCancelContext(req) - extensionRequestCancels.mu.Lock() - refs := extensionRequestCancels.entries[requestID].refs - extensionRequestCancels.mu.Unlock() - if refs != 1 { - t.Fatalf("binding a request leaked a cancellation reference: %d", refs) - } - - cancelExtensionRequest(requestID) - select { - case <-req.Context().Done(): - case <-time.After(time.Second): - t.Fatal("expected request context to be cancelled") - } -} - -func TestExtensionRuntime_SSRFProtection(t *testing.T) { - ext := &loadedExtension{ - ID: "test-ext", - Manifest: &ExtensionManifest{ - Name: "test-ext", - Permissions: ExtensionPermissions{ - Network: []string{"api.example.com"}, - }, - }, - DataDir: t.TempDir(), - } - - runtime := newExtensionRuntime(ext) - - privateIPs := []string{ - "http://localhost/admin", - "http://127.0.0.1/admin", - "http://192.168.1.1/admin", - "http://10.0.0.1/admin", - "http://172.16.0.1/admin", - "http://169.254.169.254/latest/meta-data/", // AWS metadata - "http://router.local/admin", - } - - for _, url := range privateIPs { - err := runtime.validateDomain(url) - if err == nil { - t.Errorf("Expected private IP/host '%s' to be blocked", url) - } - } - - if err := runtime.validateDomain("https://api.example.com/path"); err != nil { - t.Errorf("Expected api.example.com to be allowed, got error: %v", err) - } -} - -func TestExtensionRuntimeAPIsRequireDeclaredPermissions(t *testing.T) { - withoutPermissions := &loadedExtension{ - ID: "no-permissions", - Manifest: &ExtensionManifest{Name: "no-permissions"}, - DataDir: t.TempDir(), - } - vm := goja.New() - newExtensionRuntime(withoutPermissions).RegisterAPIs(vm) - for _, api := range []string{"storage", "credentials", "auth", "session", "file", "ffmpeg"} { - if value := vm.Get(api); value != nil && !goja.IsUndefined(value) { - t.Fatalf("%s API was exposed without permission", api) - } - } - - withPermissions := &loadedExtension{ - ID: "with-permissions", - Manifest: &ExtensionManifest{ - Name: "with-permissions", - Permissions: ExtensionPermissions{Storage: true, File: true}, - Capabilities: map[string]any{"rawFfmpeg": true}, - }, - DataDir: t.TempDir(), - } - vm = goja.New() - newExtensionRuntime(withPermissions).RegisterAPIs(vm) - for _, api := range []string{"storage", "credentials", "auth", "file", "ffmpeg"} { - if value := vm.Get(api); value == nil || goja.IsUndefined(value) { - t.Fatalf("%s API was not exposed with permission", api) - } - } -} - -func TestValidatePostProcessResultRestrictsReplacementTargets(t *testing.T) { - workDir := t.TempDir() - input := PostProcessInput{Path: filepath.Join(workDir, "input.flac"), URI: "content://input"} - ext := &loadedExtension{ - ID: "post-process", - Manifest: &ExtensionManifest{Name: "post-process"}, - DataDir: t.TempDir(), - } - if err := validatePostProcessResult(ext, input, &PostProcessResult{NewFilePath: input.Path, NewFileURI: input.URI}); err != nil { - t.Fatalf("unchanged target should be accepted: %v", err) - } - if err := validatePostProcessResult(ext, input, &PostProcessResult{NewFilePath: filepath.Join(workDir, "output.flac")}); err == nil { - t.Fatal("replacement path should require file permission") - } - ext.Manifest.Permissions.File = true - if err := validatePostProcessResult(ext, input, &PostProcessResult{NewFilePath: filepath.Join(workDir, "output.flac")}); err != nil { - t.Fatalf("sibling replacement should be accepted with file permission: %v", err) - } - if err := validatePostProcessResult(ext, input, &PostProcessResult{NewFileURI: "content://other"}); err == nil { - t.Fatal("replacement URI should be rejected") - } -} - -func TestIsPrivateIP(t *testing.T) { - tests := []struct { - host string - expected bool - }{ - {"localhost", true}, - {"127.0.0.1", true}, - {"127.0.0.2", true}, - {"10.0.0.1", true}, - {"10.255.255.255", true}, - {"172.16.0.1", true}, - {"172.31.255.255", true}, - {"192.168.0.1", true}, - {"192.168.255.255", true}, - {"169.254.169.254", true}, - {"router.local", true}, - {"mydevice.local", true}, - - {"8.8.8.8", false}, - {"1.1.1.1", false}, - {"api.example.com", false}, - {"google.com", false}, - {"172.15.0.1", false}, - {"172.32.0.1", false}, - {"192.167.0.1", false}, - } - - for _, tt := range tests { - result := isPrivateIP(tt.host) - if result != tt.expected { - t.Errorf("isPrivateIP(%s) = %v, expected %v", tt.host, result, tt.expected) - } - } -} diff --git a/go_backend/extension_timeout.go b/go_backend/extension_timeout.go deleted file mode 100644 index 8ec339f6..00000000 --- a/go_backend/extension_timeout.go +++ /dev/null @@ -1,197 +0,0 @@ -package gobackend - -import ( - "context" - "errors" - "fmt" - "runtime/debug" - "sync" - "time" - - "github.com/dop251/goja" -) - -type JSExecutionError struct { - Message string - IsTimeout bool - RuntimeUnsafe bool - Cause error - runtimeDone <-chan struct{} -} - -func (e *JSExecutionError) Error() string { - return e.Message -} - -func (e *JSExecutionError) Unwrap() error { - return e.Cause -} - -var jsInterruptGracePeriod = 5 * time.Second - -func RunWithTimeoutContext(ctx context.Context, vm *goja.Runtime, script string, timeout time.Duration) (goja.Value, error) { - return runGojaCallWithTimeoutContext(ctx, vm, func() (goja.Value, error) { - return vm.RunString(script) - }, timeout) -} - -func runGojaCallWithTimeoutContext(ctx context.Context, vm *goja.Runtime, call func() (goja.Value, error), timeout time.Duration) (goja.Value, error) { - if vm == nil { - return nil, fmt.Errorf("extension runtime unavailable") - } - if call == nil { - return nil, fmt.Errorf("extension call unavailable") - } - - if timeout <= 0 { - timeout = DefaultJSTimeout - } - - if ctx == nil { - ctx = context.Background() - } - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - type result struct { - value goja.Value - err error - } - resultCh := make(chan result, 1) - executionDone := make(chan struct{}) - - var interrupted bool - var interruptMu sync.Mutex - - go func() { - defer close(executionDone) - defer func() { - if r := recover(); r != nil { - interruptMu.Lock() - wasInterrupted := interrupted - interruptMu.Unlock() - - if wasInterrupted { - resultCh <- result{nil, &JSExecutionError{ - Message: "execution timeout exceeded", - IsTimeout: true, - }} - } else { - GoLog("[extensionRuntime] panic during JS execution: %v\n%s\n", r, string(debug.Stack())) - resultCh <- result{nil, fmt.Errorf("panic during execution: %v", r)} - } - } - }() - - val, err := call() - resultCh <- result{val, err} - }() - - select { - case res := <-resultCh: - // A host call may return as cancellation fires. Do not let a swallowed - // native timeout become a script error (or even success). - if ctx.Err() != nil { - if errors.Is(context.Cause(ctx), context.Canceled) { - return nil, ErrExtensionRequestCancelled - } - return nil, &JSExecutionError{Message: "execution timeout exceeded", IsTimeout: true} - } - return res.value, res.err - case <-ctx.Done(): - cancelled := errors.Is(context.Cause(ctx), context.Canceled) - interruptMu.Lock() - interrupted = true - interruptMu.Unlock() - - if cancelled { - vm.Interrupt("extension request cancelled") - } else { - vm.Interrupt("execution timeout") - } - - // MUST wait for the goroutine to finish before returning. - // The Goja VM is NOT thread-safe — if we return while the goroutine - // is still executing JS (e.g. blocked on an HTTP call), the next - // caller will access the VM concurrently and crash with a nil - // pointer dereference. - select { - case <-resultCh: - if cancelled { - return nil, ErrExtensionRequestCancelled - } - return nil, &JSExecutionError{ - Message: "execution timeout exceeded", - IsTimeout: true, - } - case <-time.After(jsInterruptGracePeriod): - // Goroutine is truly stuck (e.g. HTTP read with no timeout). - // Log a warning — the VM should NOT be reused after this. - GoLog("[extensionRuntime] WARNING: JS goroutine did not exit within %s after interrupt, VM may be unsafe\n", jsInterruptGracePeriod) - message := "execution timeout exceeded (runtime quarantined)" - var cause error - if cancelled { - message = "extension request cancelled (runtime quarantined)" - cause = ErrExtensionRequestCancelled - } - return nil, &JSExecutionError{ - Message: message, - IsTimeout: !cancelled, - RuntimeUnsafe: true, - Cause: cause, - runtimeDone: executionDone, - } - } - } -} - -// RunWithTimeoutAndRecover runs JS with timeout and clears interrupt state after -// This should be used when you want to continue using the VM after a timeout -func RunWithTimeoutAndRecover(vm *goja.Runtime, script string, timeout time.Duration) (goja.Value, error) { - return RunWithTimeoutContextAndRecover(context.Background(), vm, script, timeout) -} - -func RunWithTimeoutContextAndRecover(ctx context.Context, vm *goja.Runtime, script string, timeout time.Duration) (goja.Value, error) { - result, err := RunWithTimeoutContext(ctx, vm, script, timeout) - - if vm != nil && !IsRuntimeUnsafeError(err) { - vm.ClearInterrupt() - } - - return result, err -} - -func runGojaCallWithTimeoutAndRecover(vm *goja.Runtime, call func() (goja.Value, error), timeout time.Duration) (goja.Value, error) { - return runGojaCallWithTimeoutContextAndRecover(context.Background(), vm, call, timeout) -} - -func runGojaCallWithTimeoutContextAndRecover(ctx context.Context, vm *goja.Runtime, call func() (goja.Value, error), timeout time.Duration) (goja.Value, error) { - result, err := runGojaCallWithTimeoutContext(ctx, vm, call, timeout) - - if vm != nil && !IsRuntimeUnsafeError(err) { - vm.ClearInterrupt() - } - - return result, err -} - -func IsRuntimeUnsafeError(err error) bool { - var jsErr *JSExecutionError - return errors.As(err, &jsErr) && jsErr.RuntimeUnsafe -} - -func runtimeCompletion(err error) <-chan struct{} { - var jsErr *JSExecutionError - if errors.As(err, &jsErr) && jsErr.RuntimeUnsafe { - return jsErr.runtimeDone - } - return nil -} - -func IsTimeoutError(err error) bool { - var jsErr *JSExecutionError - if errors.As(err, &jsErr) { - return jsErr.IsTimeout - } - return false -} diff --git a/go_backend/extension_transfer_policy.go b/go_backend/extension_transfer_policy.go deleted file mode 100644 index f6276194..00000000 --- a/go_backend/extension_transfer_policy.go +++ /dev/null @@ -1,215 +0,0 @@ -package gobackend - -import ( - "fmt" - "math" - "strings" - "time" -) - -const ( - defaultTransferMaxAttempts = 3 - defaultTransferInitialRetryDelay = 500 * time.Millisecond - defaultTransferMaxRetryDelay = 8 * time.Second - defaultParallelSegments = 3 - maxParallelSegments = 8 - maxExtensionDownloadConcurrency = 3 -) - -// DownloadTransferPolicy is the generic manifest contract used by every -// extension-backed transfer. It deliberately describes transport behavior, -// never a provider name, so new extensions can opt into the same reliability -// and concurrency features without changes in the app. -// -// Manifests declare it under capabilities.downloadTransfer: -// -// { -// "maxAttempts": 4, -// "initialRetryDelayMs": 500, -// "maxRetryDelayMs": 8000, -// "resumePolicy": "validated", -// "persistentCheckpoint": true, -// "refreshStreamOnStatus": [401, 403], -// "maxParallelSegments": 4, -// "maxConcurrentDownloads": 2 -// } -type DownloadTransferPolicy struct { - MaxAttempts int - InitialRetryDelay time.Duration - MaxRetryDelay time.Duration - ResumePolicy string - PersistentCheckpoint bool - RefreshStreamOnStatus map[int]bool - MaxParallelSegments int - MaxConcurrentDownloads int -} - -func defaultDownloadTransferPolicy() DownloadTransferPolicy { - return DownloadTransferPolicy{ - MaxAttempts: defaultTransferMaxAttempts, - InitialRetryDelay: defaultTransferInitialRetryDelay, - MaxRetryDelay: defaultTransferMaxRetryDelay, - ResumePolicy: "none", - PersistentCheckpoint: false, - RefreshStreamOnStatus: map[int]bool{httpStatusUnauthorized: true, httpStatusForbidden: true}, - MaxParallelSegments: defaultParallelSegments, - MaxConcurrentDownloads: maxExtensionDownloadConcurrency, - } -} - -const ( - httpStatusUnauthorized = 401 - httpStatusForbidden = 403 -) - -func capabilityObject(capabilities map[string]any, key string) map[string]any { - if capabilities == nil { - return nil - } - value, ok := capabilities[key] - if !ok { - return nil - } - switch typed := value.(type) { - case map[string]any: - return typed - default: - return nil - } -} - -func capabilityInt(value any, fallback int) int { - switch typed := value.(type) { - case int: - return typed - case int32: - return int(typed) - case int64: - return int(typed) - case float32: - return int(math.Round(float64(typed))) - case float64: - return int(math.Round(typed)) - default: - return fallback - } -} - -func clampInt(value, minimum, maximum int) int { - if value < minimum { - return minimum - } - if value > maximum { - return maximum - } - return value -} - -func parseRefreshStatuses(value any, fallback map[int]bool) map[int]bool { - values, ok := value.([]any) - if !ok { - return fallback - } - parsed := make(map[int]bool) - for _, raw := range values { - status := capabilityInt(raw, 0) - if status >= 400 && status <= 599 { - parsed[status] = true - } - } - if len(parsed) == 0 { - return fallback - } - return parsed -} - -func (m *ExtensionManifest) DownloadTransferPolicy() DownloadTransferPolicy { - policy := defaultDownloadTransferPolicy() - if m == nil { - return policy - } - config := capabilityObject(m.Capabilities, "downloadTransfer") - if config == nil { - return policy - } - - policy.MaxAttempts = clampInt( - capabilityInt(config["maxAttempts"], policy.MaxAttempts), - 1, - 8, - ) - initialDelayMs := clampInt( - capabilityInt(config["initialRetryDelayMs"], int(policy.InitialRetryDelay/time.Millisecond)), - 100, - 30_000, - ) - maxDelayMs := clampInt( - capabilityInt(config["maxRetryDelayMs"], int(policy.MaxRetryDelay/time.Millisecond)), - initialDelayMs, - 120_000, - ) - policy.InitialRetryDelay = time.Duration(initialDelayMs) * time.Millisecond - policy.MaxRetryDelay = time.Duration(maxDelayMs) * time.Millisecond - - if value, ok := config["resumePolicy"].(string); ok { - switch strings.ToLower(strings.TrimSpace(value)) { - case "validated", "none": - policy.ResumePolicy = strings.ToLower(strings.TrimSpace(value)) - } - } - if value, ok := config["persistentCheckpoint"].(bool); ok { - policy.PersistentCheckpoint = value && policy.ResumePolicy == "validated" - } - policy.RefreshStreamOnStatus = parseRefreshStatuses( - config["refreshStreamOnStatus"], - policy.RefreshStreamOnStatus, - ) - policy.MaxParallelSegments = clampInt( - capabilityInt(config["maxParallelSegments"], policy.MaxParallelSegments), - 1, - maxParallelSegments, - ) - policy.MaxConcurrentDownloads = clampInt( - capabilityInt(config["maxConcurrentDownloads"], policy.MaxConcurrentDownloads), - 1, - maxExtensionDownloadConcurrency, - ) - return policy -} - -func validateDownloadTransferCapability(capabilities map[string]any) error { - if capabilities == nil { - return nil - } - _, exists := capabilities["downloadTransfer"] - if !exists { - return nil - } - config := capabilityObject(capabilities, "downloadTransfer") - if config == nil { - return fmt.Errorf("must be an object") - } - if rawResume, ok := config["resumePolicy"]; ok { - resume, ok := rawResume.(string) - if !ok || (resume != "none" && resume != "validated") { - return fmt.Errorf("resumePolicy must be 'none' or 'validated'") - } - } - if rawCheckpoint, ok := config["persistentCheckpoint"]; ok { - if _, ok := rawCheckpoint.(bool); !ok { - return fmt.Errorf("persistentCheckpoint must be a boolean") - } - } - for _, key := range []string{ - "maxAttempts", - "initialRetryDelayMs", - "maxRetryDelayMs", - "maxParallelSegments", - "maxConcurrentDownloads", - } { - if rawValue, ok := config[key]; ok && capabilityInt(rawValue, -1) < 0 { - return fmt.Errorf("%s must be a non-negative number", key) - } - } - return nil -} diff --git a/go_backend/extension_transfer_policy_test.go b/go_backend/extension_transfer_policy_test.go deleted file mode 100644 index 8e645722..00000000 --- a/go_backend/extension_transfer_policy_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package gobackend - -import ( - "testing" - "time" -) - -func TestDownloadTransferPolicyParsesAndBoundsManifestCapability(t *testing.T) { - manifest := &ExtensionManifest{Capabilities: map[string]any{ - "downloadTransfer": map[string]any{ - "maxAttempts": float64(20), - "initialRetryDelayMs": float64(25), - "maxRetryDelayMs": float64(50), - "resumePolicy": "validated", - "persistentCheckpoint": true, - "refreshStreamOnStatus": []any{float64(401), float64(410)}, - "maxParallelSegments": float64(99), - "maxConcurrentDownloads": float64(99), - }, - }} - - policy := manifest.DownloadTransferPolicy() - if policy.MaxAttempts != 8 || policy.InitialRetryDelay != 100*time.Millisecond || - policy.MaxRetryDelay != 100*time.Millisecond || policy.ResumePolicy != "validated" || - !policy.PersistentCheckpoint || !policy.RefreshStreamOnStatus[401] || - !policy.RefreshStreamOnStatus[410] || policy.MaxParallelSegments != 8 || - policy.MaxConcurrentDownloads != 3 { - t.Fatalf("unexpected policy: %#v", policy) - } -} - -func TestDownloadTransferCapabilityValidation(t *testing.T) { - valid := map[string]any{"downloadTransfer": map[string]any{ - "resumePolicy": "validated", - "persistentCheckpoint": true, - }} - if err := validateDownloadTransferCapability(valid); err != nil { - t.Fatalf("valid capability rejected: %v", err) - } - - invalid := []map[string]any{ - {"downloadTransfer": "yes"}, - {"downloadTransfer": map[string]any{"resumePolicy": "unsafe"}}, - {"downloadTransfer": map[string]any{"persistentCheckpoint": "yes"}}, - } - for _, capabilities := range invalid { - if err := validateDownloadTransferCapability(capabilities); err == nil { - t.Fatalf("invalid capability accepted: %#v", capabilities) - } - } -} diff --git a/go_backend/filename.go b/go_backend/filename.go deleted file mode 100644 index 9183f3ad..00000000 --- a/go_backend/filename.go +++ /dev/null @@ -1,418 +0,0 @@ -package gobackend - -import ( - "fmt" - "regexp" - "strconv" - "strings" - "time" - "unicode" - "unicode/utf8" -) - -var ( - invalidChars = regexp.MustCompile(`[<>:"/\\|?*\x00-\x1f]`) - multiUnderscore = regexp.MustCompile(`_+`) - formattedNumberPlaceholderExpr = regexp.MustCompile(`\{(track|disc|playlist_position|playlistPosition|position):([0-9]+)\}`) - dateFormatPlaceholderExpr = regexp.MustCompile(`\{date:([^{}]+)\}`) - yearPattern = regexp.MustCompile(`\d{4}`) - emptyFilenameGroupExpr = regexp.MustCompile(`\[\s*\]|\(\s*\)`) - danglingGroupSeparatorExpr = regexp.MustCompile(`\s*[-_|]\s*([\]\)])`) - repeatedFilenameSeparatorExpr = regexp.MustCompile(`\s*[-–—_|]\s*(?:[-–—_|]\s*)+`) -) - -const maxSanitizedFilenameBytes = 200 - -func sanitizeFilename(filename string) string { - sanitized := strings.ReplaceAll(filename, "/", " ") - sanitized = invalidChars.ReplaceAllString(sanitized, " ") - - var builder strings.Builder - for _, r := range sanitized { - if r < 0x20 && r != 0x09 && r != 0x0A && r != 0x0D { - continue - } - if r == 0x7F { - continue - } - if unicode.IsControl(r) && r != 0x09 && r != 0x0A && r != 0x0D { - continue - } - builder.WriteRune(r) - } - - sanitized = builder.String() - sanitized = strings.TrimSpace(sanitized) - sanitized = strings.Trim(sanitized, ". ") - sanitized = strings.Join(strings.Fields(sanitized), " ") - sanitized = multiUnderscore.ReplaceAllString(sanitized, "_") - sanitized = strings.Trim(sanitized, "_ ") - - if !utf8.ValidString(sanitized) { - sanitized = strings.ToValidUTF8(sanitized, "_") - } - - if len(sanitized) > maxSanitizedFilenameBytes { - sanitized = truncateUTF8Bytes(sanitized, maxSanitizedFilenameBytes) - sanitized = strings.TrimSpace(strings.Trim(sanitized, ". ")) - sanitized = strings.Trim(sanitized, "_ ") - } - - if sanitized == "" { - return "Unknown" - } - - return sanitized -} - -func sanitizeFilenamePreservingToken(filename string, token string) string { - sanitized := sanitizeFilename(filename) - token = strings.TrimSpace(token) - if token == "" || !strings.Contains(filename, token) || strings.Contains(sanitized, token) { - return sanitized - } - - safeToken := sanitizeFilename(token) - suffix := " - " + safeToken - prefixLimit := maxSanitizedFilenameBytes - len(suffix) - if prefixLimit <= 0 { - return truncateUTF8Bytes(safeToken, maxSanitizedFilenameBytes) - } - rawPrefix := strings.Trim(strings.ReplaceAll(filename, token, ""), " _-") - prefix := sanitizeFilename(rawPrefix) - prefix = truncateUTF8Bytes(prefix, prefixLimit) - prefix = strings.TrimSpace(strings.Trim(prefix, ". _-")) - if prefix == "" || prefix == "Unknown" { - return safeToken - } - return prefix + suffix -} - -func truncateUTF8Bytes(value string, maxBytes int) string { - if maxBytes <= 0 || len(value) <= maxBytes { - return value - } - - used := 0 - for i, r := range value { - runeLen := utf8.RuneLen(r) - if runeLen < 0 { - runeLen = len(string(r)) - } - if used+runeLen > maxBytes { - return value[:i] - } - used += runeLen - } - return value -} - -func buildFilenameFromTemplate(template string, metadata map[string]any) string { - if template == "" { - template = "{artist} - {title}" - } - - result := replaceFormattedNumberPlaceholders(template, metadata) - result = replaceDateFormatPlaceholders(result, metadata) - - dateValue := getDateValue(metadata) - yearValue := getString(metadata, "year") - if yearValue == "" { - yearValue = extractYear(dateValue) - } - - placeholders := map[string]string{ - "{title}": getString(metadata, "title"), - "{artist}": getString(metadata, "artist"), - "{album}": getString(metadata, "album"), - "{track}": formatTrackNumber(getInt(metadata, "track")), - "{track_raw}": formatRawNumber(getInt(metadata, "track")), - "{playlist_position}": formatTrackNumber(getPlaylistPosition(metadata)), - "{playlist position}": formatTrackNumber(getPlaylistPosition(metadata)), - "{playlistPosition}": formatTrackNumber(getPlaylistPosition(metadata)), - "{position}": formatTrackNumber(getPlaylistPosition(metadata)), - "{playlist_position_raw}": formatRawNumber(getPlaylistPosition(metadata)), - "{year}": yearValue, - "{date}": dateValue, - "{disc}": formatDiscNumber(getInt(metadata, "disc")), - "{disc_raw}": formatRawNumber(getInt(metadata, "disc")), - "{quality}": getString(metadata, "quality"), - "{quality_variant}": getString(metadata, "quality_variant"), - "{isrc}": getString(metadata, "isrc"), - "{provider}": getString(metadata, "provider"), - "{platform}": getString(metadata, "provider"), - "{provider_id}": getString(metadata, "provider_id"), - "{id}": getString(metadata, "provider_id"), - } - - hasEmptyOptionalPlaceholder := false - for placeholder, value := range placeholders { - if value == "" && isOptionalFilenamePlaceholder(placeholder) && strings.Contains(result, placeholder) { - hasEmptyOptionalPlaceholder = true - } - result = strings.ReplaceAll(result, placeholder, value) - } - if hasEmptyOptionalPlaceholder { - result = cleanupEmptyFilenameDecorations(result) - } - - return result -} - -func isOptionalFilenamePlaceholder(placeholder string) bool { - switch placeholder { - case "{isrc}", "{provider}", "{platform}", "{provider_id}", "{id}": - return true - default: - return false - } -} - -func cleanupEmptyFilenameDecorations(value string) string { - cleaned := emptyFilenameGroupExpr.ReplaceAllString(value, "") - cleaned = danglingGroupSeparatorExpr.ReplaceAllString(cleaned, "$1") - cleaned = repeatedFilenameSeparatorExpr.ReplaceAllString(cleaned, " - ") - cleaned = strings.Join(strings.Fields(cleaned), " ") - return strings.Trim(cleaned, " -–—_|") -} - -func replaceFormattedNumberPlaceholders(template string, metadata map[string]any) string { - return formattedNumberPlaceholderExpr.ReplaceAllStringFunc(template, func(match string) string { - parts := formattedNumberPlaceholderExpr.FindStringSubmatch(match) - if len(parts) != 3 { - return "" - } - - number := getInt(metadata, parts[1]) - if parts[1] == "playlist_position" || parts[1] == "playlistPosition" || parts[1] == "position" { - number = getPlaylistPosition(metadata) - } - width, err := strconv.Atoi(parts[2]) - if err != nil { - return "" - } - - return formatNumberWithWidth(number, width) - }) -} - -func replaceDateFormatPlaceholders(template string, metadata map[string]any) string { - return dateFormatPlaceholderExpr.ReplaceAllStringFunc(template, func(match string) string { - parts := dateFormatPlaceholderExpr.FindStringSubmatch(match) - if len(parts) != 2 { - return "" - } - - return formatDateWithPattern(getDateValue(metadata), parts[1]) - }) -} - -func getDateValue(metadata map[string]any) string { - date := getString(metadata, "date") - if date != "" { - return date - } - - releaseDate := getString(metadata, "release_date") - if releaseDate != "" { - return releaseDate - } - - return getString(metadata, "year") -} - -func getString(m map[string]any, key string) string { - if v, ok := m[key]; ok { - switch value := v.(type) { - case string: - return strings.TrimSpace(value) - case int: - return strconv.Itoa(value) - case int64: - return strconv.FormatInt(value, 10) - case float64: - return strconv.Itoa(int(value)) - } - } - return "" -} - -func getInt(m map[string]any, key string) int { - candidateKeys := []string{key} - switch key { - case "track": - candidateKeys = append(candidateKeys, "track_number") - case "disc": - candidateKeys = append(candidateKeys, "disc_number") - case "playlist_position", "playlistPosition", "playlist position", "position": - candidateKeys = append(candidateKeys, "playlistPosition", "playlist position", "position") - } - - for _, candidate := range candidateKeys { - if v, ok := m[candidate]; ok { - switch n := v.(type) { - case int: - return n - case int64: - return int(n) - case float64: - return int(n) - case string: - parsed, err := strconv.Atoi(strings.TrimSpace(n)) - if err == nil { - return parsed - } - } - } - } - - return 0 -} - -func getPlaylistPosition(metadata map[string]any) int { - return getInt(metadata, "playlist_position") -} - -func formatTrackNumber(n int) string { - if n <= 0 { - return "" - } - return fmt.Sprintf("%02d", n) -} - -func formatDiscNumber(n int) string { - if n <= 0 { - return "" - } - return fmt.Sprintf("%d", n) -} - -func formatRawNumber(n int) string { - if n <= 0 { - return "" - } - return fmt.Sprintf("%d", n) -} - -func formatNumberWithWidth(n int, width int) string { - if n <= 0 || width <= 0 { - return "" - } - if width <= 1 { - return formatRawNumber(n) - } - return fmt.Sprintf("%0*d", width, n) -} - -func formatDateWithPattern(rawDate string, strftimePattern string) string { - if rawDate == "" || strftimePattern == "" { - return "" - } - - parsedDate, ok := parseMetadataDate(rawDate) - if !ok { - return "" - } - - goLayout := convertStrftimeToGoLayout(strftimePattern) - if goLayout == "" { - return "" - } - - return parsedDate.Format(goLayout) -} - -func parseMetadataDate(rawDate string) (time.Time, bool) { - clean := strings.TrimSpace(rawDate) - if clean == "" { - return time.Time{}, false - } - - layouts := []string{ - time.RFC3339Nano, - time.RFC3339, - "2006-01-02", - "2006-01", - "2006", - "2006/01/02", - "2006/01", - "2006.01.02", - "2006.01", - } - - for _, layout := range layouts { - parsed, err := time.Parse(layout, clean) - if err == nil { - return parsed, true - } - } - - if len(clean) >= 10 { - parsed, err := time.Parse("2006-01-02", clean[:10]) - if err == nil { - return parsed, true - } - } - - yearMatch := yearPattern.FindString(clean) - if yearMatch == "" { - return time.Time{}, false - } - - year, err := strconv.Atoi(yearMatch) - if err != nil || year <= 0 { - return time.Time{}, false - } - - return time.Date(year, time.January, 1, 0, 0, 0, 0, time.UTC), true -} - -func convertStrftimeToGoLayout(pattern string) string { - if pattern == "" { - return "" - } - - var builder strings.Builder - for i := 0; i < len(pattern); i++ { - ch := pattern[i] - if ch != '%' { - builder.WriteByte(ch) - continue - } - - if i+1 >= len(pattern) { - builder.WriteByte('%') - break - } - - i++ - switch pattern[i] { - case 'Y': - builder.WriteString("2006") - case 'y': - builder.WriteString("06") - case 'm': - builder.WriteString("01") - case 'd': - builder.WriteString("02") - case 'b': - builder.WriteString("Jan") - case 'B': - builder.WriteString("January") - case '%': - builder.WriteByte('%') - default: - builder.WriteByte('%') - builder.WriteByte(pattern[i]) - } - } - - return builder.String() -} - -func extractYear(date string) string { - if len(date) >= 4 { - return date[:4] - } - return date -} diff --git a/go_backend/filename_test.go b/go_backend/filename_test.go deleted file mode 100644 index 51ffaa68..00000000 --- a/go_backend/filename_test.go +++ /dev/null @@ -1,238 +0,0 @@ -package gobackend - -import ( - "strings" - "testing" - "unicode/utf8" -) - -func TestBuildFilenameFromTemplate_WithRawTrackAndDisc(t *testing.T) { - metadata := map[string]any{ - "title": "Song Name", - "artist": "Artist Name", - "album": "Album Name", - "track": 1, - "disc": 2, - "year": "2025", - } - - formatted := buildFilenameFromTemplate( - "{artist} - {track} - {track_raw} - d{disc} - d{disc_raw} - {title}", - metadata, - ) - - expected := "Artist Name - 01 - 1 - d2 - d2 - Song Name" - if formatted != expected { - t.Fatalf("expected %q, got %q", expected, formatted) - } -} - -func TestBuildFilenameFromTemplate_RawPlaceholdersEmptyWhenZero(t *testing.T) { - metadata := map[string]any{ - "title": "Song Name", - "artist": "Artist Name", - "track": 0, - "disc": 0, - } - - formatted := buildFilenameFromTemplate("{track_raw}-{disc_raw}-{title}", metadata) - expected := "--Song Name" - if formatted != expected { - t.Fatalf("expected %q, got %q", expected, formatted) - } -} - -func TestBuildFilenameFromTemplate_InlineNumberFormatting(t *testing.T) { - metadata := map[string]any{ - "track": 3, - "disc": 2, - } - - formatted := buildFilenameFromTemplate("{track:1}-{track:02}-{disc:03}", metadata) - expected := "3-03-002" - if formatted != expected { - t.Fatalf("expected %q, got %q", expected, formatted) - } -} - -func TestBuildFilenameFromTemplate_PlaylistPositionFormatting(t *testing.T) { - metadata := map[string]any{ - "playlist_position": 4, - "artist": "Artist Name", - "title": "Song Name", - } - - formatted := buildFilenameFromTemplate( - "{playlist_position:02} - {artist} - {title}", - metadata, - ) - expected := "04 - Artist Name - Song Name" - if formatted != expected { - t.Fatalf("expected %q, got %q", expected, formatted) - } -} - -func TestBuildFilenameFromTemplate_QualityVariant(t *testing.T) { - metadata := map[string]any{ - "artist": "Artist Name", - "title": "Song Name", - "quality": "HI_RES_LOSSLESS", - } - - formatted := buildFilenameFromTemplate( - "{artist} - {title} - {quality}", - metadata, - ) - if formatted != "Artist Name - Song Name - HI_RES_LOSSLESS" { - t.Fatalf("unexpected quality filename: %q", formatted) - } -} - -func TestBuildFilenameFromTemplate_QualityVariantStagingToken(t *testing.T) { - metadata := map[string]any{ - "artist": "Artist Name", - "title": "Song Name", - "quality_variant": "qv_12345678", - } - - formatted := buildFilenameFromTemplate( - "{artist} - {title} - {quality_variant}", - metadata, - ) - if formatted != "Artist Name - Song Name - qv_12345678" { - t.Fatalf("unexpected quality variant filename: %q", formatted) - } -} - -func TestBuildDownloadFilename_ProvidesRequestedQuality(t *testing.T) { - filename := buildDownloadFilename(DownloadRequest{ - TrackName: "Song Name", - ArtistName: "Artist Name", - FilenameFormat: "{artist} - {title} - {quality}", - Quality: "LOSSLESS", - OutputExt: ".flac", - }) - - if filename != "Artist Name - Song Name - LOSSLESS.flac" { - t.Fatalf("unexpected download filename: %q", filename) - } -} - -func TestBuildDownloadFilename_ProvidesTraceabilityPlaceholders(t *testing.T) { - filename := buildDownloadFilename(DownloadRequest{ - TrackName: "Song Name", - ArtistName: "Artist Name", - ISRC: "USABC1234567", - DownloadProvider: "download-provider", - ProviderTrackID: "123456789", - FilenameFormat: "{artist} - {title} [{isrc}] [{provider}-{provider_id}]", - OutputExt: ".flac", - }) - - expected := "Artist Name - Song Name [USABC1234567] [download-provider-123456789].flac" - if filename != expected { - t.Fatalf("expected %q, got %q", expected, filename) - } -} - -func TestBuildFilenameFromTemplate_TraceabilityAliases(t *testing.T) { - metadata := map[string]any{ - "provider": "download-provider", - "provider_id": "998877", - } - - formatted := buildFilenameFromTemplate("{platform}-{id}", metadata) - if formatted != "download-provider-998877" { - t.Fatalf("unexpected alias filename: %q", formatted) - } -} - -func TestBuildFilenameFromTemplate_CleansEmptyTraceabilityDecorations(t *testing.T) { - metadata := map[string]any{ - "title": "Song Name", - "provider": "download-provider", - } - - formatted := buildFilenameFromTemplate( - "{title} [{isrc}] [{provider}-{provider_id}]", - metadata, - ) - if formatted != "Song Name [download-provider]" { - t.Fatalf("unexpected empty placeholder cleanup: %q", formatted) - } -} - -func TestBuildDownloadFilename_PreservesVariantTokenWhenTruncated(t *testing.T) { - filename := buildDownloadFilename(DownloadRequest{ - TrackName: strings.Repeat("Very Long Song ", 30), - ArtistName: "Artist Name", - FilenameFormat: "{artist} - {title} - {quality_variant}", - QualityVariant: "qv_12345678", - OutputExt: ".flac", - }) - - if !strings.Contains(filename, "qv_12345678") { - t.Fatalf("quality variant token was truncated: %q", filename) - } - if len(strings.TrimSuffix(filename, ".flac")) > maxSanitizedFilenameBytes { - t.Fatalf("filename base exceeds limit: %d bytes", len(filename)) - } -} - -func TestBuildFilenameFromTemplate_DateStrftimeFormatting(t *testing.T) { - metadata := map[string]any{ - "artist": "Artist Name", - "title": "Song Name", - "release_date": "2024-03-09", - "track_number": 7, - "disc_number": 1, - } - - formatted := buildFilenameFromTemplate( - "{artist} - {track:02} - {title} - {date:%Y-%m-%d} - {year}", - metadata, - ) - expected := "Artist Name - 07 - Song Name - 2024-03-09 - 2024" - if formatted != expected { - t.Fatalf("expected %q, got %q", expected, formatted) - } -} - -func TestBuildFilenameFromTemplate_DateStrftimeFormattingWithYearOnly(t *testing.T) { - metadata := map[string]any{ - "artist": "Artist Name", - "title": "Song Name", - "date": "2019", - } - - formatted := buildFilenameFromTemplate("{date:%Y}-{date:%m}-{date:%d}", metadata) - expected := "2019-01-01" - if formatted != expected { - t.Fatalf("expected %q, got %q", expected, formatted) - } -} - -func TestSanitizeFilenameMatchesDesktopSpacingBehavior(t *testing.T) { - got := sanitizeFilename(` "Text In Quotes"?%* / Demo `) - want := "Text In Quotes % Demo" - if got != want { - t.Fatalf("expected %q, got %q", want, got) - } -} - -func TestSanitizeFilenameFallsBackToUnknownWhenEmpty(t *testing.T) { - got := sanitizeFilename(`<>:"/\|?*`) - if got != "Unknown" { - t.Fatalf("expected %q, got %q", "Unknown", got) - } -} - -func TestSanitizeFilenameTruncatesWithoutSplittingUTF8(t *testing.T) { - got := sanitizeFilename(strings.Repeat("あ", 80)) - if !utf8.ValidString(got) { - t.Fatalf("sanitizeFilename returned invalid UTF-8: %q", got) - } - if len(got) > 200 { - t.Fatalf("sanitizeFilename length = %d, want <= 200", len(got)) - } -} diff --git a/go_backend/flac_save.go b/go_backend/flac_save.go deleted file mode 100644 index 73e11f79..00000000 --- a/go_backend/flac_save.go +++ /dev/null @@ -1,69 +0,0 @@ -package gobackend - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - flac "github.com/go-flac/go-flac/v2" -) - -// saveFlacFile persists an edited FLAC crash-safely. go-flac's Save(samePath) -// rewrites the file in place by shifting the audio body within the same inode, -// so a process kill or power loss mid-save destroys the file with no recovery -// copy. Instead, stream the whole edited file to a sibling temp, fsync it, and -// rename it over the target: an interruption at any point leaves either the -// old intact file or the new complete one. -// -// fd-backed targets (/proc/self/fd/N) have no directory entry to rename over, -// so those keep the library's in-place path. -func saveFlacFile(f *flac.File, filePath string) error { - if strings.HasPrefix(filePath, "/proc/self/fd/") { - return f.Save(filePath) - } - - // The ".partial" suffix keeps the temp invisible to library scans and - // extension duplicate checks; the extra ".tag" avoids colliding with the - // download staging sibling of the same final path. - tmpPath := filePath + ".tag.partial" - os.Remove(tmpPath) - tmp, err := os.Create(tmpPath) - if err != nil { - return fmt.Errorf("failed to create temp tag file: %w", err) - } - // WriteTo streams the audio frames from the still-open source handle and - // closes it when done, so the rename below can replace the source even on - // Windows. - if _, err := f.WriteTo(tmp); err != nil { - tmp.Close() - os.Remove(tmpPath) - return err - } - if err := tmp.Sync(); err != nil { - tmp.Close() - os.Remove(tmpPath) - return fmt.Errorf("failed to sync temp tag file: %w", err) - } - if err := tmp.Close(); err != nil { - os.Remove(tmpPath) - return err - } - if err := os.Rename(tmpPath, filePath); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("failed to publish tagged file: %w", err) - } - syncDir(filepath.Dir(filePath)) - return nil -} - -// syncDir best-effort fsyncs a directory so a just-renamed entry survives -// power loss. Unsupported on some platforms/filesystems; errors are ignored. -func syncDir(dir string) { - d, err := os.Open(dir) - if err != nil { - return - } - _ = d.Sync() - _ = d.Close() -} diff --git a/go_backend/go.mod b/go_backend/go.mod deleted file mode 100644 index 9ad7e24e..00000000 --- a/go_backend/go.mod +++ /dev/null @@ -1,31 +0,0 @@ -module github.com/zarz/spotiflac_android/go_backend - -// Needs >= 1.26.3: fixes the cgo "bulkBarrierPreWrite" crash (golang/go#46893) -// without the 1.26.0-1.26.2 arm32 SIGSYS regression (golang/go#78936). -// Full patch version here because gomobile's bind module inherits it. -go 1.26.6 - -require ( - github.com/dop251/goja v0.0.0-20260826204918-8f1c0696a37b - github.com/go-flac/flacpicture/v2 v2.0.2 - github.com/go-flac/flacvorbis/v2 v2.0.2 - github.com/go-flac/go-flac/v2 v2.0.4 - github.com/refraction-networking/utls v1.8.2 - golang.org/x/crypto v0.55.0 - golang.org/x/image v0.45.0 - golang.org/x/mobile v0.0.0-20260821190718-4776eadac327 - golang.org/x/net v0.58.0 - golang.org/x/sync v0.22.0 - golang.org/x/text v0.41.0 -) - -require ( - github.com/andybalholm/brotli v1.2.3 // indirect - github.com/dlclark/regexp2/v2 v2.7.1 // indirect - github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect - github.com/google/pprof v0.0.0-20260825171938-4d453200e7d9 // indirect - github.com/klauspost/compress v1.19.2 // indirect - golang.org/x/mod v0.40.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/tools v0.49.0 // indirect -) diff --git a/go_backend/go.sum b/go_backend/go.sum deleted file mode 100644 index 47e0d42b..00000000 --- a/go_backend/go.sum +++ /dev/null @@ -1,54 +0,0 @@ -github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= -github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/andybalholm/brotli v1.2.3 h1:8H1qwOkl2LPfjf3YezB90JnCliZb6SInJ/OJkEbA5NQ= -github.com/andybalholm/brotli v1.2.3/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4= -github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= -github.com/dop251/goja v0.0.0-20260826204918-8f1c0696a37b h1:mYHoARbZ0mUYXXsaNeHoDFBft3TK4PpFEe3KU7hdDgg= -github.com/dop251/goja v0.0.0-20260826204918-8f1c0696a37b/go.mod h1:u8yZRUavu+N4EnFFy6J5fVtjE7lEcZ2YyV2GcBXY9c8= -github.com/go-flac/flacpicture/v2 v2.0.2 h1:HCaJIVZpxnpdWs6G3ECEVRelzqS5xOi1Ba1AGmtXbzE= -github.com/go-flac/flacpicture/v2 v2.0.2/go.mod h1:DMZBPWPAmdLqNhqFSy5ZBs9wyBzOekXutGfP7/TFCuo= -github.com/go-flac/flacvorbis/v2 v2.0.2 h1:xCL3OhxrxWkHrbWUBvGNe+6FQ03yLmBbz0v5z4V2PoQ= -github.com/go-flac/flacvorbis/v2 v2.0.2/go.mod h1:SwTB5gs13VaM/N7rstwPoUsPibiMKklgwybYP9dYo2g= -github.com/go-flac/go-flac/v2 v2.0.4 h1:atf/kFa8U9idtkA//NO22XGr+MzQLeXZecnmP9sYBf0= -github.com/go-flac/go-flac/v2 v2.0.4/go.mod h1:sYOlTKxutMW0RDYF+KlD6Zn+VOCZlIFQG/r/usPveCs= -github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q= -github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= -github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20260825171938-4d453200e7d9 h1:dl4UZiszMU+NKHirOiCKTC+hRuNAQ0moHPxSg6WcU1o= -github.com/google/pprof v0.0.0-20260825171938-4d453200e7d9/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= -github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= -github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= -golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= -golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= -golang.org/x/mobile v0.0.0-20260821190718-4776eadac327 h1:D/wiQ6AoTYjDtSD0HMPhU8O40NUP8EF0UmDhIYCnG4I= -golang.org/x/mobile v0.0.0-20260821190718-4776eadac327/go.mod h1:D9q8rgXu13Q3uuM+Vuy6F/DG1WF/giTPLtqQ9on5B1M= -golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= -golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go_backend/httputil.go b/go_backend/httputil.go deleted file mode 100644 index b7b11b0f..00000000 --- a/go_backend/httputil.go +++ /dev/null @@ -1,696 +0,0 @@ -package gobackend - -import ( - "bytes" - "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "io" - "math/rand" - "net" - "net/http" - "net/url" - "strconv" - "strings" - "sync" - "syscall" - "time" -) - -func userAgentForURL(u *url.URL) string { - if u == nil { - return getRandomUserAgent() - } - - host := strings.ToLower(strings.TrimSpace(u.Hostname())) - if host == "api.zarz.moe" { - return appUserAgent() - } - - return getRandomUserAgent() -} - -func getRandomUserAgent() string { - chromeVersion := rand.Intn(26) + 120 - chromeBuild := rand.Intn(1500) + 6000 - chromePatch := rand.Intn(200) + 100 - - return fmt.Sprintf( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%d.0.%d.%d Safari/537.36", - chromeVersion, - chromeBuild, - chromePatch, - ) -} - -const ( - DefaultTimeout = 60 * time.Second - DownloadTimeout = 24 * time.Hour - SongLinkTimeout = 30 * time.Second - DefaultMaxRetries = 3 - DefaultRetryDelay = 1 * time.Second - Second = time.Second - // Error responses are diagnostic data, not media payloads. Keeping this - // bounded prevents a hostile intermediary from turning a retry/status path - // into a large allocation while still allowing small bodies to be drained - // for HTTP keep-alive reuse. - maxRetryResponseBodyBytes = int64(64 << 10) -) - -type NetworkCompatibilityOptions struct { - AllowHTTP bool - InsecureTLS bool -} - -var ( - networkCompatibilityMu sync.RWMutex - networkCompatibilityOptions NetworkCompatibilityOptions -) - -var transportDialer = &net.Dialer{ - Timeout: 10 * time.Second, - KeepAlive: 30 * time.Second, -} - -func transportDialContext(ctx context.Context, network, addr string) (net.Conn, error) { - return dialWithDoHFallback(ctx, transportDialer, network, addr) -} - -var sharedTransport = &http.Transport{ - DialContext: transportDialContext, - MaxIdleConns: 100, - MaxIdleConnsPerHost: 10, - MaxConnsPerHost: 20, - IdleConnTimeout: 60 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - // Downloads ride this transport; some extension providers prepare the - // file server-side before the first byte, so give TTFB more headroom - // than the API/metadata transports. The 60s stall watchdog still bounds - // dead transfers. - ResponseHeaderTimeout: 120 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DisableKeepAlives: false, - ForceAttemptHTTP2: true, - WriteBufferSize: 64 * 1024, - ReadBufferSize: 64 * 1024, - DisableCompression: true, - TLSClientConfig: newTLSCompatibilityConfig(false), -} - -var extensionAPITransport = &http.Transport{ - DialContext: transportDialContext, - MaxIdleConns: 100, - MaxIdleConnsPerHost: 10, - MaxConnsPerHost: 20, - IdleConnTimeout: 60 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ResponseHeaderTimeout: 45 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DisableKeepAlives: false, - ForceAttemptHTTP2: true, - WriteBufferSize: 64 * 1024, - ReadBufferSize: 64 * 1024, - DisableCompression: false, - TLSClientConfig: newTLSCompatibilityConfig(false), -} - -var metadataTransport = &http.Transport{ - DialContext: transportDialContext, - MaxIdleConns: 30, - MaxIdleConnsPerHost: 5, - MaxConnsPerHost: 10, - IdleConnTimeout: 60 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ResponseHeaderTimeout: 45 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DisableKeepAlives: false, - ForceAttemptHTTP2: true, - WriteBufferSize: 32 * 1024, - ReadBufferSize: 32 * 1024, - // Metadata responses are JSON; transparent gzip cuts transfer size several - // times over. Downloads stay on sharedTransport with compression disabled. - DisableCompression: false, - TLSClientConfig: newTLSCompatibilityConfig(false), -} - -var sharedClient = &http.Client{ - Transport: newCompatibilityTransport(sharedTransport), - Timeout: DefaultTimeout, -} - -func NewHTTPClientWithTimeout(timeout time.Duration) *http.Client { - return &http.Client{ - Transport: newCompatibilityTransport(sharedTransport), - Timeout: timeout, - } -} - -func NewMetadataHTTPClient(timeout time.Duration) *http.Client { - return &http.Client{ - Transport: newCompatibilityTransport(metadataTransport), - Timeout: timeout, - } -} - -func CloseIdleConnections() { - sharedTransport.CloseIdleConnections() - extensionAPITransport.CloseIdleConnections() - metadataTransport.CloseIdleConnections() - closeUTLSIdleConnections() -} - -func SetNetworkCompatibilityOptions(allowHTTP, insecureTLS bool) { - if insecureTLS { - GoLog("[HTTP] Ignoring insecure TLS compatibility request; certificate verification remains enabled\n") - } - networkCompatibilityMu.Lock() - networkCompatibilityOptions = NetworkCompatibilityOptions{ - AllowHTTP: allowHTTP, - InsecureTLS: false, - } - networkCompatibilityMu.Unlock() - - applyTLSCompatibility(sharedTransport, false) - applyTLSCompatibility(extensionAPITransport, false) - applyTLSCompatibility(metadataTransport, false) - CloseIdleConnections() - - GoLog("[HTTP] Network compatibility options updated: allow_http=%v insecure_tls=false\n", allowHTTP) -} - -func GetNetworkCompatibilityOptions() NetworkCompatibilityOptions { - networkCompatibilityMu.RLock() - defer networkCompatibilityMu.RUnlock() - return networkCompatibilityOptions -} - -func applyTLSCompatibility(transport *http.Transport, insecureTLS bool) { - transport.TLSClientConfig = newTLSCompatibilityConfig(insecureTLS) -} - -type compatibilityTransport struct { - base http.RoundTripper -} - -func newCompatibilityTransport(base http.RoundTripper) http.RoundTripper { - return &compatibilityTransport{base: base} -} - -func (t *compatibilityTransport) RoundTrip(req *http.Request) (*http.Response, error) { - if req == nil || req.URL == nil { - return t.base.RoundTrip(req) - } - - opts := GetNetworkCompatibilityOptions() - if !opts.AllowHTTP || req.URL.Scheme != "https" { - return t.base.RoundTrip(req) - } - - // Compatibility mode should prefer HTTPS and only fallback to HTTP on - // transport-level failures. Forcing HTTP unconditionally can trigger - // redirect loops (http -> https) on providers that enforce HTTPS. - resp, err := t.base.RoundTrip(req) - if err == nil { - return resp, nil - } - - if !canFallbackToHTTP(req) { - return nil, err - } - - fallbackReq, cloneErr := cloneRequestWithHTTPScheme(req, "http") - if cloneErr != nil { - return nil, err - } - - GoLog("[HTTP] HTTPS request failed for %s, retrying over HTTP: %v\n", req.URL.Host, err) - return t.base.RoundTrip(fallbackReq) -} - -func canFallbackToHTTP(req *http.Request) bool { - if req == nil { - return false - } - - switch strings.ToUpper(req.Method) { - case http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodDelete: - return true - default: - return req.GetBody != nil - } -} - -func cloneRequestWithHTTPScheme(req *http.Request, scheme string) (*http.Request, error) { - reqCopy := req.Clone(req.Context()) - if req.Body != nil && req.GetBody != nil { - bodyCopy, err := req.GetBody() - if err != nil { - return nil, err - } - reqCopy.Body = bodyCopy - } - - urlCopy := *req.URL - urlCopy.Scheme = scheme - reqCopy.URL = &urlCopy - return reqCopy, nil -} - -func DoRequestWithUserAgent(client *http.Client, req *http.Request) (*http.Response, error) { - req.Header.Set("User-Agent", userAgentForURL(req.URL)) - resp, err := client.Do(req) - if err != nil { - CheckAndLogISPBlocking(err, req.URL.String(), "HTTP") - } - return resp, err -} - -type RetryConfig struct { - MaxRetries int - InitialDelay time.Duration - MaxDelay time.Duration - BackoffFactor float64 -} - -func DefaultRetryConfig() RetryConfig { - return RetryConfig{ - MaxRetries: DefaultMaxRetries, - InitialDelay: DefaultRetryDelay, - MaxDelay: 16 * time.Second, - BackoffFactor: 2.0, - } -} - -func DoRequestWithRetry(client *http.Client, req *http.Request, config RetryConfig) (*http.Response, error) { - var lastErr error - delay := config.InitialDelay - - for attempt := 0; attempt <= config.MaxRetries; attempt++ { - reqCopy := req.Clone(req.Context()) - reqCopy.Header.Set("User-Agent", userAgentForURL(reqCopy.URL)) - - resp, err := client.Do(reqCopy) - if err != nil { - lastErr = err - - if isHardConnectivityBlock(err) { - return nil, WrapErrorWithISPCheck(err, reqCopy.URL.String(), "HTTP") - } - - if attempt < config.MaxRetries { - GoLog("[HTTP] Request failed (attempt %d/%d): %v, retrying in %v...\n", - attempt+1, config.MaxRetries+1, err, delay) - if err := sleepRetry(req.Context(), delay); err != nil { - return nil, err - } - delay = calculateNextDelay(delay, config) - } - continue - } - - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - return resp, nil - } - - if resp.StatusCode == 429 { - drainAndCloseResponseBody(resp.Body, maxRetryResponseBodyBytes) - retryAfter := getRetryAfterDuration(resp) - if retryAfter > 0 { - delay = retryAfter - } - lastErr = fmt.Errorf("rate limited (429)") - if attempt < config.MaxRetries { - GoLog("[HTTP] Rate limited, waiting %v before retry...\n", delay) - if err := sleepRetry(req.Context(), delay); err != nil { - return nil, err - } - delay = calculateNextDelay(delay, config) - } - continue - } - - if resp.StatusCode == 403 || resp.StatusCode == 451 { - body, _ := io.ReadAll(io.LimitReader(resp.Body, maxRetryResponseBodyBytes+1)) - resp.Body.Close() - if int64(len(body)) > maxRetryResponseBodyBytes { - body = body[:maxRetryResponseBodyBytes] - } - bodyStr := strings.ToLower(string(body)) - - ispBlockingIndicators := []string{ - "blocked", "forbidden", "access denied", "not available in your", - "restricted", "censored", "unavailable for legal", "blocked by", - } - - for _, indicator := range ispBlockingIndicators { - if strings.Contains(bodyStr, indicator) { - LogError("HTTP", "ISP BLOCKING DETECTED via HTTP %d response", resp.StatusCode) - LogError("HTTP", "Domain: %s", req.URL.Host) - LogError("HTTP", "Response contains: %s", indicator) - LogError("HTTP", "Suggestion: Try using a VPN or changing your DNS to 1.1.1.1 or 8.8.8.8") - return nil, fmt.Errorf("ISP blocking detected for %s (HTTP %d) - try using VPN or change DNS", req.URL.Host, resp.StatusCode) - } - } - - // No blocking marker: hand the caller back a readable body in - // place of the one consumed by the scan above. - resp.Body = io.NopCloser(bytes.NewReader(body)) - } - - if resp.StatusCode >= 500 { - drainAndCloseResponseBody(resp.Body, maxRetryResponseBodyBytes) - if retryAfter := getRetryAfterDuration(resp); retryAfter > 0 { - delay = retryAfter - } - lastErr = fmt.Errorf("server error: HTTP %d", resp.StatusCode) - if attempt < config.MaxRetries { - GoLog("[HTTP] Server error %d, retrying in %v...\n", resp.StatusCode, delay) - if err := sleepRetry(req.Context(), delay); err != nil { - return nil, err - } - delay = calculateNextDelay(delay, config) - } - continue - } - - return resp, nil - } - - return nil, fmt.Errorf("request failed after %d retries: %w", config.MaxRetries+1, lastErr) -} - -func drainAndCloseResponseBody(body io.ReadCloser, limit int64) { - if body == nil { - return - } - if limit > 0 { - _, _ = io.Copy(io.Discard, io.LimitReader(body, limit)) - } - _ = body.Close() -} - -// sleepRetry waits out a retry delay, aborting early when the request context -// is cancelled so a cancelled download never sits in a backoff sleep. -func sleepRetry(ctx context.Context, d time.Duration) error { - timer := time.NewTimer(d) - defer timer.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - return nil - } -} - -// jitterFloat returns a fraction in [0,1); overridable in tests for -// deterministic backoff assertions. -var jitterFloat = rand.Float64 - -func calculateNextDelay(currentDelay time.Duration, config RetryConfig) time.Duration { - nextDelay := time.Duration(float64(currentDelay) * config.BackoffFactor) - capped := min(nextDelay, config.MaxDelay) - // Full jitter: spread retries between InitialDelay and the capped - // exponential ceiling to avoid synchronized thundering-herd retries. - if capped <= config.InitialDelay { - return capped - } - span := capped - config.InitialDelay - return config.InitialDelay + time.Duration(jitterFloat()*float64(span)) -} - -// maxRetryAfterDelay caps honored Retry-After values so a hostile or -// misconfigured server cannot park a retry loop for an hour. -const maxRetryAfterDelay = 2 * time.Minute - -// Returns 0 if the header is missing or invalid so callers can keep their -// normal exponential backoff instead of stalling for an arbitrary minute. -func getRetryAfterDuration(resp *http.Response) time.Duration { - retryAfter := resp.Header.Get("Retry-After") - if retryAfter == "" { - return 0 - } - - if seconds, err := strconv.Atoi(retryAfter); err == nil { - return min(time.Duration(seconds)*time.Second, maxRetryAfterDelay) - } - - if t, err := http.ParseTime(retryAfter); err == nil { - duration := time.Until(t) - if duration > 0 { - return min(duration, maxRetryAfterDelay) - } - } - - return 0 -} - -func ReadResponseBody(resp *http.Response) ([]byte, error) { - if resp == nil { - return nil, fmt.Errorf("response is nil") - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - - if len(body) == 0 { - return nil, fmt.Errorf("response body is empty") - } - - return body, nil -} - -type ISPBlockingError struct { - Domain string - Reason string - OriginalErr error -} - -func (e *ISPBlockingError) Error() string { - return fmt.Sprintf("ISP blocking detected for %s: %s", e.Domain, e.Reason) -} - -// isTransientNetworkError reports retryable transport failures such as -// timeouts and temporary DNS errors. Permanent DNS misses are excluded. -func isTransientNetworkError(err error) bool { - if err == nil { - return false - } - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { - return true - } - var netErr net.Error - return errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) -} - -// isConnectivityFailure reports DNS, dial, timeout, TLS, or truncated transport -// errors. Application-level API messages are excluded. -func isConnectivityFailure(err error) bool { - return connectivityFailureReason(err) != "" -} - -func connectivityFailureReason(err error) string { - if err == nil { - return "" - } - if errors.Is(err, context.DeadlineExceeded) { - return "Request timed out - ISP may be throttling" - } - if errors.Is(err, io.ErrUnexpectedEOF) { - return "Connection closed unexpectedly - ISP may be blocking" - } - - var urlErr *url.Error - if errors.As(err, &urlErr) { - if urlErr.Timeout() { - return "Connection timed out - ISP may be blocking access" - } - if urlErr.Err != nil { - if reason := connectivityFailureReason(urlErr.Err); reason != "" { - return reason - } - } - } - - var dnsErr *net.DNSError - if errors.As(err, &dnsErr) { - if dnsErr.IsNotFound || dnsErr.IsTimeout || dnsErr.IsTemporary { - return "DNS resolution failed - domain may be blocked by ISP" - } - } - - var opErr *net.OpError - if errors.As(err, &opErr) { - if opErr.Timeout() { - return "Connection timed out - ISP may be blocking access" - } - var errno syscall.Errno - if errors.As(opErr.Err, &errno) { - switch errno { - case syscall.ECONNREFUSED: - return "Connection refused - port may be blocked by ISP/firewall" - case syscall.ECONNRESET: - return "Connection reset - ISP may be intercepting traffic" - case syscall.ETIMEDOUT: - return "Connection timed out - ISP may be blocking access" - case syscall.ENETUNREACH: - return "Network unreachable - ISP may be blocking route" - case syscall.EHOSTUNREACH: - return "Host unreachable - ISP may be blocking destination" - } - } - } - - var tlsErr *tls.RecordHeaderError - if errors.As(err, &tlsErr) { - return "TLS handshake failed - ISP may be intercepting HTTPS traffic" - } - - var certErr x509.CertificateInvalidError - if errors.As(err, &certErr) { - return "Certificate error - ISP may be using MITM proxy" - } - var hostnameErr x509.HostnameError - if errors.As(err, &hostnameErr) { - return "Certificate error - ISP may be using MITM proxy" - } - var unknownAuth x509.UnknownAuthorityError - if errors.As(err, &unknownAuth) { - return "Certificate error - ISP may be using MITM proxy" - } - - return "" -} - -// isTLSHandshakeOrResetError reports TLS handshake/cert failures and TCP resets -// that should trigger a Chrome fingerprint retry. -func isTLSHandshakeOrResetError(err error) bool { - if err == nil { - return false - } - var recordErr *tls.RecordHeaderError - if errors.As(err, &recordErr) { - return true - } - var certErr x509.CertificateInvalidError - if errors.As(err, &certErr) { - return true - } - var hostnameErr x509.HostnameError - if errors.As(err, &hostnameErr) { - return true - } - var unknownAuth x509.UnknownAuthorityError - if errors.As(err, &unknownAuth) { - return true - } - var opErr *net.OpError - if errors.As(err, &opErr) { - var errno syscall.Errno - if errors.As(opErr.Err, &errno) && errno == syscall.ECONNRESET { - return true - } - } - return false -} - -// isHardConnectivityBlock reports transport failures that signal an active -// block (DNS not found, connection refused/reset, TLS/cert MITM) and should -// abort retries immediately. Timeouts and deadline-exceeded are treated as -// transient and excluded so they fall through to normal retry backoff. -func isHardConnectivityBlock(err error) bool { - if err == nil { - return false - } - - var urlErr *url.Error - if errors.As(err, &urlErr) && urlErr.Timeout() { - return false - } - - var dnsErr *net.DNSError - if errors.As(err, &dnsErr) { - return dnsErr.IsNotFound - } - - var opErr *net.OpError - if errors.As(err, &opErr) { - if opErr.Timeout() { - return false - } - var errno syscall.Errno - if errors.As(opErr.Err, &errno) { - switch errno { - case syscall.ECONNREFUSED, syscall.ECONNRESET: - return true - } - } - } - - return isTLSHandshakeOrResetError(err) -} - -func IsISPBlocking(err error, requestURL string) *ISPBlockingError { - if err == nil { - return nil - } - reason := connectivityFailureReason(err) - if reason == "" { - return nil - } - return &ISPBlockingError{ - Domain: extractDomain(requestURL), - Reason: reason, - OriginalErr: err, - } -} - -func CheckAndLogISPBlocking(err error, requestURL string, tag string) bool { - ispErr := IsISPBlocking(err, requestURL) - if ispErr != nil { - LogError(tag, "ISP BLOCKING DETECTED: %s", ispErr.Error()) - LogError(tag, "Domain: %s", ispErr.Domain) - LogError(tag, "Reason: %s", ispErr.Reason) - LogError(tag, "Original error: %v", ispErr.OriginalErr) - LogError(tag, "Suggestion: Try using a VPN or changing your DNS to 1.1.1.1 or 8.8.8.8") - return true - } - return false -} - -func extractDomain(rawURL string) string { - if rawURL == "" { - return "unknown" - } - - parsed, err := url.Parse(rawURL) - if err != nil { - rawURL = strings.TrimPrefix(rawURL, "https://") - rawURL = strings.TrimPrefix(rawURL, "http://") - if idx := strings.Index(rawURL, "/"); idx > 0 { - return rawURL[:idx] - } - return rawURL - } - - if parsed.Host != "" { - return parsed.Host - } - return "unknown" -} - -func WrapErrorWithISPCheck(err error, requestURL string, tag string) error { - if err == nil { - return nil - } - - if CheckAndLogISPBlocking(err, requestURL, tag) { - domain := extractDomain(requestURL) - return fmt.Errorf("ISP blocking detected for %s - try using VPN or change DNS to 1.1.1.1/8.8.8.8: %w", domain, err) - } - - return err -} diff --git a/go_backend/httputil_ios.go b/go_backend/httputil_ios.go deleted file mode 100644 index 58acba1e..00000000 --- a/go_backend/httputil_ios.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build ios - -package gobackend - -import ( - "net/http" -) - -func GetCloudflareBypassClient() *http.Client { - return sharedClient -} - -func closeUTLSIdleConnections() {} - -func DoRequestWithCloudflareBypass(req *http.Request) (*http.Response, error) { - req.Header.Set("User-Agent", userAgentForURL(req.URL)) - resp, err := sharedClient.Do(req) - if err != nil { - CheckAndLogISPBlocking(err, req.URL.String(), "HTTP") - } - return resp, err -} diff --git a/go_backend/httputil_retry_test.go b/go_backend/httputil_retry_test.go deleted file mode 100644 index 3b0c6ae8..00000000 --- a/go_backend/httputil_retry_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package gobackend - -import ( - "bytes" - "context" - "io" - "net/http" - "strings" - "testing" - "time" -) - -func TestRetryHardening(t *testing.T) { - resp := &http.Response{Header: http.Header{"Retry-After": []string{"3600"}}} - if d := getRetryAfterDuration(resp); d != maxRetryAfterDelay { - t.Fatalf("Retry-After 3600s clamped to %v, want %v", d, maxRetryAfterDelay) - } - - // A 403 without ISP-blocking markers must reach the caller with a - // readable body even though the marker scan consumed the original. - client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: 403, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader("quota exceeded")), - Request: req, - }, nil - })} - got, err := DoRequestWithRetry(client, mustNewRequest(t, "https://example.com/x"), DefaultRetryConfig()) - if err != nil || got.StatusCode != 403 { - t.Fatalf("DoRequestWithRetry = %#v/%v", got, err) - } - body, err := io.ReadAll(got.Body) - got.Body.Close() - if err != nil || string(body) != "quota exceeded" { - t.Fatalf("403 body = %q/%v, want restored body", body, err) - } - - // Backoff sleeps must abort on context cancellation. - ctx, cancel := context.WithCancel(context.Background()) - go func() { - time.Sleep(10 * time.Millisecond) - cancel() - }() - start := time.Now() - if err := sleepRetry(ctx, 5*time.Second); err == nil { - t.Fatal("sleepRetry ignored cancellation") - } - if time.Since(start) > time.Second { - t.Fatal("sleepRetry did not abort promptly on cancel") - } -} - -func TestRetryDrainsSmallFailureBodyForConnectionReuse(t *testing.T) { - failedBody := bytes.NewBufferString("temporary failure") - attempts := 0 - client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - attempts++ - if attempts == 1 { - return &http.Response{ - StatusCode: http.StatusServiceUnavailable, - Header: make(http.Header), - Body: io.NopCloser(failedBody), - Request: req, - }, nil - } - return &http.Response{ - StatusCode: http.StatusNoContent, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader("")), - Request: req, - }, nil - })} - resp, err := DoRequestWithRetry( - client, - mustNewRequest(t, "https://example.com/reuse"), - RetryConfig{MaxRetries: 1}, - ) - if err != nil || resp.StatusCode != http.StatusNoContent { - t.Fatalf("DoRequestWithRetry = %#v/%v", resp, err) - } - resp.Body.Close() - if failedBody.Len() != 0 { - t.Fatalf("retry body retained %d unread bytes", failedBody.Len()) - } -} - -func TestRetryCapsInspectedForbiddenBody(t *testing.T) { - huge := strings.Repeat("x", int(maxRetryResponseBodyBytes)+1024) - client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: http.StatusForbidden, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(huge)), - Request: req, - }, nil - })} - resp, err := DoRequestWithRetry( - client, - mustNewRequest(t, "https://example.com/capped"), - RetryConfig{MaxRetries: 0}, - ) - if err != nil { - t.Fatalf("DoRequestWithRetry returned error: %v", err) - } - body, readErr := io.ReadAll(resp.Body) - resp.Body.Close() - if readErr != nil || int64(len(body)) != maxRetryResponseBodyBytes { - t.Fatalf("capped body length = %d/%v, want %d", len(body), readErr, maxRetryResponseBodyBytes) - } -} diff --git a/go_backend/httputil_supplement_test.go b/go_backend/httputil_supplement_test.go deleted file mode 100644 index 701d1bc4..00000000 --- a/go_backend/httputil_supplement_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package gobackend - -import ( - "context" - "crypto/x509" - "encoding/pem" - "io" - "net" - "net/http" - "net/url" - "strings" - "syscall" - "testing" - "time" -) - -func TestHTTPUtilityHelpers(t *testing.T) { - SetAppVersion("7.0.0") - apiURL := mustParseURL(t, "https://api.zarz.moe/test") - if ua := userAgentForURL(apiURL); !strings.Contains(ua, "7.0.0") { - t.Fatalf("api user agent = %q", ua) - } - if userAgentForURL(nil) == "" || userAgentForURL(mustParseURL(t, "https://example.com")) == "" { - t.Fatal("expected fallback user agent") - } - if NewHTTPClientWithTimeout(time.Second).Timeout != time.Second || NewMetadataHTTPClient(time.Second).Timeout != time.Second { - t.Fatal("client timeout mismatch") - } - if sharedTransport.TLSClientConfig == nil || sharedTransport.TLSClientConfig.RootCAs == nil { - t.Fatal("expected supplemental TLS root pool") - } - block, _ := pem.Decode([]byte(isrgRootX2PEM)) - if block == nil { - t.Fatal("failed to decode ISRG Root X2") - } - rootX2, err := x509.ParseCertificate(block.Bytes) - if err != nil { - t.Fatalf("failed to parse ISRG Root X2: %v", err) - } - if _, err := rootX2.Verify(x509.VerifyOptions{ - Roots: supplementalRootCAs(), - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, - }); err != nil { - t.Fatalf("ISRG Root X2 should verify with supplemental roots: %v", err) - } - SetNetworkCompatibilityOptions(true, true) - if opts := GetNetworkCompatibilityOptions(); !opts.AllowHTTP || opts.InsecureTLS { - t.Fatalf("network opts = %#v", opts) - } - if sharedTransport.TLSClientConfig.InsecureSkipVerify { - t.Fatal("TLS verification must remain enabled in compatibility mode") - } - SetNetworkCompatibilityOptions(false, false) - if sharedTransport.TLSClientConfig == nil || sharedTransport.TLSClientConfig.InsecureSkipVerify { - t.Fatal("expected secure TLS config to be restored") - } - if !canFallbackToHTTP(&http.Request{Method: http.MethodGet}) { - t.Fatal("GET should fallback") - } - if canFallbackToHTTP(&http.Request{Method: http.MethodPost}) { - t.Fatal("POST without GetBody should not fallback") - } - req, _ := http.NewRequest(http.MethodPost, "https://example.com/path", strings.NewReader("body")) - req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("body")), nil } - cloned, err := cloneRequestWithHTTPScheme(req, "http") - if err != nil || cloned.URL.Scheme != "http" || cloned.Body == nil { - t.Fatalf("cloneRequestWithHTTPScheme = %#v/%v", cloned, err) - } - - client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - if req.Header.Get("User-Agent") == "" { - t.Fatal("missing User-Agent") - } - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("ok")), Request: req}, nil - })} - resp, err := DoRequestWithUserAgent(client, mustNewRequest(t, "https://example.com/ok")) - if err != nil || resp.StatusCode != 200 { - t.Fatalf("DoRequestWithUserAgent = %#v/%v", resp, err) - } - resp.Body.Close() - - attempts := 0 - retryClient := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - attempts++ - switch attempts { - case 1: - return &http.Response{StatusCode: 500, Body: io.NopCloser(strings.NewReader("server")), Request: req}, nil - case 2: - return &http.Response{StatusCode: 429, Header: http.Header{"Retry-After": []string{"0"}}, Body: io.NopCloser(strings.NewReader("rate")), Request: req}, nil - default: - return &http.Response{StatusCode: 204, Body: io.NopCloser(strings.NewReader("")), Request: req}, nil - } - })} - resp, err = DoRequestWithRetry(retryClient, mustNewRequest(t, "https://example.com/retry"), RetryConfig{MaxRetries: 3, InitialDelay: 0, MaxDelay: time.Millisecond, BackoffFactor: 2}) - if err != nil || resp.StatusCode != 204 || attempts != 3 { - t.Fatalf("DoRequestWithRetry = %#v/%v attempts=%d", resp, err, attempts) - } - resp.Body.Close() - blockingClient := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: 403, Body: io.NopCloser(strings.NewReader("access denied by region")), Request: req}, nil - })} - if _, err := DoRequestWithRetry(blockingClient, mustNewRequest(t, "https://blocked.example.com"), RetryConfig{MaxRetries: 0}); err == nil { - t.Fatal("expected blocking retry error") - } - - if _, err := ReadResponseBody(nil); err == nil { - t.Fatal("expected nil response body error") - } - if _, err := ReadResponseBody(&http.Response{Body: io.NopCloser(strings.NewReader(""))}); err == nil { - t.Fatal("expected empty response body error") - } - if body, err := ReadResponseBody(&http.Response{Body: io.NopCloser(strings.NewReader("ok"))}); err != nil || string(body) != "ok" { - t.Fatalf("ReadResponseBody = %q/%v", body, err) - } - origJitter := jitterFloat - jitterFloat = func() float64 { return 1 } - if calculateNextDelay(10*time.Millisecond, RetryConfig{BackoffFactor: 3, MaxDelay: 20 * time.Millisecond}) != 20*time.Millisecond { - t.Fatal("calculateNextDelay mismatch") - } - jitterFloat = origJitter - if getRetryAfterDuration(&http.Response{Header: http.Header{"Retry-After": []string{"bad"}}}) != 0 { - t.Fatal("invalid retry-after should be zero") - } - resetErr := &net.OpError{Op: "read", Err: syscall.ECONNRESET} - if isp := IsISPBlocking(resetErr, "https://example.com/x"); isp == nil || !strings.Contains(isp.Error(), "example.com") { - t.Fatalf("IsISPBlocking = %#v", isp) - } - timeoutErr := &net.OpError{Op: "dial", Err: syscall.ETIMEDOUT} - if !CheckAndLogISPBlocking(timeoutErr, "https://timeout.example/x", "test") { - t.Fatal("expected logged ISP blocking") - } - refusedErr := &net.OpError{Op: "dial", Err: syscall.ECONNREFUSED} - if wrapped := WrapErrorWithISPCheck(refusedErr, "https://refused.example/x", "test"); wrapped == nil || !strings.Contains(wrapped.Error(), "ISP blocking") { - t.Fatalf("WrapErrorWithISPCheck = %v", wrapped) - } - if !isTransientNetworkError(context.DeadlineExceeded) || isTransientNetworkError(&net.DNSError{IsNotFound: true}) { - t.Fatal("isTransientNetworkError mismatch") - } - if !isConnectivityFailure(&net.DNSError{IsNotFound: true}) || !isConnectivityFailure(context.DeadlineExceeded) { - t.Fatal("isConnectivityFailure mismatch") - } - if WrapErrorWithISPCheck(nil, "", "test") != nil { - t.Fatal("nil wrap should stay nil") - } - if extractDomain("https://example.com/path") != "example.com" || extractDomain("bad://") != "unknown" || extractDomain("") != "unknown" { - t.Fatal("extractDomain mismatch") - } -} - -func TestRateLimiterHelpers(t *testing.T) { - limiter := NewRateLimiter(1, time.Hour) - if limiter.Available() != 1 { - t.Fatalf("available = %d", limiter.Available()) - } - if !limiter.TryAcquire() || limiter.TryAcquire() { - t.Fatal("TryAcquire mismatch") - } - if limiter.Available() != 0 { - t.Fatalf("available after acquire = %d", limiter.Available()) - } -} - -func mustNewRequest(t *testing.T, rawURL string) *http.Request { - t.Helper() - req, err := http.NewRequest(http.MethodGet, rawURL, nil) - if err != nil { - t.Fatal(err) - } - return req -} - -func mustParseURL(t *testing.T, rawURL string) *url.URL { - t.Helper() - parsed, err := url.Parse(rawURL) - if err != nil { - t.Fatal(err) - } - return parsed -} diff --git a/go_backend/httputil_utls.go b/go_backend/httputil_utls.go deleted file mode 100644 index 39112924..00000000 --- a/go_backend/httputil_utls.go +++ /dev/null @@ -1,337 +0,0 @@ -//go:build !ios - -package gobackend - -import ( - "context" - "io" - "net" - "net/http" - "net/url" - "strings" - "sync" - "time" - - utls "github.com/refraction-networking/utls" - "golang.org/x/net/http2" -) - -// utlsSessionCache is shared by every uTLS handshake so TLS 1.3 tickets enable -// resumption (fewer round-trips) across requests and hosts. -var utlsSessionCache = utls.NewLRUClientSessionCache(0) - -// utlsTransport dials with a Chrome TLS fingerprint and pools one healthy HTTP/2 -// connection per host, re-dialing when it dies (e.g. after a network switch). -type utlsTransport struct { - dialer *net.Dialer - h2 *http2.Transport - mu sync.Mutex - conns map[string]pooledHTTP2ClientConn -} - -type pooledHTTP2ClientConn interface { - RoundTrip(*http.Request) (*http.Response, error) - ReserveNewRequest() bool - State() http2.ClientConnState - Close() error - Shutdown(context.Context) error -} - -func newUTLSTransport() *utlsTransport { - return &utlsTransport{ - dialer: &net.Dialer{ - Timeout: 10 * Second, - KeepAlive: 30 * Second, - }, - h2: &http2.Transport{}, - conns: make(map[string]pooledHTTP2ClientConn), - } -} - -func (t *utlsTransport) RoundTrip(req *http.Request) (*http.Response, error) { - if req.URL.Scheme != "https" { - return sharedTransport.RoundTrip(req) - } - - host := req.URL.Hostname() - addr := net.JoinHostPort(host, t.getPort(req.URL)) - - if cc := t.cachedConn(addr); cc != nil { - resp, err := cc.RoundTrip(req) - if err == nil { - return resp, nil - } - if req.Context().Err() != nil { - return nil, err - } - // A pooled conn can be silently dead after a network switch. Drop it - // and, when the request is safely repeatable, fall through to a fresh - // dial instead of failing where the old dial-per-request code would - // have succeeded. - t.invalidate(addr, cc) - retryReq, ok := rewindRequestBody(req) - if !ok { - return nil, err - } - req = retryReq - } - - tlsConn, proto, err := t.dial(req.Context(), host, addr) - if err != nil { - return nil, err - } - - if proto != "h2" { - // HTTP/1.1: single-use conn closed once the body is drained. - transport := &http.Transport{ - DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return tlsConn, nil - }, - DisableKeepAlives: true, - } - return transport.RoundTrip(req) - } - - cc, err := t.h2.NewClientConn(tlsConn) - if err != nil { - tlsConn.Close() - return nil, err - } - pooled := t.storeConn(addr, cc) - return pooled.RoundTrip(req) -} - -// rewindRequestBody returns a request whose body can be sent again after a -// failed attempt on a pooled connection: bodyless requests as-is, requests -// with GetBody with a rebuilt body, anything else not ok. -func rewindRequestBody(req *http.Request) (*http.Request, bool) { - if req.Body == nil { - return req, true - } - if req.GetBody == nil { - return nil, false - } - body, err := req.GetBody() - if err != nil { - return nil, false - } - retryReq := req.Clone(req.Context()) - retryReq.Body = body - return retryReq, true -} - -// dial opens a TCP connection and completes the Chrome-fingerprint TLS handshake, -// returning the connection and negotiated ALPN protocol. -func (t *utlsTransport) dial(ctx context.Context, host, addr string) (*utls.UConn, string, error) { - conn, err := dialWithDoHFallback(ctx, t.dialer, "tcp", addr) - if err != nil { - return nil, "", err - } - - opts := GetNetworkCompatibilityOptions() - tlsConn := utls.UClient(conn, &utls.Config{ - RootCAs: supplementalRootCAs(), - InsecureSkipVerify: opts.InsecureTLS, - ServerName: host, - NextProtos: []string{"h2", "http/1.1"}, - ClientSessionCache: utlsSessionCache, - }, utls.HelloChrome_Auto) - - if err := tlsConn.Handshake(); err != nil { - conn.Close() - return nil, "", err - } - return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol, nil -} - -func (t *utlsTransport) cachedConn(addr string) pooledHTTP2ClientConn { - t.mu.Lock() - cc := t.conns[addr] - if cc != nil && cc.ReserveNewRequest() { - t.mu.Unlock() - return cc - } - if cc != nil { - delete(t.conns, addr) - } - t.mu.Unlock() - if cc != nil { - retirePooledHTTP2Conn(cc) - } - return nil -} - -func (t *utlsTransport) invalidate(addr string, cc pooledHTTP2ClientConn) { - t.mu.Lock() - removed := false - if t.conns[addr] == cc { - delete(t.conns, addr) - removed = true - } - t.mu.Unlock() - if removed { - retirePooledHTTP2Conn(cc) - } -} - -// storeConn caches cc, but if a concurrent dial already cached a healthy conn for -// addr it discards the freshly built cc (no in-flight requests) and returns the -// existing one, avoiding a leaked connection. -func (t *utlsTransport) storeConn(addr string, cc pooledHTTP2ClientConn) pooledHTTP2ClientConn { - t.mu.Lock() - if existing := t.conns[addr]; existing != nil && existing.ReserveNewRequest() { - t.mu.Unlock() - _ = cc.Close() - return existing - } - stale := t.conns[addr] - t.conns[addr] = cc - _ = cc.ReserveNewRequest() - t.mu.Unlock() - if stale != nil { - retirePooledHTTP2Conn(stale) - } - return cc -} - -// retirePooledHTTP2Conn prevents new streams while allowing existing streams -// to finish. The independent watchdog also bounds Shutdown implementations -// that block before observing their context. -func pooledHTTP2RetirementTimeout(state http2.ClientConnState) time.Duration { - if state.StreamsActive > 0 || state.StreamsPending > 0 || state.StreamsReserved > 0 { - return 0 - } - return 5 * Second -} - -func retirePooledHTTP2Conn(conn pooledHTTP2ClientConn) { - go func() { - retirePooledHTTP2ConnWithTimeout(conn, pooledHTTP2RetirementTimeout(conn.State())) - }() -} - -func retirePooledHTTP2ConnWithTimeout(conn pooledHTTP2ClientConn, timeout time.Duration) { - go func() { - var closeOnce sync.Once - forceClose := func() { - closeOnce.Do(func() { _ = conn.Close() }) - } - if timeout <= 0 { - if err := conn.Shutdown(context.Background()); err != nil { - forceClose() - } - return - } - - watchdogDone := make(chan struct{}) - watchdog := time.AfterFunc(timeout, func() { - forceClose() - close(watchdogDone) - }) - if err := conn.Shutdown(context.Background()); err != nil { - forceClose() - } - if !watchdog.Stop() { - <-watchdogDone - } - }() -} - -// closeIdleConnections drops every pooled conn so the next request re-dials — -// needed after a network switch, where pooled conns are silently dead and the -// first request would otherwise hang on one until its timeout. Conns are shut -// down gracefully so in-flight streams finish (or fail) before the close. -func (t *utlsTransport) closeIdleConnections() { - t.mu.Lock() - conns := t.conns - t.conns = make(map[string]pooledHTTP2ClientConn) - t.mu.Unlock() - for _, cc := range conns { - retirePooledHTTP2Conn(cc) - } -} - -// closeUTLSIdleConnections lets platform-neutral code (CloseIdleConnections) -// reach the uTLS pool; the ios build provides a no-op stub. -func closeUTLSIdleConnections() { - cloudflareBypassTransport.closeIdleConnections() -} - -func (t *utlsTransport) getPort(u *url.URL) string { - if u.Port() != "" { - return u.Port() - } - if u.Scheme == "https" { - return "443" - } - return "80" -} - -var cloudflareBypassTransport = newUTLSTransport() - -var cloudflareBypassClient = &http.Client{ - Transport: cloudflareBypassTransport, - Timeout: DefaultTimeout, -} - -func GetCloudflareBypassClient() *http.Client { - return cloudflareBypassClient -} - -func DoRequestWithCloudflareBypass(req *http.Request) (*http.Response, error) { - req.Header.Set("User-Agent", userAgentForURL(req.URL)) - - resp, err := sharedClient.Do(req) - if err == nil { - if resp.StatusCode == 403 || resp.StatusCode == 503 { - body, readErr := io.ReadAll(resp.Body) - resp.Body.Close() - - if readErr == nil { - bodyStr := strings.ToLower(string(body)) - cloudflareMarkers := []string{ - "cloudflare", "cf-ray", "checking your browser", - "please wait", "ddos protection", "ray id", - "enable javascript", "challenge-platform", - } - - isCloudflare := false - for _, marker := range cloudflareMarkers { - if strings.Contains(bodyStr, marker) { - isCloudflare = true - break - } - } - - if isCloudflare { - LogDebug("HTTP", "Cloudflare detected, retrying with Chrome TLS fingerprint...") - - reqCopy := req.Clone(req.Context()) - reqCopy.Header.Set("User-Agent", userAgentForURL(reqCopy.URL)) - - return cloudflareBypassClient.Do(reqCopy) - } - } - - return &http.Response{ - Status: resp.Status, - StatusCode: resp.StatusCode, - Header: resp.Header, - Body: io.NopCloser(strings.NewReader(string(body))), - }, nil - } - return resp, nil - } - - if isTLSHandshakeOrResetError(err) { - LogDebug("HTTP", "TLS error detected, retrying with Chrome TLS fingerprint: %v", err) - - reqCopy := req.Clone(req.Context()) - reqCopy.Header.Set("User-Agent", userAgentForURL(reqCopy.URL)) - - return cloudflareBypassClient.Do(reqCopy) - } - - CheckAndLogISPBlocking(err, req.URL.String(), "HTTP") - return nil, err -} diff --git a/go_backend/httputil_utls_pool_test.go b/go_backend/httputil_utls_pool_test.go deleted file mode 100644 index f33a78f6..00000000 --- a/go_backend/httputil_utls_pool_test.go +++ /dev/null @@ -1,187 +0,0 @@ -//go:build !ios - -package gobackend - -import ( - "context" - "net/http" - "sync" - "sync/atomic" - "testing" - "time" - - "golang.org/x/net/http2" -) - -type fakePooledHTTP2Conn struct { - healthy bool - streamsActive int - blockShutdown bool - ignoreShutdownCtx bool - closeCount atomic.Int32 - shutdownCount atomic.Int32 - shutdownOnce sync.Once - shutdownDone chan struct{} - forceCloseUnblocked chan struct{} - forceCloseOnce sync.Once -} - -func newFakePooledHTTP2Conn(healthy bool) *fakePooledHTTP2Conn { - return &fakePooledHTTP2Conn{ - healthy: healthy, - shutdownDone: make(chan struct{}), - forceCloseUnblocked: make(chan struct{}), - } -} - -func (c *fakePooledHTTP2Conn) RoundTrip(*http.Request) (*http.Response, error) { - return nil, nil -} - -func (c *fakePooledHTTP2Conn) ReserveNewRequest() bool { return c.healthy } - -func (c *fakePooledHTTP2Conn) State() http2.ClientConnState { - if c.healthy { - return http2.ClientConnState{ - StreamsActive: c.streamsActive, - MaxConcurrentStreams: 100, - } - } - return http2.ClientConnState{Closing: true, StreamsActive: c.streamsActive} -} - -func (c *fakePooledHTTP2Conn) Close() error { - c.closeCount.Add(1) - c.forceCloseOnce.Do(func() { close(c.forceCloseUnblocked) }) - return nil -} - -func (c *fakePooledHTTP2Conn) Shutdown(ctx context.Context) error { - c.shutdownCount.Add(1) - c.shutdownOnce.Do(func() { close(c.shutdownDone) }) - if c.ignoreShutdownCtx { - <-c.forceCloseUnblocked - return context.DeadlineExceeded - } - if c.blockShutdown { - <-ctx.Done() - return ctx.Err() - } - return nil -} - -func waitForShutdown(t *testing.T, conn *fakePooledHTTP2Conn) { - t.Helper() - select { - case <-conn.shutdownDone: - case <-time.After(time.Second): - t.Fatal("connection did not begin graceful shutdown") - } -} - -func TestUTLSPoolRetiresStaleCachedConnection(t *testing.T) { - transport := newUTLSTransport() - stale := newFakePooledHTTP2Conn(false) - transport.conns["example:443"] = stale - - if got := transport.cachedConn("example:443"); got != nil { - t.Fatalf("cachedConn returned stale connection: %#v", got) - } - waitForShutdown(t, stale) - if stale.closeCount.Load() != 0 { - t.Fatalf("gracefully retired connection was force closed") - } - if _, exists := transport.conns["example:443"]; exists { - t.Fatal("stale connection was not removed") - } -} - -func TestUTLSPoolStoreClosesDiscardedAndRetiresReplacedConnection(t *testing.T) { - transport := newUTLSTransport() - healthy := newFakePooledHTTP2Conn(true) - transport.conns["example:443"] = healthy - fresh := newFakePooledHTTP2Conn(true) - - if got := transport.storeConn("example:443", fresh); got != healthy { - t.Fatal("healthy pooled connection was not reused") - } - if fresh.closeCount.Load() != 1 { - t.Fatalf("discarded fresh close count = %d", fresh.closeCount.Load()) - } - - stale := newFakePooledHTTP2Conn(false) - transport.conns["example:443"] = stale - replacement := newFakePooledHTTP2Conn(true) - if got := transport.storeConn("example:443", replacement); got != replacement { - t.Fatal("stale connection was not replaced") - } - waitForShutdown(t, stale) - if stale.closeCount.Load() != 0 { - t.Fatalf("replaced connection was force closed") - } -} - -func TestUTLSPoolInvalidateRetiresOnlyRequestedConnection(t *testing.T) { - transport := newUTLSTransport() - current := newFakePooledHTTP2Conn(true) - old := newFakePooledHTTP2Conn(false) - transport.conns["example:443"] = current - - transport.invalidate("example:443", old) - if transport.conns["example:443"] != current { - t.Fatal("invalidating an old connection removed the replacement") - } - if old.shutdownCount.Load() != 0 { - t.Fatal("connection already removed from the pool was retired again") - } - - transport.invalidate("example:443", current) - waitForShutdown(t, current) - if _, exists := transport.conns["example:443"]; exists { - t.Fatal("invalidated current connection remained in the pool") - } -} - -func TestUTLSPoolCloseIdleUsesBoundedShutdown(t *testing.T) { - transport := newUTLSTransport() - conn := newFakePooledHTTP2Conn(true) - transport.conns["example:443"] = conn - - transport.closeIdleConnections() - if len(transport.conns) != 0 { - t.Fatal("pool was not cleared synchronously") - } - select { - case <-conn.shutdownDone: - case <-time.After(time.Second): - t.Fatal("pooled connection was not shut down") - } - if conn.shutdownCount.Load() != 1 { - t.Fatalf("shutdown count = %d", conn.shutdownCount.Load()) - } -} - -func TestUTLSPoolForcesCloseWhenGracefulShutdownTimesOut(t *testing.T) { - conn := newFakePooledHTTP2Conn(true) - conn.ignoreShutdownCtx = true - - retirePooledHTTP2ConnWithTimeout(conn, 20*time.Millisecond) - waitForShutdown(t, conn) - - deadline := time.After(time.Second) - for conn.closeCount.Load() == 0 { - select { - case <-deadline: - t.Fatal("timed-out graceful shutdown did not force close") - case <-time.After(10 * time.Millisecond): - } - } -} - -func TestUTLSPoolDoesNotPutADeadlineOnActiveStreams(t *testing.T) { - conn := newFakePooledHTTP2Conn(false) - conn.streamsActive = 1 - if timeout := pooledHTTP2RetirementTimeout(conn.State()); timeout != 0 { - t.Fatalf("active connection retirement timeout = %v", timeout) - } -} diff --git a/go_backend/id3_reader.go b/go_backend/id3_reader.go deleted file mode 100644 index 74aa119d..00000000 --- a/go_backend/id3_reader.go +++ /dev/null @@ -1,199 +0,0 @@ -package gobackend - -import ( - "bufio" - "encoding/binary" - "fmt" - "io" -) - -// A malformed frame must not allocate an entire declared tag (up to 256 MiB). -// Artwork is optional; tag-only callers seek past it without allocating it. -const maxID3FrameBytes = 32 << 20 - -func readID3v2WithCover(file io.ReadSeeker, includeCover bool) (*AudioMetadata, []byte, string, error) { - if _, err := file.Seek(0, io.SeekStart); err != nil { - return nil, nil, "", err - } - var header [10]byte - if _, err := io.ReadFull(file, header[:]); err != nil { - return nil, nil, "", err - } - if string(header[:3]) != "ID3" { - return nil, nil, "", fmt.Errorf("no ID3v2 header") - } - version, flags := header[3], header[5] - if version < 2 || version > 4 || header[6]|header[7]|header[8]|header[9] >= 128 { - return nil, nil, "", fmt.Errorf("invalid ID3 version or tag size") - } - size := int64(syncsafeToInt(header[6:10])) - end, err := file.Seek(0, io.SeekEnd) - if err != nil { - return nil, nil, "", err - } - if size > end-10 { - return nil, nil, "", io.ErrUnexpectedEOF - } - if _, err := file.Seek(10, io.SeekStart); err != nil { - return nil, nil, "", err - } - var reader io.Reader = file - // ID3v2.2/2.3 unsynchronization covers the entire tag, including headers; - // decode it as a bounded stream so large pictures still need no allocation. - if flags&0x80 != 0 && version < 4 { - reader = &id3UnsyncReader{source: bufio.NewReader(io.LimitReader(file, size))} - } - if flags&0x40 != 0 { - if version == 2 { - return nil, nil, "", fmt.Errorf("compressed ID3v2.2 tag unsupported") - } - var extended [4]byte - if _, err := io.ReadFull(reader, extended[:]); err != nil { - return nil, nil, "", err - } - length := int64(binary.BigEndian.Uint32(extended[:])) - if version == 4 { - length = int64(syncsafeToInt(extended[:])) - 4 - } - if length < 0 || length > size-4 { - return nil, nil, "", fmt.Errorf("invalid ID3 extended header") - } - if err := skipID3Bytes(reader, length); err != nil { - return nil, nil, "", err - } - size -= length + 4 - } - metadata := &AudioMetadata{} - var cover []byte - var mime string - err = walkID3Frames(reader, size, version, version == 4 && flags&0x80 != 0, func() bool { return includeCover && len(cover) == 0 }, - func(id string, data []byte) { - if id == "APIC" || id == "PIC" { - if len(cover) == 0 { - cover, mime = parseAPICFrame(data, version) - } - } else { - applyID3Frame(metadata, version, id, data) - } - }) - return metadata, cover, mime, err -} - -// walkID3Frames is shared by metadata, cover extraction and combined scanning. -// Only selected frame payloads are read; the enclosing tag bounds every seek. -func walkID3Frames(reader io.Reader, remaining int64, version byte, tagUnsync bool, wantCover func() bool, visit func(string, []byte)) error { - headerLength, idLength := 10, 4 - if version == 2 { - headerLength, idLength = 6, 3 - } - var header [10]byte - for remaining >= int64(headerLength) { - if count, err := io.ReadFull(reader, header[:headerLength]); err != nil { - // A globally unsynchronized tag can have fewer decoded bytes than its - // raw size; exhaustion between frames is normal. - if err == io.EOF { - return nil - } - if err == io.ErrUnexpectedEOF { - padding := true - for _, value := range header[:count] { - if value != 0 { - padding = false - break - } - } - if padding { - return nil - } - } - return err - } - remaining -= int64(headerLength) - if header[0] == 0 || string(header[:3]) == "3DI" { - return nil - } - id := string(header[:idLength]) - var size int64 - switch version { - case 2: - size = int64(header[3])<<16 | int64(header[4])<<8 | int64(header[5]) - case 4: - if header[4]|header[5]|header[6]|header[7] >= 128 { - return fmt.Errorf("invalid ID3 frame size") - } - size = int64(syncsafeToInt(header[4:8])) - default: - size = int64(binary.BigEndian.Uint32(header[4:8])) - } - if size <= 0 || size > remaining { - return fmt.Errorf("invalid ID3 frame bounds") - } - remaining -= size - picture := id == "APIC" || id == "PIC" - wanted := (picture && wantCover != nil && wantCover()) || (!picture && (id[0] == 'T' || id == "COMM" || id == "USLT" || id == "ULT")) - flags := byte(0) - if version != 2 { - flags = header[9] - } - unsupported := (version == 3 && flags&0xc0 != 0) || (version == 4 && flags&0x0c != 0) - if !wanted || unsupported || size > maxID3FrameBytes { - if err := skipID3Bytes(reader, size); err != nil { - return err - } - continue - } - data := make([]byte, int(size)) - if _, err := io.ReadFull(reader, data); err != nil { - return err - } - if version == 3 && flags&0x20 != 0 || version == 4 && flags&0x40 != 0 { - if len(data) < 1 { - continue - } - data = data[1:] - } - if version == 4 && flags&0x01 != 0 { - if len(data) < 4 { - continue - } - data = data[4:] - } - if tagUnsync || version == 4 && flags&0x02 != 0 { - data = removeUnsync(data) - } - visit(id, data) - } - return nil -} - -func skipID3Bytes(reader io.Reader, size int64) error { - if seeker, ok := reader.(io.Seeker); ok { - _, err := seeker.Seek(size, io.SeekCurrent) - return err - } - _, err := io.CopyN(io.Discard, reader, size) - return err -} - -type id3UnsyncReader struct { - source *bufio.Reader - afterFF bool -} - -func (reader *id3UnsyncReader) Read(output []byte) (int, error) { - count := 0 - for count < len(output) { - value, err := reader.source.ReadByte() - if err != nil { - return count, err - } - if reader.afterFF && value == 0 { - reader.afterFF = false - continue - } - reader.afterFF = value == 0xff - output[count] = value - count++ - } - return count, nil -} diff --git a/go_backend/id3_reader_test.go b/go_backend/id3_reader_test.go deleted file mode 100644 index 4849c70f..00000000 --- a/go_backend/id3_reader_test.go +++ /dev/null @@ -1,220 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "os" - "path/filepath" - "testing" -) - -func id3ReaderTestFrame(version byte, id string, payload []byte) []byte { - if version == 2 { - out := []byte{id[0], id[1], id[2], byte(len(payload) >> 16), byte(len(payload) >> 8), byte(len(payload))} - return append(out, payload...) - } - frame := id3v23Frame(id, payload) - if version == 4 { - copy(frame[4:8], syncsafeBytes(len(payload))) - } - return frame -} - -func id3ReaderTestTag(version, flags byte, body []byte) []byte { - header := []byte{'I', 'D', '3', version, 0, flags, 0, 0, 0, 0} - copy(header[6:10], syncsafeBytes(len(body))) - return append(header, body...) -} - -func TestID3CombinedReaderVersionsExtendedHeadersAndUnsync(t *testing.T) { - cover := []byte{0xff, 0xd8, 0xff, 0xe0, 1, 2, 3} - for _, version := range []byte{2, 3, 4} { - for _, extended := range []bool{false, true} { - if version == 2 && extended { - continue - } - for _, unsync := range []bool{false, true} { - t.Run(fmt.Sprintf("v%d/extended=%v/unsync=%v", version, extended, unsync), func(t *testing.T) { - titleID, artistID, pictureID := "TIT2", "TPE1", "APIC" - picture := append([]byte{0, 'i', 'm', 'a', 'g', 'e', '/', 'j', 'p', 'e', 'g', 0, 3, 0}, cover...) - if version == 2 { - titleID, artistID, pictureID = "TT2", "TP1", "PIC" - picture = append([]byte{0, 'J', 'P', 'G', 3, 0}, cover...) - } - frames := append(id3ReaderTestFrame(version, titleID, []byte{0, 'T', 'i', 't', 'l', 'e'}), id3ReaderTestFrame(version, artistID, []byte{0, 'A', 'r', 't', 'i', 's', 't'})...) - var flags byte - if version == 4 && unsync { - picture = bytes.ReplaceAll(picture, []byte{0xff}, []byte{0xff, 0}) - flags |= 0x80 - } - frames = append(frames, id3ReaderTestFrame(version, pictureID, picture)...) - if extended { - flags |= 0x40 - prefix := []byte{0, 0, 0, 6, 0, 0} - if version == 3 { - prefix = append(prefix, 0, 0, 0, 0) - } - frames = append(prefix, frames...) - } - if version < 4 && unsync { - frames = bytes.ReplaceAll(frames, []byte{0xff}, []byte{0xff, 0}) - flags |= 0x80 - } - data := id3ReaderTestTag(version, flags, frames) - metadata, got, mime, err := readID3v2WithCover(bytes.NewReader(data), true) - if err != nil || metadata.Title != "Title" || metadata.Artist != "Artist" || !bytes.Equal(got, cover) || mime != "image/jpeg" { - t.Fatalf("metadata=%+v cover=%x mime=%s err=%v", metadata, got, mime, err) - } - path := filepath.Join(t.TempDir(), "track.mp3") - if err := os.WriteFile(path, data, 0600); err != nil { - t.Fatal(err) - } - standalone, _, err := extractMP3CoverArt(path) - if err != nil || !bytes.Equal(standalone, got) { - t.Fatalf("standalone cover=%x err=%v", standalone, err) - } - }) - } - } - } -} - -type countingID3Reader struct { - *bytes.Reader - bytesRead int -} - -func (reader *countingID3Reader) Read(out []byte) (int, error) { - count, err := reader.Reader.Read(out) - reader.bytesRead += count - return count, err -} - -func TestID3MetadataSkipsLargeCoverPayload(t *testing.T) { - picture := append([]byte{0, 'i', 'm', 'a', 'g', 'e', '/', 'j', 'p', 'e', 'g', 0, 3, 0}, bytes.Repeat([]byte{1}, 8<<20)...) - tag := buildID3v23Tag(id3TextFrame("TIT2", "Song"), id3v23Frame("APIC", picture), id3TextFrame("TSRC", "USRC17607839")) - reader := &countingID3Reader{Reader: bytes.NewReader(tag)} - metadata, cover, _, err := readID3v2WithCover(reader, false) - if err != nil || metadata.ISRC != "USRC17607839" || len(cover) != 0 { - t.Fatalf("metadata=%+v cover=%d err=%v", metadata, len(cover), err) - } - if reader.bytesRead > 1024 { - t.Fatalf("tag-only read consumed %d bytes, including artwork", reader.bytesRead) - } - reader = &countingID3Reader{Reader: bytes.NewReader(tag)} - _, cover, _, err = readID3v2WithCover(reader, true) - if err != nil || len(cover) != 8<<20 { - t.Fatalf("cover=%d err=%v", len(cover), err) - } - if reader.bytesRead != len(tag) { - t.Fatalf("combined read consumed %d bytes for %d-byte tag", reader.bytesRead, len(tag)) - } -} - -func TestID3ReaderRejectsMalformedBoundsAndSkipsUnsupportedFrames(t *testing.T) { - tag := buildID3v23Tag(id3TextFrame("TIT2", "Song")) - truncated := append([]byte{}, tag[:len(tag)-1]...) - if _, _, _, err := readID3v2WithCover(bytes.NewReader(truncated), false); err == nil { - t.Fatal("accepted truncated declared tag") - } - oversized := append([]byte{}, tag...) - binary.BigEndian.PutUint32(oversized[14:18], 1<<30) - if _, _, _, err := readID3v2WithCover(bytes.NewReader(oversized), true); err == nil { - t.Fatal("accepted frame outside tag") - } - compressed := id3TextFrame("TIT2", "unsupported") - compressed[9] = 0x80 - good := id3TextFrame("TIT2", "Song") - metadata, _, _, err := readID3v2WithCover(bytes.NewReader(buildID3v23Tag(compressed, good)), false) - if err != nil || metadata.Title != "Song" { - t.Fatalf("metadata=%+v err=%v", metadata, err) - } -} - -func TestScanMP3CombinedMetadataAndCover(t *testing.T) { - dir := t.TempDir() - cover := []byte{0xff, 0xd8, 0xff, 1, 2, 3} - picture := append([]byte{0, 'i', 'm', 'a', 'g', 'e', '/', 'j', 'p', 'e', 'g', 0, 3, 0}, cover...) - path, _ := writeTestMP3(t, dir, id3TextFrame("TIT2", "Song"), id3TextFrame("TPE1", "Artist"), id3CommentFrame("USLT", "words"), id3v23Frame("APIC", picture)) - cache := filepath.Join(dir, "covers") - for _, pass := range []string{"cold", "warm"} { - result, err := scanMP3FileWithCoverCache(path, &LibraryScanResult{FilePath: path}, "", cache, "key") - if err != nil || result.TrackName != "Song" || !result.HasLyrics || result.CoverPath == "" { - t.Fatalf("%s result=%+v err=%v", pass, result, err) - } - got, err := os.ReadFile(result.CoverPath) - if err != nil || !bytes.Equal(got, cover) { - t.Fatalf("cover=%x err=%v", got, err) - } - } -} - -func BenchmarkID3MetadataLargeCover(b *testing.B) { - picture := append([]byte{0, 'i', 'm', 'a', 'g', 'e', '/', 'j', 'p', 'e', 'g', 0, 3, 0}, bytes.Repeat([]byte{1}, 8<<20)...) - tag := buildID3v23Tag(id3TextFrame("TIT2", "Song"), id3v23Frame("APIC", picture)) - reader := bytes.NewReader(tag) - b.ReportAllocs() - b.ResetTimer() - for b.Loop() { - reader.Seek(0, io.SeekStart) - if _, _, _, err := readID3v2WithCover(reader, false); err != nil { - b.Fatal(err) - } - } -} - -func TestID3PartialTagsAndCoverSurviveMalformedTrailingFrame(t *testing.T) { - cover := []byte{0xff, 0xd8, 0xff, 1, 2, 3} - picture := append([]byte{0, 'i', 'm', 'a', 'g', 'e', '/', 'j', 'p', 'e', 'g', 0, 3, 0}, cover...) - bad := id3TextFrame("TALB", "bad") - binary.BigEndian.PutUint32(bad[4:8], 1<<30) - tag := buildID3v23Tag(id3TextFrame("TIT2", "Song"), id3v23Frame("APIC", picture), bad) - metadata, _, _, err := readID3v2WithCover(bytes.NewReader(tag), true) - if err == nil || metadata.Title != "Song" { - t.Fatalf("partial metadata=%+v err=%v", metadata, err) - } - path := filepath.Join(t.TempDir(), "song.mp3") - if err := os.WriteFile(path, tag, 0600); err != nil { - t.Fatal(err) - } - metadata, err = ReadID3Tags(path) - if err != nil || metadata.Title != "Song" { - t.Fatalf("path metadata=%+v err=%v", metadata, err) - } - got, _, err := extractMP3CoverArt(path) - if err != nil || !bytes.Equal(got, cover) { - t.Fatalf("partial cover=%x err=%v", got, err) - } -} - -func TestID3GlobalUnsyncAllowsShortDecodedPadding(t *testing.T) { - for _, version := range []byte{2, 3} { - pictureID, titleID := "APIC", "TIT2" - if version == 2 { - pictureID, titleID = "PIC", "TT2" - } - body := id3ReaderTestFrame(version, titleID, []byte{0, 'S', 'o', 'n', 'g'}) - body = append(body, id3ReaderTestFrame(version, pictureID, bytes.Repeat([]byte{0xff, 0xe0}, 20))...) - body = append(body, 0, 0, 0) - body = bytes.ReplaceAll(body, []byte{0xff}, []byte{0xff, 0}) - metadata, _, _, err := readID3v2WithCover(bytes.NewReader(id3ReaderTestTag(version, 0x80, body)), false) - if err != nil || metadata.Title != "Song" { - t.Fatalf("v%d metadata=%+v err=%v", version, metadata, err) - } - } -} - -func TestID3CombinedReaderSkipsAdditionalArtwork(t *testing.T) { - picture := append([]byte{0, 'i', 'm', 'a', 'g', 'e', '/', 'j', 'p', 'e', 'g', 0, 3, 0}, bytes.Repeat([]byte{1}, 2<<20)...) - tag := buildID3v23Tag(id3TextFrame("TIT2", "Song"), id3v23Frame("APIC", picture), id3v23Frame("APIC", picture), id3TextFrame("TSRC", "USRC17607839")) - reader := &countingID3Reader{Reader: bytes.NewReader(tag)} - metadata, cover, _, err := readID3v2WithCover(reader, true) - if err != nil || metadata.ISRC != "USRC17607839" || len(cover) != 2<<20 { - t.Fatalf("metadata=%+v cover=%d err=%v", metadata, len(cover), err) - } - if reader.bytesRead > len(picture)+1024 { - t.Fatalf("read %d bytes, including unused second cover", reader.bytesRead) - } -} diff --git a/go_backend/json_util.go b/go_backend/json_util.go deleted file mode 100644 index 43d19219..00000000 --- a/go_backend/json_util.go +++ /dev/null @@ -1,13 +0,0 @@ -package gobackend - -import "encoding/json" - -// marshalJSONString marshals v and returns it as a string, the shape every -// gomobile export returns. -func marshalJSONString(v any) (string, error) { - jsonBytes, err := json.Marshal(v) - if err != nil { - return "", err - } - return string(jsonBytes), nil -} diff --git a/go_backend/library_scan.go b/go_backend/library_scan.go deleted file mode 100644 index 7f92daec..00000000 --- a/go_backend/library_scan.go +++ /dev/null @@ -1,632 +0,0 @@ -package gobackend - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "path/filepath" - "runtime" - "strings" - "sync" - "time" -) - -type LibraryScanResult struct { - ID string `json:"id"` - TrackName string `json:"trackName"` - ArtistName string `json:"artistName"` - AlbumName string `json:"albumName"` - AlbumArtist string `json:"albumArtist,omitempty"` - FilePath string `json:"filePath"` - CoverPath string `json:"coverPath,omitempty"` - ScannedAt string `json:"scannedAt"` - FileModTime int64 `json:"fileModTime,omitempty"` // Unix timestamp in milliseconds - ISRC string `json:"isrc,omitempty"` - TrackNumber int `json:"trackNumber,omitempty"` - TotalTracks int `json:"totalTracks,omitempty"` - DiscNumber int `json:"discNumber,omitempty"` - TotalDiscs int `json:"totalDiscs,omitempty"` - Duration int `json:"duration,omitempty"` - ReleaseDate string `json:"releaseDate,omitempty"` - BitDepth int `json:"bitDepth,omitempty"` - SampleRate int `json:"sampleRate,omitempty"` - Bitrate int `json:"bitrate,omitempty"` // average kbps for both lossless and lossy audio - Genre string `json:"genre,omitempty"` - Composer string `json:"composer,omitempty"` - Label string `json:"label,omitempty"` - Copyright string `json:"copyright,omitempty"` - Comment string `json:"comment,omitempty"` - AlbumType string `json:"albumType,omitempty"` - Explicit bool `json:"explicit,omitempty"` - HasLyrics bool `json:"hasLyrics"` - UPC string `json:"upc,omitempty"` - Format string `json:"format,omitempty"` - MetadataFromFilename bool `json:"metadataFromFilename,omitempty"` -} - -type LibraryScanProgress struct { - TotalFiles int `json:"total_files"` - ScannedFiles int `json:"scanned_files"` - CurrentFile string `json:"current_file"` - ErrorCount int `json:"error_count"` - ProgressPct float64 `json:"progress_pct"` - IsComplete bool `json:"is_complete"` -} - -type IncrementalScanResult struct { - Scanned []LibraryScanResult `json:"scanned"` // New or updated files - DeletedPaths []string `json:"deletedPaths"` // Files that no longer exist - SkippedCount int `json:"skippedCount"` // Files that were unchanged - TotalFiles int `json:"totalFiles"` // Total files in folder -} - -var ( - libraryScanProgress LibraryScanProgress - libraryScanProgressMu sync.RWMutex - libraryScanCancel chan struct{} - libraryScanCancelMu sync.Mutex - libraryCoverCacheDir string - libraryCoverCacheMu sync.RWMutex -) - -var supportedAudioFormats = map[string]bool{ - ".flac": true, - ".m4a": true, - ".mp4": true, - ".aac": true, - ".mp3": true, - ".opus": true, - ".ogg": true, - ".ape": true, - ".wv": true, - ".mpc": true, - ".wav": true, - ".aiff": true, - ".aif": true, - ".cue": true, -} - -type libraryAudioFileInfo struct { - path string - modTime int64 - size int64 -} - -type scannedCueFileInfo struct { - sheet *CueSheet - audioPath string -} - -type libraryScanTask struct { - index int - info libraryAudioFileInfo -} - -type libraryScanTaskResult struct { - index int - path string - results []LibraryScanResult - err error -} - -func isLibraryStagingFile(path string) bool { - name := strings.ToLower(filepath.Base(path)) - if strings.HasSuffix(name, ".partial") { - return true - } - for ext := range supportedAudioFormats { - if strings.HasSuffix(name, ".partial"+ext) { - return true - } - } - return false -} - -func collectLibraryAudioFiles(folderPath string, cancelCh <-chan struct{}) ([]libraryAudioFileInfo, error) { - var files []libraryAudioFileInfo - - err := filepath.WalkDir(folderPath, func(path string, entry os.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("walk library path %s: %w", path, err) - } - - select { - case <-cancelCh: - return fmt.Errorf("scan cancelled") - default: - } - - if entry.IsDir() { - return nil - } - if isLibraryStagingFile(path) { - return nil - } - - ext := strings.ToLower(filepath.Ext(path)) - if !supportedAudioFormats[ext] { - return nil - } - - info, err := entry.Info() - if err != nil { - return fmt.Errorf("read library file info %s: %w", path, err) - } - - files = append(files, libraryAudioFileInfo{ - path: path, - modTime: info.ModTime().UnixMilli(), - size: info.Size(), - }) - return nil - }) - - if err != nil { - return nil, err - } - - return files, nil -} - -func libraryAudioCoverCacheKey(info libraryAudioFileInfo) string { - return fmt.Sprintf("%s|%d|%d", info.path, info.size, info.modTime) -} - -func libraryScanWorkerCount(taskCount int) int { - if taskCount < 16 { - return 1 - } - workers := runtime.NumCPU() - if workers > 4 { - workers = 4 - } - if workers < 2 { - workers = 2 - } - if workers > taskCount { - workers = taskCount - } - return workers -} - -func updateLibraryScanProgress(scannedFiles, totalFiles int, currentPath string) { - libraryScanProgressMu.Lock() - libraryScanProgress.ScannedFiles = scannedFiles - libraryScanProgress.CurrentFile = filepath.Base(currentPath) - if totalFiles > 0 { - libraryScanProgress.ProgressPct = float64(scannedFiles) / float64(totalFiles) * 100 - } - libraryScanProgressMu.Unlock() -} - -func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, cancelCh <-chan struct{}, totalFiles int, completed *int) (map[int][]LibraryScanResult, int, error) { - return scanLibraryAudioTasksParallelWithSink( - tasks, - scanTime, - cancelCh, - totalFiles, - completed, - nil, - ) -} - -func scanLibraryAudioTasksParallelWithSink( - tasks []libraryScanTask, - scanTime string, - cancelCh <-chan struct{}, - totalFiles int, - completed *int, - sink func([]LibraryScanResult) error, -) (map[int][]LibraryScanResult, int, error) { - var resultsByIndex map[int][]LibraryScanResult - if sink == nil { - resultsByIndex = make(map[int][]LibraryScanResult, len(tasks)) - } - if len(tasks) == 0 { - return resultsByIndex, 0, nil - } - - workers := libraryScanWorkerCount(len(tasks)) - if workers <= 1 { - errorCount := 0 - for _, task := range tasks { - select { - case <-cancelCh: - return resultsByIndex, errorCount, fmt.Errorf("scan cancelled") - default: - } - result, err := scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey( - task.info.path, - "", - libraryAudioCoverCacheKey(task.info), - scanTime, - task.info.modTime, - ) - *completed++ - updateLibraryScanProgress(*completed, totalFiles, task.info.path) - if err != nil { - errorCount++ - GoLog("[LibraryScan] Error scanning %s: %v\n", task.info.path, err) - continue - } - results := []LibraryScanResult{*result} - if sink != nil { - if err := sink(results); err != nil { - return resultsByIndex, errorCount, err - } - } else { - resultsByIndex[task.index] = results - } - } - return resultsByIndex, errorCount, nil - } - - taskCh := make(chan libraryScanTask) - resultCh := make(chan libraryScanTaskResult, workers) - var wg sync.WaitGroup - - for i := 0; i < workers; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for task := range taskCh { - select { - case <-cancelCh: - return - default: - } - result, err := scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey( - task.info.path, - "", - libraryAudioCoverCacheKey(task.info), - scanTime, - task.info.modTime, - ) - taskResult := libraryScanTaskResult{ - index: task.index, - path: task.info.path, - err: err, - } - if err == nil && result != nil { - taskResult.results = []LibraryScanResult{*result} - } - select { - case <-cancelCh: - return - case resultCh <- taskResult: - } - } - }() - } - - go func() { - defer close(taskCh) - for _, task := range tasks { - select { - case <-cancelCh: - return - case taskCh <- task: - } - } - }() - - go func() { - wg.Wait() - close(resultCh) - }() - - errorCount := 0 - var sinkErr error - for taskResult := range resultCh { - *completed++ - updateLibraryScanProgress(*completed, totalFiles, taskResult.path) - if taskResult.err != nil { - errorCount++ - GoLog("[LibraryScan] Error scanning %s: %v\n", taskResult.path, taskResult.err) - continue - } - if sink != nil { - if sinkErr == nil { - sinkErr = sink(taskResult.results) - } - } else { - resultsByIndex[taskResult.index] = taskResult.results - } - } - if sinkErr != nil { - return resultsByIndex, errorCount, sinkErr - } - - select { - case <-cancelCh: - return resultsByIndex, errorCount, fmt.Errorf("scan cancelled") - default: - } - return resultsByIndex, errorCount, nil -} - -func SetLibraryCoverCacheDir(cacheDir string) { - libraryCoverCacheMu.Lock() - libraryCoverCacheDir = cacheDir - libraryCoverCacheMu.Unlock() -} - -func scanLibraryFolderWithSink( - folderPath string, - sink func(LibraryScanResult) error, - preserveOrder bool, -) (int, error) { - if folderPath == "" { - return 0, fmt.Errorf("folder path is empty") - } - - info, err := os.Stat(folderPath) - if err != nil { - return 0, fmt.Errorf("folder not found: %w", err) - } - if !info.IsDir() { - return 0, fmt.Errorf("path is not a folder: %s", folderPath) - } - - libraryScanProgressMu.Lock() - libraryScanProgress = LibraryScanProgress{} - libraryScanProgressMu.Unlock() - - libraryScanCancelMu.Lock() - if libraryScanCancel != nil { - close(libraryScanCancel) - } - libraryScanCancel = make(chan struct{}) - cancelCh := libraryScanCancel - libraryScanCancelMu.Unlock() - - audioFileInfos, err := collectLibraryAudioFiles(folderPath, cancelCh) - if err != nil { - return 0, err - } - - totalFiles := len(audioFileInfos) - libraryScanProgressMu.Lock() - libraryScanProgress.TotalFiles = totalFiles - libraryScanProgressMu.Unlock() - - if totalFiles == 0 { - libraryScanProgressMu.Lock() - libraryScanProgress.IsComplete = true - libraryScanProgress.ProgressPct = 100 - libraryScanProgressMu.Unlock() - return 0, nil - } - - GoLog("[LibraryScan] Found %d audio files to scan\n", totalFiles) - - scanTime := time.Now().UTC().Format(time.RFC3339) - errorCount := 0 - emittedCount := 0 - emitResults := func(results []LibraryScanResult) error { - for i := range results { - if err := sink(results[i]); err != nil { - return err - } - emittedCount++ - } - return nil - } - - cueReferencedAudioFiles := make(map[string]bool) - parsedCueFiles := make(map[string]scannedCueFileInfo) - for _, fileInfo := range audioFileInfos { - filePath := fileInfo.path - if strings.ToLower(filepath.Ext(filePath)) != ".cue" { - continue - } - sheet, parseErr := ParseCueFile(filePath) - if parseErr == nil && sheet.FileName != "" { - audioPath := ResolveCueAudioPath(filePath, sheet.FileName) - if audioPath != "" { - parsedCueFiles[filePath] = scannedCueFileInfo{ - sheet: sheet, - audioPath: audioPath, - } - cueReferencedAudioFiles[audioPath] = true - } - } - } - - audioTasks := make([]libraryScanTask, 0, totalFiles) - var orderedResults map[int][]LibraryScanResult - if preserveOrder { - orderedResults = make(map[int][]LibraryScanResult, totalFiles) - } - completedFiles := 0 - for i, fileInfo := range audioFileInfos { - filePath := fileInfo.path - select { - case <-cancelCh: - return emittedCount, fmt.Errorf("scan cancelled") - default: - } - - ext := strings.ToLower(filepath.Ext(filePath)) - if ext == ".cue" { - var cueResults []LibraryScanResult - cueInfo, ok := parsedCueFiles[filePath] - if ok { - cueResults, err = scanCueSheetForLibrary( - filePath, - cueInfo.sheet, - cueInfo.audioPath, - "", - fileInfo.modTime, - "", - scanTime, - ) - } else { - cueResults, err = ScanCueFileForLibrary(filePath, scanTime) - } - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, filePath) - if err != nil { - errorCount++ - GoLog("[LibraryScan] Error scanning cue %s: %v\n", filePath, err) - continue - } - if preserveOrder { - orderedResults[i] = cueResults - } else if err := emitResults(cueResults); err != nil { - return emittedCount, fmt.Errorf("write scan result: %w", err) - } - GoLog("[LibraryScan] CUE sheet %s: %d tracks\n", filepath.Base(filePath), len(cueResults)) - continue - } - - if cueReferencedAudioFiles[filePath] { - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, filePath) - GoLog("[LibraryScan] Skipping %s (referenced by .cue sheet)\n", filepath.Base(filePath)) - continue - } - - audioTasks = append(audioTasks, libraryScanTask{index: i, info: fileInfo}) - } - - var audioSink func([]LibraryScanResult) error - if !preserveOrder { - audioSink = emitResults - } - audioResults, audioErrors, err := scanLibraryAudioTasksParallelWithSink( - audioTasks, - scanTime, - cancelCh, - totalFiles, - &completedFiles, - audioSink, - ) - if err != nil { - return emittedCount, err - } - errorCount += audioErrors - if preserveOrder { - for index, results := range audioResults { - orderedResults[index] = results - } - for i := range audioFileInfos { - if err := emitResults(orderedResults[i]); err != nil { - return emittedCount, fmt.Errorf("write scan result: %w", err) - } - } - } - - libraryScanProgressMu.Lock() - libraryScanProgress.ErrorCount = errorCount - libraryScanProgress.IsComplete = true - libraryScanProgress.ScannedFiles = totalFiles - libraryScanProgress.ProgressPct = 100 - libraryScanProgressMu.Unlock() - - GoLog("[LibraryScan] Scan complete: %d tracks found, %d errors\n", emittedCount, errorCount) - return emittedCount, nil -} - -func ScanLibraryFolder(folderPath string) (string, error) { - results := make([]LibraryScanResult, 0) - _, err := scanLibraryFolderWithSink(folderPath, func(result LibraryScanResult) error { - results = append(results, result) - return nil - }, true) - if err != nil { - return "[]", err - } - - jsonBytes, err := json.Marshal(results) - if err != nil { - return "[]", fmt.Errorf("failed to marshal results: %w", err) - } - - return string(jsonBytes), nil -} - -// ScanLibraryFolderToNDJSONFile writes one JSON object per line so mobile -// clients can decode and ingest bounded batches instead of materializing a -// full-library JSON array in both the Go and Dart heaps. -func ScanLibraryFolderToNDJSONFile(folderPath, outputPath string) (int, error) { - if outputPath == "" { - return 0, fmt.Errorf("output path is empty") - } - file, err := os.Create(outputPath) - if err != nil { - return 0, fmt.Errorf("create scan output: %w", err) - } - removeOnError := true - defer func() { - _ = file.Close() - if removeOnError { - _ = os.Remove(outputPath) - } - }() - - writer := bufio.NewWriterSize(file, 64*1024) - encoder := json.NewEncoder(writer) - count, err := scanLibraryFolderWithSink(folderPath, func(result LibraryScanResult) error { - return encoder.Encode(result) - }, false) - if err != nil { - return count, err - } - if err := writer.Flush(); err != nil { - return count, fmt.Errorf("flush scan output: %w", err) - } - if err := file.Close(); err != nil { - return count, fmt.Errorf("close scan output: %w", err) - } - removeOnError = false - return count, nil -} - -func GetLibraryScanProgress() string { - libraryScanProgressMu.RLock() - defer libraryScanProgressMu.RUnlock() - - jsonBytes, _ := json.Marshal(libraryScanProgress) - return string(jsonBytes) -} - -func CancelLibraryScan() { - libraryScanCancelMu.Lock() - defer libraryScanCancelMu.Unlock() - - if libraryScanCancel != nil { - close(libraryScanCancel) - libraryScanCancel = nil - } -} - -func ReadAudioMetadata(filePath string) (string, error) { - return ReadAudioMetadataWithDisplayName(filePath, "") -} - -func ReadAudioMetadataWithDisplayName(filePath, displayNameHint string) (string, error) { - return ReadAudioMetadataWithDisplayNameAndCoverCacheKey(filePath, displayNameHint, "") -} - -func ReadAudioMetadataWithDisplayNameAndCoverCacheKey(filePath, displayNameHint, coverCacheKey string) (string, error) { - scanTime := time.Now().UTC().Format(time.RFC3339) - result, err := scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey( - filePath, - displayNameHint, - coverCacheKey, - scanTime, - 0, - ) - if err != nil { - return "", err - } - - jsonBytes, err := json.Marshal(result) - if err != nil { - return "", fmt.Errorf("failed to marshal result: %w", err) - } - - return string(jsonBytes), nil -} diff --git a/go_backend/library_scan_formats.go b/go_backend/library_scan_formats.go deleted file mode 100644 index 76814781..00000000 --- a/go_backend/library_scan_formats.go +++ /dev/null @@ -1,400 +0,0 @@ -package gobackend - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - -func scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(filePath, displayNameHint, coverCacheKey, scanTime string, knownModTime int64) (*LibraryScanResult, error) { - ext := resolveLibraryAudioExt(filePath, displayNameHint) - - result := &LibraryScanResult{ - ID: generateLibraryID(filePath), - FilePath: filePath, - ScannedAt: scanTime, - Format: strings.TrimPrefix(ext, "."), - } - - if knownModTime > 0 { - result.FileModTime = knownModTime - } else if info, err := os.Stat(filePath); err == nil { - result.FileModTime = info.ModTime().UnixMilli() - } - - libraryCoverCacheMu.RLock() - coverCacheDir := libraryCoverCacheDir - libraryCoverCacheMu.RUnlock() - var scanned *LibraryScanResult - var scanErr error - switch ext { - case ".flac": - scanned, scanErr = scanFLACFileWithCoverCache(filePath, result, displayNameHint, coverCacheDir, coverCacheKey) - case ".m4a", ".mp4", ".aac": - scanned, scanErr = scanM4AFileWithCoverCache(filePath, result, displayNameHint, coverCacheDir, coverCacheKey) - case ".mp3": - scanned, scanErr = scanMP3FileWithCoverCache(filePath, result, displayNameHint, coverCacheDir, coverCacheKey) - default: - if coverCacheDir != "" { - coverPath, err := SaveCoverToCacheWithHintAndKey( - filePath, - displayNameHint, - coverCacheDir, - coverCacheKey, - ) - if err == nil && coverPath != "" { - result.CoverPath = coverPath - } - } - - switch ext { - case ".opus", ".ogg": - scanned, scanErr = scanOggFile(filePath, result, displayNameHint) - case ".ape", ".wv", ".mpc": - scanned, scanErr = scanAPEFile(filePath, result, displayNameHint) - case ".wav": - scanned, scanErr = scanWAVFile(filePath, result, displayNameHint) - case ".aiff", ".aif", ".aifc": - scanned, scanErr = scanAIFFFile(filePath, result, displayNameHint) - default: - scanned, scanErr = scanFromFilename(filePath, displayNameHint, result) - } - } - if scanErr != nil || scanned == nil { - return scanned, scanErr - } - if !scanned.HasLyrics { - if sidecar, err := extractLyricsFromSidecarLRC(filePath); err == nil { - scanned.HasLyrics = rawLyricsHasUsableContent(sidecar) - } - } - return scanned, nil -} - -func embeddedCoverMIME(data []byte) string { - if len(data) >= 8 && - data[0] == 0x89 && - data[1] == 0x50 && - data[2] == 0x4e && - data[3] == 0x47 { - return "image/png" - } - return "image/jpeg" -} - -func cacheScannedCover(filePath, cacheDir, coverCacheKey string, coverData []byte) string { - if cacheDir == "" || len(coverData) == 0 { - return "" - } - cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) - path, err := saveLibraryCoverDataToCache(cacheDir, cacheKey, coverData, embeddedCoverMIME(coverData)) - if err != nil { - return "" - } - return path -} - -func resolveLibraryAudioExt(filePath, displayNameHint string) string { - ext := strings.ToLower(filepath.Ext(filePath)) - if ext != "" { - return ext - } - return strings.ToLower(filepath.Ext(displayNameHint)) -} - -func libraryDisplayNameOrPath(filePath, displayNameHint string) string { - if displayNameHint != "" { - return displayNameHint - } - return filePath -} - -func applyDefaultLibraryMetadata(filePath, displayNameHint string, result *LibraryScanResult) { - nameSource := libraryDisplayNameOrPath(filePath, displayNameHint) - if result.TrackName == "" { - result.TrackName = strings.TrimSuffix(filepath.Base(nameSource), filepath.Ext(nameSource)) - } - if result.ArtistName == "" { - result.ArtistName = "Unknown Artist" - } - if result.AlbumName == "" { - result.AlbumName = "Unknown Album" - } -} - -func scanFLACFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - return scanFLACFileWithCoverCache(filePath, result, displayNameHint, "", "") -} - -func scanFLACFileWithCoverCache(filePath string, result *LibraryScanResult, displayNameHint, coverCacheDir, coverCacheKey string) (*LibraryScanResult, error) { - f, err := parseFlacFile(filePath) - if err != nil { - return scanFromFilename(filePath, displayNameHint, result) - } - defer f.Close() - metadata := metadataFromParsedFlac(f) - - result.TrackName = metadata.Title - result.ArtistName = metadata.Artist - result.AlbumName = metadata.Album - result.AlbumArtist = metadata.AlbumArtist - result.ISRC = metadata.ISRC - result.TrackNumber = metadata.TrackNumber - result.TotalTracks = metadata.TotalTracks - result.DiscNumber = metadata.DiscNumber - result.TotalDiscs = metadata.TotalDiscs - result.ReleaseDate = metadata.Date - result.Genre = metadata.Genre - result.Composer = metadata.Composer - result.Label = metadata.Label - result.Copyright = metadata.Copyright - result.Comment = metadata.Comment - result.AlbumType = metadata.AlbumType - result.Explicit = metadata.Explicit - result.HasLyrics = rawLyricsHasUsableContent(metadata.Lyrics) - result.UPC = metadata.UPC - - quality, err := audioQualityFromParsedFlac(f) - if err == nil { - result.BitDepth = quality.BitDepth - result.SampleRate = quality.SampleRate - if quality.SampleRate > 0 && quality.TotalSamples > 0 { - durationSeconds := float64(quality.TotalSamples) / float64(quality.SampleRate) - result.Duration = int(durationSeconds) - if info, statErr := os.Stat(filePath); statErr == nil && info.Size() > 0 { - result.Bitrate = int(float64(info.Size()) * 8 / durationSeconds / 1000) - } - } - } - if coverCacheDir != "" { - cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) - if existing := existingLibraryCoverCachePath(coverCacheDir, cacheKey); existing != "" { - result.CoverPath = existing - } else if coverData, coverErr := coverArtFromParsedFlac(f); coverErr == nil { - result.CoverPath = cacheScannedCover(filePath, coverCacheDir, cacheKey, coverData) - } - } - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - - return result, nil -} - -func scanM4AFileWithCoverCache(filePath string, result *LibraryScanResult, displayNameHint, coverCacheDir, coverCacheKey string) (*LibraryScanResult, error) { - f, err := os.Open(filePath) - if err != nil { - return scanFromFilename(filePath, displayNameHint, result) - } - defer f.Close() - info, err := f.Stat() - if err != nil { - return scanFromFilename(filePath, displayNameHint, result) - } - fileSize := info.Size() - - var metadata *AudioMetadata - ilst, ilstErr := findM4AIlstAtom(f, fileSize) - if ilstErr == nil { - metadata, err = readM4ATagsFromIlst(f, fileSize, ilst) - } else { - err = ilstErr - } - if err != nil { - GoLog("[LibraryScan] M4A read error for %s: %v\n", filePath, err) - } - - if metadata != nil { - applyAudioMetadataToScan(metadata, result) - } - - quality, err := m4aQualityFromFile(f, fileSize) - if err == nil { - result.BitDepth = quality.BitDepth - result.SampleRate = quality.SampleRate - result.Duration = quality.Duration - if quality.Bitrate > 0 { - result.Bitrate = quality.Bitrate - } - if format := libraryFormatForM4ACodec(quality.Codec); format != "" { - result.Format = format - } - } - if coverCacheDir != "" { - cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) - if existing := existingLibraryCoverCachePath(coverCacheDir, cacheKey); existing != "" { - result.CoverPath = existing - } else if ilstErr == nil { - if coverData, coverErr := extractCoverFromM4AIlst(f, fileSize, ilst); coverErr == nil { - result.CoverPath = cacheScannedCover(filePath, coverCacheDir, cacheKey, coverData) - } - } - } - - if metadata == nil { - return scanFromFilename(filePath, displayNameHint, result) - } - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - return result, nil -} - -func libraryFormatForM4ACodec(codec string) string { - switch strings.ToLower(strings.TrimSpace(codec)) { - case "flac": - return "flac" - case "alac": - return "alac" - case "eac3", "ec-3": - return "eac3" - case "ac3", "ac-3": - return "ac3" - case "ac4", "ac-4": - return "ac4" - case "opus": - return "opus" - case "aac", "mp4a": - return "m4a" - default: - return "" - } -} - -func isLosslessLibraryFormat(format string) bool { - switch strings.ToLower(strings.TrimSpace(format)) { - case "flac", "alac", "wav", "aiff", "aif", "aifc": - return true - default: - return false - } -} - -func scanMP3FileWithCoverCache(filePath string, result *LibraryScanResult, displayNameHint, cacheDir, cacheKey string) (*LibraryScanResult, error) { - wantCover := cacheDir != "" - if wantCover { - cacheKey = resolveLibraryCoverCacheKey(filePath, cacheKey) - result.CoverPath = existingLibraryCoverCachePath(cacheDir, cacheKey) - wantCover = result.CoverPath == "" - } - metadata, cover, mime, err := readID3TagsAndCover(filePath, wantCover) - if wantCover && len(cover) > 0 { - result.CoverPath, _ = saveLibraryCoverDataToCache(cacheDir, cacheKey, cover, mime) - } - if err != nil { - GoLog("[LibraryScan] ID3 read error for %s: %v\n", filePath, err) - return scanFromFilename(filePath, displayNameHint, result) - } - - applyAudioMetadataToScan(metadata, result) - - quality, err := GetMP3Quality(filePath) - if err == nil { - result.SampleRate = quality.SampleRate - result.BitDepth = quality.BitDepth // 0 for lossy - result.Duration = quality.Duration - if quality.Bitrate > 0 { - result.Bitrate = quality.Bitrate / 1000 // convert bps to kbps - } - } - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - - return result, nil -} - -func scanOggFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - metadata, err := ReadOggVorbisComments(filePath) - if err != nil { - GoLog("[LibraryScan] Ogg/Opus read error for %s: %v\n", filePath, err) - return scanFromFilename(filePath, displayNameHint, result) - } - - applyAudioMetadataToScan(metadata, result) - - quality, err := GetOggQuality(filePath) - if err == nil { - result.SampleRate = quality.SampleRate - result.BitDepth = quality.BitDepth // 0 for lossy - result.Duration = quality.Duration - if quality.Bitrate > 0 { - result.Bitrate = quality.Bitrate / 1000 // convert bps to kbps - } - } - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - - return result, nil -} - -func scanAPEFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - tag, err := ReadAPETags(filePath) - if err != nil { - GoLog("[LibraryScan] APE tag read error for %s: %v\n", filePath, err) - return scanFromFilename(filePath, displayNameHint, result) - } - - metadata := APETagToAudioMetadata(tag) - if metadata == nil { - return scanFromFilename(filePath, displayNameHint, result) - } - - applyAudioMetadataToScan(metadata, result) - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - - return result, nil -} - -func scanFromFilename(filePath, displayNameHint string, result *LibraryScanResult) (*LibraryScanResult, error) { - result.MetadataFromFilename = true - nameSource := libraryDisplayNameOrPath(filePath, displayNameHint) - filename := strings.TrimSuffix(filepath.Base(nameSource), filepath.Ext(nameSource)) - - parts := strings.SplitN(filename, " - ", 2) - if len(parts) == 2 { - if len(parts[0]) <= 3 && isNumeric(parts[0]) { - result.TrackName = parts[1] - result.ArtistName = "Unknown Artist" - } else { - result.ArtistName = parts[0] - result.TrackName = parts[1] - } - } else { - if len(filename) > 3 && isNumeric(filename[:2]) { - title := strings.TrimLeft(filename[2:], " .-") - result.TrackName = title - } else { - result.TrackName = filename - } - result.ArtistName = "Unknown Artist" - } - - dir := filepath.Dir(filePath) - result.AlbumName = filepath.Base(dir) - if result.AlbumName == "." || result.AlbumName == "" || result.AlbumName == "fd" || result.AlbumName == "self" { - result.AlbumName = "Unknown Album" - } - - return result, nil -} - -func isNumeric(s string) bool { - for _, c := range s { - if c < '0' || c > '9' { - return false - } - } - return len(s) > 0 -} - -func generateLibraryID(filePath string) string { - return fmt.Sprintf("lib_%x", hashString(filePath)) -} - -func hashString(s string) uint32 { - var hash uint32 = 5381 - for _, c := range s { - hash = ((hash << 5) + hash) + uint32(c) - } - return hash -} diff --git a/go_backend/library_scan_incremental.go b/go_backend/library_scan_incremental.go deleted file mode 100644 index af4d4fb1..00000000 --- a/go_backend/library_scan_incremental.go +++ /dev/null @@ -1,291 +0,0 @@ -package gobackend - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "time" -) - -func loadExistingFilesSnapshot(snapshotPath string) (map[string]int64, error) { - existingFiles := make(map[string]int64) - if snapshotPath == "" { - return existingFiles, nil - } - - file, err := os.Open(snapshotPath) - if err != nil { - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if line == "" { - continue - } - parts := strings.SplitN(line, "\t", 2) - if len(parts) != 2 { - continue - } - modTime, err := strconv.ParseInt(parts[0], 10, 64) - if err != nil { - continue - } - existingFiles[parts[1]] = modTime - } - - if err := scanner.Err(); err != nil { - return nil, err - } - - return existingFiles, nil -} - -func scanLibraryFolderIncrementalWithExistingFiles(folderPath string, existingFiles map[string]int64) (string, error) { - if folderPath == "" { - return "{}", fmt.Errorf("folder path is empty") - } - - info, err := os.Stat(folderPath) - if err != nil { - return "{}", fmt.Errorf("folder not found: %w", err) - } - if !info.IsDir() { - return "{}", fmt.Errorf("path is not a folder: %s", folderPath) - } - - GoLog("[LibraryScan] Incremental scan starting, %d existing files in database\n", len(existingFiles)) - - libraryScanProgressMu.Lock() - libraryScanProgress = LibraryScanProgress{} - libraryScanProgressMu.Unlock() - - libraryScanCancelMu.Lock() - if libraryScanCancel != nil { - close(libraryScanCancel) - } - libraryScanCancel = make(chan struct{}) - cancelCh := libraryScanCancel - libraryScanCancelMu.Unlock() - - currentFiles, err := collectLibraryAudioFiles(folderPath, cancelCh) - if err != nil { - return "{}", err - } - currentPathSet := make(map[string]bool, len(currentFiles)) - for _, fileInfo := range currentFiles { - currentPathSet[fileInfo.path] = true - } - - totalFiles := len(currentFiles) - libraryScanProgressMu.Lock() - libraryScanProgress.TotalFiles = totalFiles - libraryScanProgressMu.Unlock() - - var filesToScan []libraryAudioFileInfo - skippedCount := 0 - existingCueTrackModTimes := make(map[string]int64) - for existingPath, modTime := range existingFiles { - if idx := strings.LastIndex(existingPath, "#track"); idx > 0 { - baseCuePath := existingPath[:idx] - if _, exists := existingCueTrackModTimes[baseCuePath]; !exists { - existingCueTrackModTimes[baseCuePath] = modTime - } - } - } - - for _, f := range currentFiles { - existingModTime, exists := existingFiles[f.path] - if !exists { - if strings.ToLower(filepath.Ext(f.path)) == ".cue" { - if cueTrackModTime, hasCueTracks := existingCueTrackModTimes[f.path]; hasCueTracks { - if f.modTime == cueTrackModTime { - skippedCount++ - } else { - filesToScan = append(filesToScan, f) - } - continue - } - } - filesToScan = append(filesToScan, f) - } else if f.modTime != existingModTime { - filesToScan = append(filesToScan, f) - } else { - skippedCount++ - } - } - - var deletedPaths []string - for existingPath := range existingFiles { - if idx := strings.LastIndex(existingPath, "#track"); idx > 0 { - baseCuePath := existingPath[:idx] - if currentPathSet[baseCuePath] { - continue - } - deletedPaths = append(deletedPaths, existingPath) - } else if !currentPathSet[existingPath] { - deletedPaths = append(deletedPaths, existingPath) - } - } - - GoLog("[LibraryScan] Incremental: %d to scan, %d skipped, %d deleted\n", - len(filesToScan), skippedCount, len(deletedPaths)) - - if len(filesToScan) == 0 { - libraryScanProgressMu.Lock() - libraryScanProgress.ScannedFiles = totalFiles - libraryScanProgress.IsComplete = true - libraryScanProgress.ProgressPct = 100 - libraryScanProgressMu.Unlock() - - result := IncrementalScanResult{ - Scanned: []LibraryScanResult{}, - DeletedPaths: deletedPaths, - SkippedCount: skippedCount, - TotalFiles: totalFiles, - } - jsonBytes, _ := json.Marshal(result) - return string(jsonBytes), nil - } - - results := make([]LibraryScanResult, 0, len(filesToScan)) - scanTime := time.Now().UTC().Format(time.RFC3339) - errorCount := 0 - - cueReferencedAudioFilesInc := make(map[string]bool) - parsedCueFiles := make(map[string]scannedCueFileInfo) - for _, f := range filesToScan { - ext := strings.ToLower(filepath.Ext(f.path)) - if ext == ".cue" { - sheet, err := ParseCueFile(f.path) - if err == nil && sheet.FileName != "" { - audioPath := ResolveCueAudioPath(f.path, sheet.FileName) - if audioPath != "" { - parsedCueFiles[f.path] = scannedCueFileInfo{ - sheet: sheet, - audioPath: audioPath, - } - cueReferencedAudioFilesInc[audioPath] = true - } - } - } - } - - resultsByIndex := make(map[int][]LibraryScanResult, len(filesToScan)) - audioTasks := make([]libraryScanTask, 0, len(filesToScan)) - completedFiles := skippedCount - - for i, f := range filesToScan { - select { - case <-cancelCh: - return "{}", fmt.Errorf("scan cancelled") - default: - } - - ext := strings.ToLower(filepath.Ext(f.path)) - - if ext == ".cue" { - var cueResults []LibraryScanResult - cueInfo, ok := parsedCueFiles[f.path] - if ok { - cueResults, err = scanCueSheetForLibrary( - f.path, - cueInfo.sheet, - cueInfo.audioPath, - "", - f.modTime, - "", - scanTime, - ) - } else { - cueResults, err = ScanCueFileForLibrary(f.path, scanTime) - } - if err != nil { - errorCount++ - GoLog("[LibraryScan] Error scanning cue %s: %v\n", f.path, err) - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, f.path) - continue - } - resultsByIndex[i] = cueResults - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, f.path) - continue - } - - if cueReferencedAudioFilesInc[f.path] { - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, f.path) - continue - } - - audioTasks = append(audioTasks, libraryScanTask{index: i, info: f}) - } - - audioResults, audioErrors, err := scanLibraryAudioTasksParallel( - audioTasks, - scanTime, - cancelCh, - totalFiles, - &completedFiles, - ) - if err != nil { - return "{}", err - } - errorCount += audioErrors - for index, scanResults := range audioResults { - resultsByIndex[index] = scanResults - } - - for i := range filesToScan { - results = append(results, resultsByIndex[i]...) - } - - libraryScanProgressMu.Lock() - libraryScanProgress.ErrorCount = errorCount - libraryScanProgress.IsComplete = true - libraryScanProgress.ScannedFiles = totalFiles - libraryScanProgress.ProgressPct = 100 - libraryScanProgressMu.Unlock() - - GoLog("[LibraryScan] Incremental scan complete: %d scanned, %d skipped, %d deleted, %d errors\n", - len(results), skippedCount, len(deletedPaths), errorCount) - - scanResult := IncrementalScanResult{ - Scanned: results, - DeletedPaths: deletedPaths, - SkippedCount: skippedCount, - TotalFiles: totalFiles, - } - - jsonBytes, err := json.Marshal(scanResult) - if err != nil { - return "{}", fmt.Errorf("failed to marshal results: %w", err) - } - - return string(jsonBytes), nil -} - -func ScanLibraryFolderIncremental(folderPath, existingFilesJSON string) (string, error) { - existingFiles := make(map[string]int64) - if existingFilesJSON != "" && existingFilesJSON != "{}" { - if err := json.Unmarshal([]byte(existingFilesJSON), &existingFiles); err != nil { - GoLog("[LibraryScan] Warning: failed to parse existing files JSON: %v\n", err) - } - } - return scanLibraryFolderIncrementalWithExistingFiles(folderPath, existingFiles) -} - -func ScanLibraryFolderIncrementalFromSnapshot(folderPath, snapshotPath string) (string, error) { - existingFiles, err := loadExistingFilesSnapshot(snapshotPath) - if err != nil { - return "{}", fmt.Errorf("failed to load incremental snapshot: %w", err) - } - return scanLibraryFolderIncrementalWithExistingFiles(folderPath, existingFiles) -} diff --git a/go_backend/library_scan_single_pass_test.go b/go_backend/library_scan_single_pass_test.go deleted file mode 100644 index 62ea8af0..00000000 --- a/go_backend/library_scan_single_pass_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package gobackend - -import ( - "bytes" - "os" - "path/filepath" - "testing" - - "github.com/go-flac/flacpicture/v2" - "github.com/go-flac/flacvorbis/v2" - flac "github.com/go-flac/go-flac/v2" -) - -func writeSinglePassTestFlac(t testing.TB, path string, cover []byte) { - t.Helper() - streamInfo := make([]byte, 34) - const sampleRate = 44100 - const bitDepth = 16 - const totalSamples = int64(441000) - streamInfo[10] = byte((sampleRate >> 12) & 0xff) - streamInfo[11] = byte((sampleRate >> 4) & 0xff) - streamInfo[12] = byte((sampleRate&0x0f)<<4 | ((bitDepth - 1) >> 4)) - streamInfo[13] = byte(((bitDepth-1)&0x0f)<<4) | byte(totalSamples>>32) - streamInfo[14] = byte((totalSamples >> 24) & 0xff) - streamInfo[15] = byte((totalSamples >> 16) & 0xff) - streamInfo[16] = byte((totalSamples >> 8) & 0xff) - streamInfo[17] = byte(totalSamples & 0xff) - streamBlock := flac.MetaDataBlock{Type: flac.StreamInfo, Data: streamInfo} - - comments := flacvorbis.New() - setComment(comments, "TITLE", "Single Pass") - setComment(comments, "ARTIST", "Artist") - setComment(comments, "ITUNESADVISORY", "1") - setComment(comments, "RELEASETYPE", "album") - setComment(comments, "BARCODE", "4006381333931") - commentBlock := comments.Marshal() - pictureBlock := (&flacpicture.MetadataBlockPicture{ - PictureType: flacpicture.PictureTypeFrontCover, - MIME: "image/png", - Width: 1, - Height: 1, - ColorDepth: 24, - ImageData: cover, - }).Marshal() - - data := append([]byte("fLaC"), streamBlock.Marshal(false)...) - data = append(data, commentBlock.Marshal(false)...) - data = append(data, pictureBlock.Marshal(true)...) - data = append(data, 0xff, 0xf8) - if err := os.WriteFile(path, data, 0o600); err != nil { - t.Fatal(err) - } -} - -func TestScanFLACSinglePassReadsMetadataQualityAndCover(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "track.flac") - cover := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3} - writeSinglePassTestFlac(t, path, cover) - - result := &LibraryScanResult{FilePath: path, Format: "flac"} - result, err := scanFLACFileWithCoverCache(path, result, "", filepath.Join(dir, "covers"), "stable-key") - if err != nil { - t.Fatal(err) - } - if result.TrackName != "Single Pass" || result.ArtistName != "Artist" || result.SampleRate != 44100 || result.BitDepth != 16 || result.Duration != 10 { - t.Fatalf("scan result = %#v", result) - } - if !result.Explicit || result.AlbumType != "album" || result.UPC != "4006381333931" { - t.Fatalf("release identity = %#v", result) - } - if result.CoverPath == "" { - t.Fatal("cover was not cached") - } - gotCover, err := os.ReadFile(result.CoverPath) - if err != nil || !bytes.Equal(gotCover, cover) { - t.Fatalf("cached cover = %x, %v", gotCover, err) - } -} - -func TestScanFLACPersistsAverageBitrate(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "bitrate.flac") - writeSinglePassTestFlac(t, path, nil) - - f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) - if err != nil { - t.Fatal(err) - } - if _, err := f.Write(make([]byte, 2_000_000)); err != nil { - f.Close() - t.Fatal(err) - } - if err := f.Close(); err != nil { - t.Fatal(err) - } - - result := &LibraryScanResult{FilePath: path, Format: "flac"} - result, err = scanFLACFile(path, result, "") - if err != nil { - t.Fatal(err) - } - if result.Bitrate < 1_590 || result.Bitrate > 1_610 { - t.Fatalf("average bitrate = %d kbps", result.Bitrate) - } -} - -func TestScanM4ASingleOpenReadsMetadataAndCover(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "track.m4a") - cover := []byte{0xff, 0xd8, 0xff, 1, 2, 3} - ilst := append(buildM4ATextAtom("\xa9nam", "Single Open"), buildM4ACoverAtom(cover)...) - data, _ := buildTestM4A(t, ilst, []byte("audio")) - if err := os.WriteFile(path, data, 0o600); err != nil { - t.Fatal(err) - } - - result := &LibraryScanResult{FilePath: path, Format: "m4a"} - result, err := scanM4AFileWithCoverCache(path, result, "", filepath.Join(dir, "covers"), "stable-key") - if err != nil { - t.Fatal(err) - } - if result.TrackName != "Single Open" || result.CoverPath == "" { - t.Fatalf("scan result = %#v", result) - } - gotCover, err := os.ReadFile(result.CoverPath) - if err != nil || !bytes.Equal(gotCover, cover) { - t.Fatalf("cached cover = %x, %v", gotCover, err) - } -} - -func TestAudioQualityFromParsedFlacRejectsMissingStreamInfo(t *testing.T) { - if _, err := audioQualityFromParsedFlac(&flac.File{}); err == nil { - t.Fatal("missing STREAMINFO was accepted") - } -} - -func TestLibraryAudioCoverCacheKeyIncludesFileIdentity(t *testing.T) { - info := libraryAudioFileInfo{path: "track.flac", size: 123, modTime: 456} - if got := libraryAudioCoverCacheKey(info); got != "track.flac|123|456" { - t.Fatalf("cache key = %q", got) - } -} diff --git a/go_backend/library_scan_supplement_test.go b/go_backend/library_scan_supplement_test.go deleted file mode 100644 index 88b9719d..00000000 --- a/go_backend/library_scan_supplement_test.go +++ /dev/null @@ -1,222 +0,0 @@ -package gobackend - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - "testing" -) - -func TestLibraryScanFullIncrementalAndMetadataFallbacks(t *testing.T) { - dir := t.TempDir() - albumDir := filepath.Join(dir, "Album") - if err := os.MkdirAll(albumDir, 0755); err != nil { - t.Fatal(err) - } - mp3Path := filepath.Join(albumDir, "Artist - Song.mp3") - if err := os.WriteFile(mp3Path, []byte("not really mp3"), 0600); err != nil { - t.Fatal(err) - } - numberedPath := filepath.Join(albumDir, "01 - Intro.ogg") - if err := os.WriteFile(numberedPath, []byte("not really ogg"), 0600); err != nil { - t.Fatal(err) - } - apePath := filepath.Join(albumDir, "tagged.ape") - if err := os.WriteFile(apePath, []byte("audio"), 0600); err != nil { - t.Fatal(err) - } - if err := WriteAPETags(apePath, &APETag{Items: AudioMetadataToAPEItems(&AudioMetadata{ - Title: "Tagged", - Artist: "APE Artist", - Album: "APE Album", - TrackNumber: 2, - TotalTracks: 3, - Date: "2026", - Genre: "Pop", - Composer: "Composer", - })}); err != nil { - t.Fatalf("write ape tags: %v", err) - } - cuePath, _ := writeExportCueFixture(t, albumDir) - if err := os.WriteFile(filepath.Join(albumDir, "ignored.txt"), []byte("ignore"), 0600); err != nil { - t.Fatal(err) - } - legacyPartialPath := filepath.Join(albumDir, "Artist - Song.partial.flac") - if err := os.WriteFile(legacyPartialPath, []byte("partial flac"), 0600); err != nil { - t.Fatal(err) - } - newPartialPath := filepath.Join(albumDir, "Artist - Song.flac.partial") - if err := os.WriteFile(newPartialPath, []byte("partial flac"), 0600); err != nil { - t.Fatal(err) - } - - files, err := collectLibraryAudioFiles(dir, make(chan struct{})) - if err != nil { - t.Fatalf("collectLibraryAudioFiles: %v", err) - } - if len(files) < 4 { - t.Fatalf("files = %#v", files) - } - for _, file := range files { - if file.path == legacyPartialPath || file.path == newPartialPath { - t.Fatalf("staging file should be ignored: %#v", files) - } - } - cancelCh := make(chan struct{}) - close(cancelCh) - if _, err := collectLibraryAudioFiles(dir, cancelCh); err == nil { - t.Fatal("expected cancelled collect") - } - - jsonText, err := ScanLibraryFolder(dir) - if err != nil { - t.Fatalf("ScanLibraryFolder: %v", err) - } - var results []LibraryScanResult - if err := json.Unmarshal([]byte(jsonText), &results); err != nil { - t.Fatalf("decode scan results: %v", err) - } - if len(results) < 4 { - t.Fatalf("scan results = %#v", results) - } - foundTagged := false - for _, result := range results { - if result.FilePath == apePath { - foundTagged = result.TrackName == "Tagged" && result.ArtistName == "APE Artist" - } - } - if !foundTagged { - t.Fatalf("tagged APE not found in %#v", results) - } - - ndjsonPath := filepath.Join(t.TempDir(), "library.ndjson") - streamedCount, err := ScanLibraryFolderToNDJSONFile(dir, ndjsonPath) - if err != nil { - t.Fatalf("ScanLibraryFolderToNDJSONFile: %v", err) - } - ndjsonFile, err := os.Open(ndjsonPath) - if err != nil { - t.Fatal(err) - } - defer ndjsonFile.Close() - decodedCount := 0 - scanner := bufio.NewScanner(ndjsonFile) - for scanner.Scan() { - var result LibraryScanResult - if err := json.Unmarshal(scanner.Bytes(), &result); err != nil { - t.Fatalf("decode NDJSON row: %v", err) - } - if result.FilePath == "" { - t.Fatal("NDJSON row has no file path") - } - decodedCount++ - } - if err := scanner.Err(); err != nil { - t.Fatal(err) - } - if decodedCount != streamedCount || decodedCount != len(results) { - t.Fatalf("NDJSON counts = decoded:%d streamed:%d array:%d", decodedCount, streamedCount, len(results)) - } - if progress := GetLibraryScanProgress(); !strings.Contains(progress, `"IsComplete":true`) && !strings.Contains(progress, `"is_complete":true`) { - t.Fatalf("progress = %s", progress) - } - - metaJSON, err := ReadAudioMetadataWithDisplayName(mp3Path, "Display Artist - Display Song.mp3") - if err != nil { - t.Fatalf("ReadAudioMetadataWithDisplayName: %v", err) - } - if !strings.Contains(metaJSON, "Display Song") { - t.Fatalf("metadata json = %s", metaJSON) - } - noExtPath := filepath.Join(albumDir, "noext") - if err := os.WriteFile(noExtPath, []byte("audio"), 0600); err != nil { - t.Fatal(err) - } - noExtJSON, err := ReadAudioMetadataWithDisplayNameAndCoverCacheKey(noExtPath, "Artist - No Ext.mp3", "cache-key") - if err != nil { - t.Fatalf("ReadAudioMetadataWithDisplayNameAndCoverCacheKey: %v", err) - } - if !strings.Contains(noExtJSON, "No Ext") { - t.Fatalf("no ext metadata = %s", noExtJSON) - } - - existing := map[string]int64{} - for _, file := range files { - existing[file.path] = file.modTime - } - if info, err := os.Stat(cuePath); err == nil { - existing[cuePath+"#track01"] = info.ModTime().UnixMilli() - } - incJSON, err := scanLibraryFolderIncrementalWithExistingFiles(dir, existing) - if err != nil { - t.Fatalf("incremental existing: %v", err) - } - var inc IncrementalScanResult - if err := json.Unmarshal([]byte(incJSON), &inc); err != nil { - t.Fatalf("decode incremental: %v", err) - } - if inc.SkippedCount == 0 { - t.Fatalf("incremental = %#v", inc) - } - if _, err := ScanLibraryFolderIncremental("", "{}"); err == nil { - t.Fatal("expected empty incremental folder error") - } - if incJSON, err := ScanLibraryFolderIncremental(dir, `not-json`); err != nil || incJSON == "" { - t.Fatalf("incremental invalid existing JSON = %q/%v", incJSON, err) - } - - snapshot := filepath.Join(dir, "snapshot.txt") - if err := os.WriteFile(snapshot, []byte("bad\n123\t"+mp3Path+"\nnotint\tpath\n999\t"+filepath.Join(dir, "deleted.mp3")+"\n"), 0600); err != nil { - t.Fatal(err) - } - fromSnapshot, err := ScanLibraryFolderIncrementalFromSnapshot(dir, snapshot) - if err != nil { - t.Fatalf("snapshot incremental: %v", err) - } - if !strings.Contains(fromSnapshot, "deleted.mp3") { - t.Fatalf("snapshot result = %s", fromSnapshot) - } - if _, err := ScanLibraryFolder(""); err == nil { - t.Fatal("expected empty folder scan error") - } - fileInsteadOfFolder := filepath.Join(dir, "file.flac") - if err := os.WriteFile(fileInsteadOfFolder, []byte("audio"), 0600); err != nil { - t.Fatal(err) - } - if _, err := ScanLibraryFolder(fileInsteadOfFolder); err == nil { - t.Fatal("expected not folder error") - } - CancelLibraryScan() - SetLibraryCoverCacheDir("") -} - -func TestScanLibraryFolderPreservesFileOrderWithParallelWorkers(t *testing.T) { - dir := t.TempDir() - for i := 0; i < 20; i++ { - name := fmt.Sprintf("%02d - Track.mp3", i) - if err := os.WriteFile(filepath.Join(dir, name), []byte("not really mp3"), 0600); err != nil { - t.Fatal(err) - } - } - - jsonText, err := ScanLibraryFolder(dir) - if err != nil { - t.Fatal(err) - } - var results []LibraryScanResult - if err := json.Unmarshal([]byte(jsonText), &results); err != nil { - t.Fatal(err) - } - if len(results) != 20 { - t.Fatalf("results = %d", len(results)) - } - for i, result := range results { - expected := filepath.Join(dir, fmt.Sprintf("%02d - Track.mp3", i)) - if result.FilePath != expected { - t.Fatalf("result %d path = %q, want %q", i, result.FilePath, expected) - } - } -} diff --git a/go_backend/library_scan_test.go b/go_backend/library_scan_test.go deleted file mode 100644 index 859d3a60..00000000 --- a/go_backend/library_scan_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package gobackend - -import "testing" - -func TestScanFromFilenameMarksMetadataFallback(t *testing.T) { - result := &LibraryScanResult{} - - scanned, err := scanFromFilename( - "/proc/self/fd/209", - "189.mp3", - result, - ) - if err != nil { - t.Fatalf("scanFromFilename returned error: %v", err) - } - if !scanned.MetadataFromFilename { - t.Fatal("expected filename fallback marker to be set") - } - if scanned.TrackName != "189" { - t.Fatalf("unexpected track name: %q", scanned.TrackName) - } - if scanned.ArtistName != "Unknown Artist" { - t.Fatalf("unexpected artist name: %q", scanned.ArtistName) - } -} diff --git a/go_backend/log_progress_timeout_supplement_test.go b/go_backend/log_progress_timeout_supplement_test.go deleted file mode 100644 index e613ad11..00000000 --- a/go_backend/log_progress_timeout_supplement_test.go +++ /dev/null @@ -1,274 +0,0 @@ -package gobackend - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "strings" - "testing" - "time" - - "github.com/dop251/goja" -) - -func TestLogBufferExportedHelpersAndRedaction(t *testing.T) { - ClearLogs() - SetLoggingEnabled(false) - LogInfo("test", "ignored access_token=secret") - LogError("test", "Authorization: Bearer secret-token api_key=value") - if GetLogBuffer().Count() != 1 { - t.Fatalf("disabled logging should keep errors only, got %d", GetLogBuffer().Count()) - } - - SetLoggingEnabled(true) - defer SetLoggingEnabled(false) - LogDebug("debug", "client_secret=secret") - LogWarn("warn", "warning password=secret") - GoLog("[GoTag] success token=abc") - LogError("json", `{"access_token":"json-secret","session_secret":"session-secret"}`) - LogError("query", "https://example.test/?X-Amz-Signature=signed-secret&X-Amz-Security-Token=session-token") - LogError("ffmpeg", "-decryption_key raw-media-key -i https://example.test/audio") - LogError("bounded", "%s", strings.Repeat("x", maxLogMessageLength+500)) - - var entries []LogEntry - if err := json.Unmarshal([]byte(GetLogBuffer().GetAll()), &entries); err != nil { - t.Fatalf("GetAll JSON: %v", err) - } - if len(entries) < 4 { - t.Fatalf("expected log entries, got %#v", entries) - } - for _, entry := range entries { - if strings.Contains(entry.Message, "secret-token") || strings.Contains(entry.Message, "api_key=value") || strings.Contains(entry.Message, "password=secret") || strings.Contains(entry.Message, "json-secret") || strings.Contains(entry.Message, "session-secret") || strings.Contains(entry.Message, "signed-secret") || strings.Contains(entry.Message, "session-token") || strings.Contains(entry.Message, "raw-media-key") { - t.Fatalf("log was not redacted: %#v", entry) - } - if len(entry.Message) > maxLogMessageLength+len("...[truncated]") { - t.Fatalf("log was not bounded: %d bytes", len(entry.Message)) - } - } - - sinceJSON := GetLogsSince(1) - if !strings.Contains(sinceJSON, `"next_index"`) || !strings.Contains(sinceJSON, `"logs"`) { - t.Fatalf("GetLogsSince = %q", sinceJSON) - } - if emptyJSON := GetLogsSince(999); !strings.Contains(emptyJSON, `"logs":[]`) { - t.Fatalf("GetLogsSince empty = %q", emptyJSON) - } - if negativeJSON := GetLogsSince(-5); !strings.Contains(negativeJSON, `"logs"`) { - t.Fatalf("GetLogsSince negative = %q", negativeJSON) - } - - ClearLogs() - if GetLogBuffer().Count() != 0 || GetLogBuffer().GetAll() != "[]" { - t.Fatalf("logs were not cleared: count=%d logs=%s", GetLogBuffer().Count(), GetLogBuffer().GetAll()) - } -} - -func TestLogBufferCursorSurvivesRollover(t *testing.T) { - lb := &LogBuffer{ - entries: make([]LogEntry, 3), - maxSize: 3, - loggingEnabled: true, - } - for _, message := range []string{"one", "two", "three"} { - lb.Add("INFO", "Test", message) - } - initial, cursor := lb.getSince(0) - if cursor != 3 || len(initial) != 3 || initial[0].Message != "one" { - t.Fatalf("initial logs/cursor = %#v/%d", initial, cursor) - } - - lb.Add("INFO", "Test", "four") - newLogs, cursor := lb.getSince(cursor) - if cursor != 4 || len(newLogs) != 1 || newLogs[0].Message != "four" { - t.Fatalf("rollover logs/cursor = %#v/%d", newLogs, cursor) - } - - lb.Add("INFO", "Test", "five") - lb.Add("INFO", "Test", "six") - retained, cursor := lb.getSince(1) - if cursor != 6 || len(retained) != 3 || retained[0].Message != "four" || retained[2].Message != "six" { - t.Fatalf("retained logs/cursor = %#v/%d", retained, cursor) - } - - lb.Clear() - lb.Add("INFO", "Test", "seven") - afterClear, cursor := lb.getSince(6) - if cursor != 7 || len(afterClear) != 1 || afterClear[0].Message != "seven" { - t.Fatalf("after clear logs/cursor = %#v/%d", afterClear, cursor) - } -} - -func TestProgressItemHelpersAndWriter(t *testing.T) { - ClearAllItemProgress() - itemID := "progress-writer" - StartItemProgress(itemID) - SetItemBytesTotal(itemID, int64(progressUpdateThreshold*2)) - SetItemBytesReceived(itemID, int64(progressUpdateThreshold)) - - progressJSON := GetItemProgress(itemID) - if !strings.Contains(progressJSON, `"bytes_received":131072`) || !strings.Contains(progressJSON, `"progress":0.5`) { - t.Fatalf("GetItemProgress = %q", progressJSON) - } - if missing := GetItemProgress("missing"); missing != "{}" { - t.Fatalf("missing progress = %q", missing) - } - - var out bytes.Buffer - writer := NewItemProgressWriter(&out, itemID) - payload := bytes.Repeat([]byte("x"), progressUpdateThreshold+1) - n, err := writer.Write(payload) - if err != nil || n != len(payload) { - t.Fatalf("progress writer = %d/%v", n, err) - } - if out.Len() != len(payload) { - t.Fatalf("writer output length = %d", out.Len()) - } - if progressJSON = GetItemProgress(itemID); !strings.Contains(progressJSON, `"bytes_received":131073`) { - t.Fatalf("progress after writer = %q", progressJSON) - } - - cancelDownload(itemID) - defer clearDownloadCancel(itemID) - n, err = writer.Write([]byte("cancelled")) - if n != 0 || !errors.Is(err, ErrDownloadCancelled) { - t.Fatalf("cancelled writer = %d/%v", n, err) - } - - ClearAllItemProgress() -} - -func TestRunWithTimeoutBranches(t *testing.T) { - timeoutVM := goja.New() - _, err := RunWithTimeoutAndRecover(timeoutVM, "for (;;) {}", 10*time.Millisecond) - if err == nil { - t.Fatal("expected timeout error") - } - if !IsTimeoutError(&JSExecutionError{Message: "timeout", IsTimeout: true}) { - t.Fatal("JSExecutionError should be recognized as timeout") - } - if IsTimeoutError(errors.New("plain")) { - t.Fatal("plain error should not be timeout") - } - if (&JSExecutionError{Message: "boom"}).Error() != "boom" { - t.Fatal("JSExecutionError Error mismatch") - } -} - -func TestRunWithTimeoutQuarantinesUnresponsiveRuntime(t *testing.T) { - previousGrace := jsInterruptGracePeriod - jsInterruptGracePeriod = 10 * time.Millisecond - defer func() { jsInterruptGracePeriod = previousGrace }() - - vm := goja.New() - release := make(chan struct{}) - if err := vm.Set("block", func() { <-release }); err != nil { - t.Fatal(err) - } - _, err := RunWithTimeoutAndRecover(vm, "block()", 10*time.Millisecond) - if !IsRuntimeUnsafeError(err) { - close(release) - t.Fatalf("expected unsafe runtime error, got %v", err) - } - done := runtimeCompletion(err) - if done == nil { - close(release) - t.Fatal("unsafe runtime error should expose a completion signal") - } - select { - case <-done: - close(release) - t.Fatal("completion signal closed while the JS goroutine was blocked") - default: - } - close(release) - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("completion signal did not close after the JS goroutine exited") - } -} - -func TestRunWithTimeoutQuarantinesUnresponsiveCancelledRuntime(t *testing.T) { - previousGrace := jsInterruptGracePeriod - jsInterruptGracePeriod = 10 * time.Millisecond - defer func() { jsInterruptGracePeriod = previousGrace }() - - vm := goja.New() - entered := make(chan struct{}) - release := make(chan struct{}) - if err := vm.Set("block", func() { - close(entered) - <-release - }); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithCancel(context.Background()) - result := make(chan error, 1) - go func() { - _, err := RunWithTimeoutContextAndRecover(ctx, vm, "block()", time.Second) - result <- err - }() - <-entered - cancel() - err := <-result - if !IsRuntimeUnsafeError(err) || !errors.Is(err, ErrExtensionRequestCancelled) { - close(release) - t.Fatalf("expected unsafe cancellation error, got %v", err) - } - done := runtimeCompletion(err) - if done == nil { - close(release) - t.Fatal("unsafe cancellation should expose a completion signal") - } - close(release) - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("cancelled runtime did not report completion") - } -} - -func TestQuarantinedRuntimeBlocksReplacementUntilExecutionStops(t *testing.T) { - vm := goja.New() - runtime := &extensionRuntime{} - ext := &loadedExtension{ - ID: "quarantine-test", - VM: vm, - runtime: runtime, - initialized: true, - } - done := make(chan struct{}) - err := &JSExecutionError{ - Message: "runtime quarantined", - RuntimeUnsafe: true, - runtimeDone: done, - } - - quarantineRuntimeLocked(ext, vm, err) - if ext.VM != nil || ext.runtime != nil || ext.initialized { - t.Fatal("quarantine should detach the unsafe runtime") - } - if !hasQuarantinedRuntime(ext) { - t.Fatal("extension should remain gated while execution is still running") - } - if err := ensureRuntimeReadyLocked(ext, false); err == nil || - !strings.Contains(err.Error(), "still stopping") { - t.Fatalf("replacement runtime should be blocked, got %v", err) - } - - close(done) - deadline := time.Now().Add(time.Second) - for hasQuarantinedRuntime(ext) && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if hasQuarantinedRuntime(ext) { - t.Fatal("extension remained gated after the old execution stopped") - } - runtime.storageMu.RLock() - closed := runtime.storageClosed - runtime.storageMu.RUnlock() - if !closed { - t.Fatal("quarantined runtime resources were not closed after completion") - } -} diff --git a/go_backend/logbuffer.go b/go_backend/logbuffer.go deleted file mode 100644 index 8a8c3104..00000000 --- a/go_backend/logbuffer.go +++ /dev/null @@ -1,238 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "regexp" - "strings" - "sync" - "time" - "unicode/utf8" -) - -type LogEntry struct { - Timestamp string `json:"timestamp"` - Level string `json:"level"` - Tag string `json:"tag"` - Message string `json:"message"` -} - -type LogBuffer struct { - entries []LogEntry - maxSize int - head int - count int - nextIndex int - mu sync.RWMutex - loggingEnabled bool -} - -const ( - defaultLogBufferSize = 500 - maxLogMessageLength = 4000 -) - -var ( - globalLogBuffer *LogBuffer - logBufferOnce sync.Once - - authorizationBearerPattern = regexp.MustCompile(`(?i)\bAuthorization\b\s*[:=]\s*Bearer\s+[A-Za-z0-9._~+/\-]+=*`) - genericKeyValuePattern = regexp.MustCompile(`(?i)("?(?:access[_\s-]?token|refresh[_\s-]?token|id[_\s-]?token|client[_\s-]?secret|authorization|password|api[_\s-]?key|session[_\s-]?secret|decryption[_\s-]?key|cookie|set-cookie)"?)(\s*[:=]\s*)("(?:\\.|[^"\\])*"|[^\s,;}\]]+)`) - queryTokenPattern = regexp.MustCompile(`(?i)([?&](?:access_token|refresh_token|id_token|token|client_secret|api_key|apikey|password|code|grant|sig|signature|x-amz-signature|x-amz-credential|x-amz-security-token|awsaccesskeyid|googleaccessid|policy|key-pair-id)=)[^&\s]+`) - bearerTokenPattern = regexp.MustCompile(`(?i)\bBearer\s+[A-Za-z0-9._~+/\-]+=*`) - decryptionKeyFlagPattern = regexp.MustCompile(`(?i)(-decryption_key\s+)[^\s]+`) -) - -func sanitizeSensitiveLogText(message string) string { - redacted := message - redacted = authorizationBearerPattern.ReplaceAllString(redacted, "Authorization: Bearer [REDACTED]") - redacted = genericKeyValuePattern.ReplaceAllString(redacted, `${1}${2}[REDACTED]`) - redacted = queryTokenPattern.ReplaceAllString(redacted, `${1}[REDACTED]`) - redacted = bearerTokenPattern.ReplaceAllString(redacted, "Bearer [REDACTED]") - redacted = decryptionKeyFlagPattern.ReplaceAllString(redacted, `${1}[REDACTED]`) - return redacted -} - -func truncateLogMessage(message string) string { - if len(message) <= maxLogMessageLength { - return message - } - truncated := message[:maxLogMessageLength] - for !utf8.ValidString(truncated) { - truncated = truncated[:len(truncated)-1] - } - return truncated + "...[truncated]" -} - -func GetLogBuffer() *LogBuffer { - logBufferOnce.Do(func() { - globalLogBuffer = &LogBuffer{ - entries: make([]LogEntry, defaultLogBufferSize), - maxSize: defaultLogBufferSize, - loggingEnabled: false, - } - }) - return globalLogBuffer -} - -func (lb *LogBuffer) SetLoggingEnabled(enabled bool) { - lb.mu.Lock() - defer lb.mu.Unlock() - lb.loggingEnabled = enabled -} - -func (lb *LogBuffer) IsLoggingEnabled() bool { - lb.mu.RLock() - defer lb.mu.RUnlock() - return lb.loggingEnabled -} - -func (lb *LogBuffer) Add(level, tag, message string) { - lb.mu.Lock() - defer lb.mu.Unlock() - - if !lb.loggingEnabled && level != "ERROR" && level != "FATAL" { - return - } - - message = truncateLogMessage(sanitizeSensitiveLogText(message)) - - entry := LogEntry{ - Timestamp: time.Now().Format("15:04:05.000"), - Level: level, - Tag: tag, - Message: message, - } - - if lb.maxSize <= 0 { - return - } - if len(lb.entries) != lb.maxSize { - lb.entries = make([]LogEntry, lb.maxSize) - lb.head = 0 - lb.count = 0 - } - if lb.count < lb.maxSize { - index := (lb.head + lb.count) % lb.maxSize - lb.entries[index] = entry - lb.count++ - } else { - lb.entries[lb.head] = entry - lb.head = (lb.head + 1) % lb.maxSize - } - lb.nextIndex++ - - fmt.Printf("[%s] %s\n", tag, message) -} - -func (lb *LogBuffer) GetAll() string { - lb.mu.RLock() - defer lb.mu.RUnlock() - - jsonBytes, _ := json.Marshal(lb.snapshotLocked(0)) - return string(jsonBytes) -} - -func (lb *LogBuffer) getSince(index int) ([]LogEntry, int) { - lb.mu.RLock() - defer lb.mu.RUnlock() - - earliest := lb.nextIndex - lb.count - if index < earliest { - index = earliest - } - if index >= lb.nextIndex { - return []LogEntry{}, lb.nextIndex - } - return lb.snapshotLocked(index - earliest), lb.nextIndex -} - -func (lb *LogBuffer) snapshotLocked(offset int) []LogEntry { - if offset < 0 { - offset = 0 - } - if offset >= lb.count { - return []LogEntry{} - } - result := make([]LogEntry, lb.count-offset) - for i := offset; i < lb.count; i++ { - result[i-offset] = lb.entries[(lb.head+i)%lb.maxSize] - } - return result -} - -func (lb *LogBuffer) Clear() { - lb.mu.Lock() - defer lb.mu.Unlock() - for i := range lb.entries { - lb.entries[i] = LogEntry{} - } - lb.head = 0 - lb.count = 0 -} - -func (lb *LogBuffer) Count() int { - lb.mu.RLock() - defer lb.mu.RUnlock() - return lb.count -} - -func LogDebug(tag, format string, args ...any) { - GetLogBuffer().Add("DEBUG", tag, fmt.Sprintf(format, args...)) -} - -func LogInfo(tag, format string, args ...any) { - GetLogBuffer().Add("INFO", tag, fmt.Sprintf(format, args...)) -} - -func LogWarn(tag, format string, args ...any) { - GetLogBuffer().Add("WARN", tag, fmt.Sprintf(format, args...)) -} - -func LogError(tag, format string, args ...any) { - GetLogBuffer().Add("ERROR", tag, fmt.Sprintf(format, args...)) -} - -func GoLog(format string, args ...any) { - message := fmt.Sprintf(format, args...) - message = strings.TrimSuffix(message, "\n") - - tag := "Go" - level := "INFO" - - if strings.HasPrefix(message, "[") { - endBracket := strings.Index(message, "]") - if endBracket > 1 { - tag = message[1:endBracket] - message = strings.TrimSpace(message[endBracket+1:]) - } - } - - msgLower := strings.ToLower(message) - if strings.Contains(msgLower, "error") || strings.Contains(msgLower, "failed") { - level = "ERROR" - } else if strings.Contains(msgLower, "warning") || strings.Contains(msgLower, "warn") { - level = "WARN" - } else if strings.Contains(msgLower, "success") || strings.Contains(msgLower, "match found") { - level = "INFO" - } else if strings.Contains(msgLower, "searching") || strings.Contains(msgLower, "trying") || strings.Contains(msgLower, "found") { - level = "DEBUG" - } - - GetLogBuffer().Add(level, tag, message) -} - -func GetLogsSince(index int) string { - entries, nextIndex := GetLogBuffer().getSince(index) - logsJson, _ := json.Marshal(entries) - result := fmt.Sprintf(`{"logs":%s,"next_index":%d}`, string(logsJson), nextIndex) - return result -} - -func ClearLogs() { - GetLogBuffer().Clear() -} - -func SetLoggingEnabled(enabled bool) { - GetLogBuffer().SetLoggingEnabled(enabled) -} diff --git a/go_backend/lyrics.go b/go_backend/lyrics.go deleted file mode 100644 index 80406489..00000000 --- a/go_backend/lyrics.go +++ /dev/null @@ -1,881 +0,0 @@ -package gobackend - -import ( - "context" - "encoding/json" - "fmt" - "math" - "net/http" - "net/url" - "sort" - "strings" - "sync" - "time" - - "golang.org/x/sync/singleflight" -) - -const ( - lyricsCacheTTL = 24 * time.Hour - durationToleranceSec = 10.0 - lyricsNegativeTTL = 5 * time.Minute - lyricsNegativeMax = 500 -) - -var ( - lyricsFetchFlight singleflight.Group - lyricsNegativeMu sync.Mutex - lyricsNegative = map[string]time.Time{} -) - -type LRCLibResponse struct { - ID int `json:"id"` - Name string `json:"name"` - TrackName string `json:"trackName"` - ArtistName string `json:"artistName"` - AlbumName string `json:"albumName"` - Duration float64 `json:"duration"` - Instrumental bool `json:"instrumental"` - PlainLyrics string `json:"plainLyrics"` - SyncedLyrics string `json:"syncedLyrics"` -} - -func lrclibTrackName(response *LRCLibResponse) string { - if response == nil { - return "" - } - if trackName := strings.TrimSpace(response.TrackName); trackName != "" { - return trackName - } - return strings.TrimSpace(response.Name) -} - -type LyricsLine struct { - StartTimeMs int64 `json:"startTimeMs"` - Words string `json:"words"` - EndTimeMs int64 `json:"endTimeMs"` -} - -type LyricsResponse struct { - Lines []LyricsLine `json:"lines"` - SyncType string `json:"syncType"` - Instrumental bool `json:"instrumental"` - PlainLyrics string `json:"plainLyrics"` - Provider string `json:"provider"` - Source string `json:"source"` -} - -type LyricsClient struct { - httpClient *http.Client -} - -type lyricsContextTransport struct { - ctx context.Context - base http.RoundTripper -} - -func (t lyricsContextTransport) RoundTrip(request *http.Request) (*http.Response, error) { - return t.base.RoundTrip(request.Clone(t.ctx)) -} - -func bindLyricsHTTPClientContext( - client *http.Client, - ctx context.Context, -) *http.Client { - if client == nil || ctx == nil { - return client - } - copy := *client - transport := client.Transport - if transport == nil { - transport = http.DefaultTransport - } - copy.Transport = lyricsContextTransport{ctx: ctx, base: transport} - return © -} - -func NewLyricsClient() *LyricsClient { - return &LyricsClient{ - httpClient: NewHTTPClientWithTimeout(15 * time.Second), - } -} - -// lrclibGet performs a GET against lrclib.net and decodes the JSON body into -// dst. 404 is reported as a typed lyrics-not-found error. -func (c *LyricsClient) lrclibGet(path string, params url.Values, dst any) error { - req, err := http.NewRequest("GET", "https://lrclib.net"+path+"?"+params.Encode(), nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("User-Agent", getRandomUserAgent()) - - resp, err := c.httpClient.Do(req) - if err != nil { - return fmt.Errorf("failed to fetch lyrics: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode == 404 { - return lyricsNotFoundErrorf("lyrics not found") - } - if resp.StatusCode != 200 { - return lyricsHTTPStatusError(resp.StatusCode, "unexpected status code: %d", resp.StatusCode) - } - - if err := json.NewDecoder(resp.Body).Decode(dst); err != nil { - return fmt.Errorf("failed to decode response: %w", err) - } - return nil -} - -func (c *LyricsClient) FetchLyricsWithMetadata(artist, track string) (*LyricsResponse, error) { - params := url.Values{} - params.Set("artist_name", artist) - params.Set("track_name", track) - - var lrcResp LRCLibResponse - if err := c.lrclibGet("/api/get", params, &lrcResp); err != nil { - return nil, err - } - if !lyricsSearchTitlesMatch(lrclibTrackName(&lrcResp), track, false) || - !lyricsSearchArtistsMatch(lrcResp.ArtistName, artist) { - return nil, lyricsNotFoundErrorf("LRCLIB returned mismatched track metadata") - } - - return c.parseLRCLibResponse(&lrcResp), nil -} - -func (c *LyricsClient) fetchLyricsFromLRCLibSearch(query, trackName, artistName string, durationSec float64) (*LyricsResponse, error) { - params := url.Values{} - params.Set("q", query) - - var results []LRCLibResponse - if err := c.lrclibGet("/api/search", params, &results); err != nil { - return nil, err - } - - if len(results) == 0 { - return nil, lyricsNotFoundErrorf("no lyrics found") - } - - bestMatch := c.findBestLRCLibSearchMatch(results, query, trackName, artistName, durationSec) - if bestMatch != nil { - return c.parseLRCLibResponse(bestMatch), nil - } - - return nil, lyricsNotFoundErrorf("no matching lyrics found") -} - -func lrclibSearchResultMatches(result *LRCLibResponse, query, trackName, artistName string, durationSec float64) bool { - if result == nil || !lyricsSearchDurationMatches(result.Duration, durationSec) { - return false - } - - candidateTrack := lrclibTrackName(result) - if strings.TrimSpace(trackName) != "" || strings.TrimSpace(artistName) != "" { - return lyricsSearchTitlesMatch(candidateTrack, trackName, false) && - lyricsSearchArtistsMatch(result.ArtistName, artistName) - } - - normalizedQuery := normalizeLooseArtistName(query) - normalizedTrack := normalizeLooseArtistName(simplifyTrackName(candidateTrack)) - normalizedArtist := normalizeLooseArtistName(normalizeArtistName(result.ArtistName)) - return normalizedQuery != "" && - normalizedTrack != "" && - normalizedArtist != "" && - containsWordSequence(normalizedQuery, normalizedTrack) && - containsWordSequence(normalizedQuery, normalizedArtist) -} - -func (c *LyricsClient) findBestLRCLibSearchMatch(results []LRCLibResponse, query, trackName, artistName string, targetDurationSec float64) *LRCLibResponse { - var bestSynced *LRCLibResponse - var bestPlain *LRCLibResponse - - for i := range results { - result := &results[i] - if !lrclibSearchResultMatches(result, query, trackName, artistName, targetDurationSec) { - continue - } - if result.SyncedLyrics != "" && bestSynced == nil { - bestSynced = result - } else if result.PlainLyrics != "" && bestPlain == nil { - bestPlain = result - } - } - - if bestSynced != nil { - return bestSynced - } - return bestPlain -} - -func plainLyricsFromTimedLines(lines []LyricsLine) string { - parts := make([]string, 0, len(lines)) - for _, line := range lines { - words := strings.TrimSpace(line.Words) - if words == "" { - continue - } - parts = append(parts, words) - } - return strings.Join(parts, "\n") -} - -func (c *LyricsClient) durationMatches(lrcDuration, targetDuration float64) bool { - diff := math.Abs(lrcDuration - targetDuration) - return diff <= durationToleranceSec -} - -func lyricsFetchCacheKey(spotifyID, trackName, artistName string, durationSec float64) string { - providers := GetLyricsProviderOrder() - extensions := make([]string, 0) - if manager := getExtensionManager(); manager != nil { - for _, provider := range manager.GetLyricsProviders() { - extensions = append(extensions, strings.ToLower(strings.TrimSpace(provider.extension.ID))) - } - } - sort.Strings(extensions) - opts := GetLyricsFetchOptions() - return fmt.Sprintf( - "%s|%s|%s|%.0f|%s|%s|%t|%t|%t|%t|%s", - strings.TrimSpace(spotifyID), - strings.ToLower(strings.TrimSpace(artistName)), - strings.ToLower(strings.TrimSpace(trackName)), - math.Round(durationSec/10)*10, - strings.Join(providers, ","), - strings.Join(extensions, ","), - opts.IncludeTranslationNetease, - opts.IncludeRomanizationNetease, - opts.MultiPersonWordByWord, - opts.AppleElrcWordSync, - opts.MusixmatchLanguage, - ) -} - -func isNegativeLyricsCached(key string) bool { - now := time.Now() - lyricsNegativeMu.Lock() - defer lyricsNegativeMu.Unlock() - expiresAt, ok := lyricsNegative[key] - if !ok { - return false - } - if !now.Before(expiresAt) { - delete(lyricsNegative, key) - return false - } - return true -} - -func cacheNegativeLyrics(key string) { - now := time.Now() - lyricsNegativeMu.Lock() - defer lyricsNegativeMu.Unlock() - if len(lyricsNegative) >= lyricsNegativeMax { - for existingKey, expiresAt := range lyricsNegative { - if !now.Before(expiresAt) { - delete(lyricsNegative, existingKey) - } - } - for len(lyricsNegative) >= lyricsNegativeMax { - for existingKey := range lyricsNegative { - delete(lyricsNegative, existingKey) - break - } - } - } - lyricsNegative[key] = now.Add(lyricsNegativeTTL) -} - -func clearNegativeLyrics(key string) { - lyricsNegativeMu.Lock() - delete(lyricsNegative, key) - lyricsNegativeMu.Unlock() -} - -func (c *LyricsClient) FetchLyricsAllSources(spotifyID, trackName, artistName string, durationSec float64) (*LyricsResponse, error) { - key := lyricsFetchCacheKey(spotifyID, trackName, artistName, durationSec) - if isNegativeLyricsCached(key) { - return nil, lyricsNotFoundErrorf("lyrics not found (cached)") - } - - value, err, _ := lyricsFetchFlight.Do(key, func() (any, error) { - lyrics, fetchErr := c.fetchLyricsAllSourcesUncoalesced( - spotifyID, - trackName, - artistName, - durationSec, - ) - if fetchErr != nil { - cacheNegativeLyrics(key) - return nil, fetchErr - } - clearNegativeLyrics(key) - return lyrics, nil - }) - if err != nil { - return nil, err - } - lyrics, _ := value.(*LyricsResponse) - if lyrics == nil { - return nil, lyricsNotFoundErrorf("lyrics not found from any source") - } - copy := *lyrics - return ©, nil -} - -func (c *LyricsClient) fetchLyricsAllSourcesUncoalesced(spotifyID, trackName, artistName string, durationSec float64) (*LyricsResponse, error) { - primaryArtist := normalizeArtistName(artistName) - fetchOptions := GetLyricsFetchOptions() - configuredProviderOrder := GetLyricsProviderOrder() - - if isLikelyInstrumentalTrack(trackName) { - GoLog("[Lyrics] Track marked instrumental by title heuristic, skipping lyrics search: %s - %s\n", artistName, trackName) - instrumental := &LyricsResponse{ - Instrumental: true, - Source: "Heuristic: Instrumental", - } - globalLyricsCache.Set(artistName, trackName, durationSec, instrumental) - return instrumental, nil - } - - extensionProviders := make(map[string]*extensionProviderWrapper) - extManager := getExtensionManager() - if extManager != nil { - for _, provider := range extManager.GetLyricsProviders() { - providerName := "extension:" + strings.ToLower(strings.TrimSpace(provider.extension.ID)) - extensionProviders[providerName] = provider - } - } - - providerOrder := resolveLyricsProviderOrder(configuredProviderOrder, extensionProviders) - selectedExtensionCount := 0 - for _, providerName := range providerOrder { - if strings.HasPrefix(providerName, "extension:") { - selectedExtensionCount++ - } - } - - var cachedNonExtension *LyricsResponse - if cached, found := globalLyricsCache.Get(artistName, trackName, durationSec); found { - isExtensionCache := strings.HasPrefix(cached.Source, "Extension:") - cachedProviderSelected := false - if isExtensionCache { - cachedProviderName := "extension:" + strings.ToLower(strings.TrimSpace(strings.TrimPrefix(cached.Source, "Extension:"))) - for _, providerName := range providerOrder { - if providerName == cachedProviderName { - cachedProviderSelected = true - break - } - } - } - if (!isExtensionCache && selectedExtensionCount == 0) || cachedProviderSelected { - cachedCopy := *cached - cachedCopy.Source = cached.Source + " (cached)" - return &cachedCopy, nil - } - - if !isExtensionCache { - // If extension providers are currently selected, don't let stale built-in - // cache mask them. It remains available as a fallback if they fail. - cachedNonExtension = cached - GoLog("[Lyrics] Ignoring cached non-extension lyrics because selected extension providers are available\n") - } else { - GoLog("[Lyrics] Ignoring cached lyrics from an unselected extension provider\n") - } - } - - isValidResult := func(l *LyricsResponse) bool { - return lyricsHasUsableText(l) - } - - simplifiedTrack := simplifyTrackName(trackName) - request := lyricsProviderSearchRequest{ - spotifyID: spotifyID, - trackName: trackName, - artistName: artistName, - primaryArtist: primaryArtist, - simplifiedTrack: simplifiedTrack, - durationSec: durationSec, - fetchOptions: fetchOptions, - } - - GoLog("[Lyrics] Searching for: %s - %s (providers: %v)\n", artistName, trackName, providerOrder) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - fetchProvider := func(providerName string, request lyricsProviderSearchRequest) (*LyricsResponse, error, bool) { - if provider, ok := extensionProviders[providerName]; ok { - lyrics, err := provider.FetchLyricsContext(ctx, request.trackName, request.artistName, "", request.durationSec) - return lyrics, err, true - } - return c.fetchBuiltInLyricsProviderContext(ctx, providerName, request) - } - - lyrics, err := fetchLyricsProvidersContext(ctx, providerOrder, request, fetchProvider) - if err == nil && isValidResult(lyrics) { - globalLyricsCache.Set(artistName, trackName, durationSec, lyrics) - return lyrics, nil - } - - if cachedNonExtension != nil { - cachedCopy := *cachedNonExtension - cachedCopy.Source = cachedNonExtension.Source + " (cached fallback)" - GoLog("[Lyrics] Selected extension providers unavailable for this track, using cached built-in lyrics\n") - return &cachedCopy, nil - } - - return nil, fmt.Errorf("lyrics not found from any source") -} - -func resolveLyricsProviderOrder( - configuredOrder []string, - extensionProviders map[string]*extensionProviderWrapper, -) []string { - providerOrder := make([]string, 0, len(configuredOrder)) - for _, providerName := range configuredOrder { - if isKnownBuiltInLyricsProvider(providerName) { - providerOrder = append(providerOrder, providerName) - continue - } - if _, available := extensionProviders[providerName]; available { - providerOrder = append(providerOrder, providerName) - } - } - return providerOrder -} - -func fetchLyricsProviders( - providerOrder []string, - request lyricsProviderSearchRequest, - fetchProvider func(string, lyricsProviderSearchRequest) (*LyricsResponse, error, bool), -) (*LyricsResponse, error) { - return fetchLyricsProvidersContext(context.Background(), providerOrder, request, fetchProvider) -} - -func fetchLyricsProvidersContext( - ctx context.Context, - providerOrder []string, - request lyricsProviderSearchRequest, - fetchProvider func(string, lyricsProviderSearchRequest) (*LyricsResponse, error, bool), -) (*LyricsResponse, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - type providerCandidate struct { - index int - name string - } - - candidates := make([]providerCandidate, 0, len(providerOrder)) - results := make(chan lyricsProviderSearchResult, len(providerOrder)) - sem := make(chan struct{}, lyricsProviderParallelism) - var wg sync.WaitGroup - - for index, providerName := range providerOrder { - if skip, remaining, reason := shouldSkipLyricsProvider(providerName); skip { - GoLog("[Lyrics] Skipping unavailable provider %s for %s: %s\n", providerName, remaining.Round(time.Second), reason) - continue - } - - knownProvider := isKnownBuiltInLyricsProvider(providerName) || - (strings.HasPrefix(providerName, "extension:") && len(providerName) > len("extension:")) - if !knownProvider { - GoLog("[Lyrics] Unknown provider: %s, skipping\n", providerName) - continue - } - - candidate := providerCandidate{index: index, name: providerName} - candidates = append(candidates, candidate) - wg.Add(1) - go func() { - defer wg.Done() - select { - case sem <- struct{}{}: - case <-ctx.Done(): - return - } - defer func() { <-sem }() - if ctx.Err() != nil { - return - } - - GoLog("[Lyrics] Trying provider: %s\n", candidate.name) - lyrics, err, ok := fetchProvider(candidate.name, request) - if !ok { - results <- lyricsProviderSearchResult{index: candidate.index, providerName: candidate.name, err: fmt.Errorf("unknown provider")} - return - } - if err == nil && lyricsHasUsableText(lyrics) { - GoLog("[Lyrics] Got lyrics from: %s\n", candidate.name) - markLyricsProviderAvailable(candidate.name) - } else if err != nil { - GoLog("[Lyrics] Provider %s failed: %v\n", candidate.name, err) - markLyricsProviderUnavailable(candidate.name, err) - } - select { - case results <- lyricsProviderSearchResult{index: candidate.index, providerName: candidate.name, lyrics: lyrics, err: err}: - case <-ctx.Done(): - } - }() - } - - if len(candidates) == 0 { - return nil, fmt.Errorf("lyrics not found from any source") - } - - go func() { - wg.Wait() - close(results) - }() - - completed := make(map[int]bool, len(candidates)) - var best *lyricsProviderSearchResult - var lastErr error - var graceTimer *time.Timer - var grace <-chan time.Time - - stopGrace := func() { - if graceTimer != nil { - if !graceTimer.Stop() { - select { - case <-graceTimer.C: - default: - } - } - graceTimer = nil - grace = nil - } - } - defer stopGrace() - - hasPendingEarlier := func(index int) bool { - for _, candidate := range candidates { - if candidate.index >= index { - return false - } - if !completed[candidate.index] { - return true - } - } - return false - } - - for remaining := len(candidates); remaining > 0; { - if best != nil && !hasPendingEarlier(best.index) { - return best.lyrics, nil - } - if best != nil && graceTimer == nil { - graceTimer = time.NewTimer(lyricsProviderPriorityGrace) - grace = graceTimer.C - } - - select { - case <-ctx.Done(): - return nil, ctx.Err() - case result, ok := <-results: - if !ok { - remaining = 0 - break - } - remaining-- - completed[result.index] = true - if result.err != nil { - lastErr = result.err - } - if lyricsHasUsableText(result.lyrics) && (best == nil || result.index < best.index) { - copied := result - best = &copied - stopGrace() - } - case <-grace: - if best != nil { - GoLog("[Lyrics] Returning provider %s after %s priority grace\n", best.providerName, lyricsProviderPriorityGrace) - return best.lyrics, nil - } - } - } - - if best != nil { - return best.lyrics, nil - } - if lastErr != nil { - return nil, lastErr - } - return nil, fmt.Errorf("lyrics not found from any source") -} - -func isKnownBuiltInLyricsProvider(providerName string) bool { - switch providerName { - case LyricsProviderLRCLIB, - LyricsProviderNetease, - LyricsProviderMusixmatch, - LyricsProviderAppleMusic, - LyricsProviderQQMusic, - LyricsProviderSpotify, - LyricsProviderDeezer, - LyricsProviderYouTube, - LyricsProviderKugou, - LyricsProviderGenius, - LyricsProviderLyricsPlus: - return true - default: - return false - } -} - -func (c *LyricsClient) fetchBuiltInLyricsProviderContext(ctx context.Context, providerName string, request lyricsProviderSearchRequest) (*LyricsResponse, error, bool) { - clientCopy := *c - clientCopy.httpClient = bindLyricsHTTPClientContext(c.httpClient, ctx) - c = &clientCopy - switch providerName { - case LyricsProviderLRCLIB: - lyrics, err := c.tryLRCLIB(request.primaryArtist, request.artistName, request.trackName, request.simplifiedTrack, request.durationSec) - return lyrics, err, true - - case LyricsProviderNetease: - neteaseClient := NewNeteaseClient() - neteaseClient.httpClient = bindLyricsHTTPClientContext(neteaseClient.httpClient, ctx) - lyrics, err := neteaseClient.FetchLyrics( - request.trackName, - request.primaryArtist, - request.durationSec, - request.fetchOptions.IncludeTranslationNetease, - request.fetchOptions.IncludeRomanizationNetease, - ) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = neteaseClient.FetchLyrics( - request.trackName, - request.artistName, - request.durationSec, - request.fetchOptions.IncludeTranslationNetease, - request.fetchOptions.IncludeRomanizationNetease, - ) - } - if err != nil && !isLyricsProviderUnavailableError(err) && request.simplifiedTrack != request.trackName { - lyrics, err = neteaseClient.FetchLyrics( - request.simplifiedTrack, - request.primaryArtist, - request.durationSec, - request.fetchOptions.IncludeTranslationNetease, - request.fetchOptions.IncludeRomanizationNetease, - ) - } - return lyrics, err, true - - case LyricsProviderMusixmatch: - musixmatchClient := NewMusixmatchClient() - musixmatchClient.httpClient = bindLyricsHTTPClientContext(musixmatchClient.httpClient, ctx) - lyrics, err := musixmatchClient.FetchLyrics( - request.trackName, - request.primaryArtist, - request.durationSec, - request.fetchOptions.MusixmatchLanguage, - ) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = musixmatchClient.FetchLyrics( - request.trackName, - request.artistName, - request.durationSec, - request.fetchOptions.MusixmatchLanguage, - ) - } - return lyrics, err, true - - case LyricsProviderAppleMusic: - appleClient := NewAppleMusicClient() - appleClient.httpClient = bindLyricsHTTPClientContext(appleClient.httpClient, ctx) - lyrics, err := appleClient.FetchLyrics(request.trackName, request.primaryArtist, request.durationSec, request.fetchOptions.MultiPersonWordByWord, request.fetchOptions.AppleElrcWordSync) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = appleClient.FetchLyrics(request.trackName, request.artistName, request.durationSec, request.fetchOptions.MultiPersonWordByWord, request.fetchOptions.AppleElrcWordSync) - } - return lyrics, err, true - - case LyricsProviderQQMusic: - qqClient := NewQQMusicClient() - qqClient.httpClient = bindLyricsHTTPClientContext(qqClient.httpClient, ctx) - lyrics, err := qqClient.FetchLyrics(request.trackName, request.primaryArtist, request.durationSec, request.fetchOptions.MultiPersonWordByWord) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = qqClient.FetchLyrics(request.trackName, request.artistName, request.durationSec, request.fetchOptions.MultiPersonWordByWord) - } - return lyrics, err, true - - case LyricsProviderSpotify: - spotifyClient := NewSpotifyLyricsClient() - spotifyClient.httpClient = bindLyricsHTTPClientContext(spotifyClient.httpClient, ctx) - lyrics, err := spotifyClient.FetchLyrics(request.spotifyID, request.trackName, request.primaryArtist, request.durationSec) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = spotifyClient.FetchLyrics(request.spotifyID, request.trackName, request.artistName, request.durationSec) - } - if err != nil && !isLyricsProviderUnavailableError(err) && request.simplifiedTrack != request.trackName { - lyrics, err = spotifyClient.FetchLyrics("", request.simplifiedTrack, request.primaryArtist, request.durationSec) - } - return lyrics, err, true - - case LyricsProviderDeezer: - deezerClient := NewDeezerLyricsClient() - deezerClient.httpClient = bindLyricsHTTPClientContext(deezerClient.httpClient, ctx) - lyrics, err := deezerClient.FetchLyrics(request.spotifyID, request.trackName, request.primaryArtist, request.durationSec) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = deezerClient.FetchLyrics(request.spotifyID, request.trackName, request.artistName, request.durationSec) - } - return lyrics, err, true - - case LyricsProviderYouTube: - youtubeClient := NewYouTubeLyricsClient() - youtubeClient.httpClient = bindLyricsHTTPClientContext(youtubeClient.httpClient, ctx) - lyrics, err := youtubeClient.FetchLyrics(request.trackName, request.primaryArtist, request.durationSec) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = youtubeClient.FetchLyrics(request.trackName, request.artistName, request.durationSec) - } - if err != nil && !isLyricsProviderUnavailableError(err) && request.simplifiedTrack != request.trackName { - lyrics, err = youtubeClient.FetchLyrics(request.simplifiedTrack, request.primaryArtist, request.durationSec) - } - return lyrics, err, true - - case LyricsProviderKugou: - kugouClient := NewKugouLyricsClient() - kugouClient.httpClient = bindLyricsHTTPClientContext(kugouClient.httpClient, ctx) - lyrics, err := kugouClient.FetchLyrics(request.trackName, request.primaryArtist, request.durationSec) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = kugouClient.FetchLyrics(request.trackName, request.artistName, request.durationSec) - } - if err != nil && !isLyricsProviderUnavailableError(err) && request.simplifiedTrack != request.trackName { - lyrics, err = kugouClient.FetchLyrics(request.simplifiedTrack, request.primaryArtist, request.durationSec) - } - return lyrics, err, true - - case LyricsProviderGenius: - geniusClient := NewGeniusLyricsClient() - geniusClient.httpClient = bindLyricsHTTPClientContext(geniusClient.httpClient, ctx) - lyrics, err := geniusClient.FetchLyrics(request.trackName, request.primaryArtist, request.durationSec) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = geniusClient.FetchLyrics(request.trackName, request.artistName, request.durationSec) - } - if err != nil && !isLyricsProviderUnavailableError(err) && request.simplifiedTrack != request.trackName { - lyrics, err = geniusClient.FetchLyrics(request.simplifiedTrack, request.primaryArtist, request.durationSec) - } - return lyrics, err, true - - case LyricsProviderLyricsPlus: - lyricsPlusClient := NewLyricsPlusClient() - lyricsPlusClient.httpClient = bindLyricsHTTPClientContext(lyricsPlusClient.httpClient, ctx) - lyrics, err := lyricsPlusClient.FetchLyrics( - request.trackName, - request.primaryArtist, - "", - request.durationSec, - request.fetchOptions.MultiPersonWordByWord, - request.fetchOptions.AppleElrcWordSync, - ) - if err != nil && !isLyricsProviderUnavailableError(err) && request.primaryArtist != request.artistName { - lyrics, err = lyricsPlusClient.FetchLyrics( - request.trackName, - request.artistName, - "", - request.durationSec, - request.fetchOptions.MultiPersonWordByWord, - request.fetchOptions.AppleElrcWordSync, - ) - } - if err != nil && !isLyricsProviderUnavailableError(err) && request.simplifiedTrack != request.trackName { - lyrics, err = lyricsPlusClient.FetchLyrics( - request.simplifiedTrack, - request.primaryArtist, - "", - request.durationSec, - request.fetchOptions.MultiPersonWordByWord, - request.fetchOptions.AppleElrcWordSync, - ) - } - return lyrics, err, true - default: - return nil, fmt.Errorf("unknown provider: %s", providerName), false - } -} - -func (c *LyricsClient) tryLRCLIB(primaryArtist, artistName, trackName, simplifiedTrack string, durationSec float64) (*LyricsResponse, error) { - var lyrics *LyricsResponse - var err error - - lyrics, err = c.FetchLyricsWithMetadata(primaryArtist, trackName) - if err == nil && lyrics != nil && (len(lyrics.Lines) > 0 || lyrics.Instrumental) { - lyrics.Source = "LRCLIB" - return lyrics, nil - } - if isLyricsProviderUnavailableError(err) { - return nil, err - } - - if primaryArtist != artistName { - lyrics, err = c.FetchLyricsWithMetadata(artistName, trackName) - if err == nil && lyrics != nil && (len(lyrics.Lines) > 0 || lyrics.Instrumental) { - lyrics.Source = "LRCLIB" - return lyrics, nil - } - if isLyricsProviderUnavailableError(err) { - return nil, err - } - } - - if simplifiedTrack != trackName { - lyrics, err = c.FetchLyricsWithMetadata(primaryArtist, simplifiedTrack) - if err == nil && lyrics != nil && (len(lyrics.Lines) > 0 || lyrics.Instrumental) { - lyrics.Source = "LRCLIB (simplified)" - return lyrics, nil - } - if isLyricsProviderUnavailableError(err) { - return nil, err - } - } - - query := primaryArtist + " " + trackName - lyrics, err = c.fetchLyricsFromLRCLibSearch(query, trackName, primaryArtist, durationSec) - if err == nil && lyrics != nil && (len(lyrics.Lines) > 0 || lyrics.Instrumental) { - lyrics.Source = "LRCLIB Search" - return lyrics, nil - } - if isLyricsProviderUnavailableError(err) { - return nil, err - } - - if simplifiedTrack != trackName { - query = primaryArtist + " " + simplifiedTrack - lyrics, err = c.fetchLyricsFromLRCLibSearch(query, simplifiedTrack, primaryArtist, durationSec) - if err == nil && lyrics != nil && (len(lyrics.Lines) > 0 || lyrics.Instrumental) { - lyrics.Source = "LRCLIB Search (simplified)" - return lyrics, nil - } - if isLyricsProviderUnavailableError(err) { - return nil, err - } - } - - return nil, lyricsNotFoundErrorf("LRCLIB: no lyrics found") -} - -func (c *LyricsClient) parseLRCLibResponse(resp *LRCLibResponse) *LyricsResponse { - result := &LyricsResponse{ - Instrumental: resp.Instrumental, - PlainLyrics: resp.PlainLyrics, - Provider: "LRCLIB", - } - - if resp.SyncedLyrics != "" { - result.Lines = parseSyncedLyrics(resp.SyncedLyrics) - result.SyncType = "LINE_SYNCED" - } else if resp.PlainLyrics != "" { - result.SyncType = "UNSYNCED" - lines := strings.Split(resp.PlainLyrics, "\n") - for _, line := range lines { - if strings.TrimSpace(line) != "" { - result.Lines = append(result.Lines, LyricsLine{ - StartTimeMs: 0, - Words: line, - EndTimeMs: 0, - }) - } - } - } - - return result -} diff --git a/go_backend/lyrics_apple.go b/go_backend/lyrics_apple.go deleted file mode 100644 index 20081a63..00000000 --- a/go_backend/lyrics_apple.go +++ /dev/null @@ -1,447 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "regexp" - "strings" - "sync" - "time" -) - -var errAppleMusicUnauthorized = errors.New("apple music catalog search unauthorized") - -type AppleMusicClient struct { - httpClient *http.Client -} - -const appleMusicCatalogBaseURL = "https://amp-api.music.apple.com/v1/catalog/us" - -type appleMusicSearchResult struct { - ID string `json:"id"` - SongName string `json:"songName"` - ArtistName string `json:"artistName"` - AlbumName string `json:"albumName"` - Duration int `json:"duration"` -} - -type appleMusicCatalogSearchResponse struct { - Results struct { - Songs *struct { - Data []struct { - ID string `json:"id"` - } `json:"data"` - } `json:"songs"` - } `json:"results"` - Resources *struct { - Songs map[string]struct { - Attributes struct { - Name string `json:"name"` - ArtistName string `json:"artistName"` - AlbumName string `json:"albumName"` - DurationInMillis int `json:"durationInMillis"` - } `json:"attributes"` - } `json:"songs"` - } `json:"resources"` -} - -type paxResponse struct { - Type string `json:"type"` // "Syllable" or "Line" - Content []paxLyrics `json:"content"` - ELRC string `json:"elrc"` - ELRCMultiPerson string `json:"elrcMultiPerson"` - Plain string `json:"plain"` - TTMLContent string `json:"ttmlContent"` -} - -type paxLyrics struct { - Text []paxLyricDetail `json:"text"` - Timestamp int `json:"timestamp"` - OppositeTurn bool `json:"oppositeTurn"` - Background bool `json:"background"` - BackgroundText []paxLyricDetail `json:"backgroundText"` - EndTime int `json:"endtime"` -} - -type paxLyricDetail struct { - Text string `json:"text"` - Part bool `json:"part"` - Timestamp *int `json:"timestamp"` - EndTime *int `json:"endtime"` -} - -var ( - appleMusicTokenMu sync.Mutex - appleMusicCachedToken string -) - -func NewAppleMusicClient() *AppleMusicClient { - return &AppleMusicClient{ - httpClient: NewMetadataHTTPClient(20 * time.Second), - } -} - -func appleMusicSearchResultMatches(result appleMusicSearchResult, trackName, artistName string, durationSec float64) bool { - if !lyricsSearchTitlesMatch(result.SongName, trackName, false) { - return false - } - if !lyricsSearchArtistsMatch(result.ArtistName, artistName) { - return false - } - if !lyricsSearchDurationMatches(float64(result.Duration)/1000.0, durationSec) { - return false - } - return true -} - -func selectBestAppleMusicSearchResult(results []appleMusicSearchResult, trackName, artistName string, durationSec float64) *appleMusicSearchResult { - best := selectBestLyricsCandidate(len(results), trackName, artistName, durationSec, func(i int) (string, string, float64, bool) { - result := &results[i] - ok := appleMusicSearchResultMatches(*result, trackName, artistName, durationSec) - return result.SongName, result.ArtistName, float64(result.Duration) / 1000.0, ok - }) - if best < 0 { - return nil - } - return &results[best] -} - -func (c *AppleMusicClient) getAppleMusicToken() (string, error) { - appleMusicTokenMu.Lock() - defer appleMusicTokenMu.Unlock() - - if appleMusicCachedToken != "" { - return appleMusicCachedToken, nil - } - - req, err := http.NewRequest("GET", "https://beta.music.apple.com", nil) - if err != nil { - return "", fmt.Errorf("failed to create apple music page request: %w", err) - } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") - - resp, err := c.httpClient.Do(req) - if err != nil { - return "", fmt.Errorf("failed to fetch apple music page: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", lyricsHTTPStatusError(resp.StatusCode, "apple music page returned HTTP %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read apple music page: %w", err) - } - - indexPath := regexp.MustCompile(`/assets/index~[^"' <]+\.js`).FindString(string(body)) - if indexPath == "" { - return "", fmt.Errorf("apple music index script not found") - } - - jsReq, err := http.NewRequest("GET", "https://beta.music.apple.com"+indexPath, nil) - if err != nil { - return "", fmt.Errorf("failed to create apple music script request: %w", err) - } - jsReq.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") - - jsResp, err := c.httpClient.Do(jsReq) - if err != nil { - return "", fmt.Errorf("failed to fetch apple music script: %w", err) - } - defer jsResp.Body.Close() - - if jsResp.StatusCode != http.StatusOK { - return "", lyricsHTTPStatusError(jsResp.StatusCode, "apple music script returned HTTP %d", jsResp.StatusCode) - } - - jsBody, err := io.ReadAll(jsResp.Body) - if err != nil { - return "", fmt.Errorf("failed to read apple music script: %w", err) - } - - token := regexp.MustCompile(`eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+`).FindString(string(jsBody)) - if token == "" { - return "", fmt.Errorf("apple music token not found") - } - - appleMusicCachedToken = token - return token, nil -} - -func clearAppleMusicToken() { - appleMusicTokenMu.Lock() - defer appleMusicTokenMu.Unlock() - appleMusicCachedToken = "" -} - -func (c *AppleMusicClient) searchSongWithToken(token, query string) ([]appleMusicSearchResult, error) { - params := url.Values{} - params.Set("term", query) - params.Set("types", "songs") - params.Set("limit", "25") - params.Set("l", "en-US") - params.Set("platform", "web") - params.Set("format[resources]", "map") - params.Set("include[songs]", "artists") - params.Set("extend", "artistUrl") - - searchURL := appleMusicCatalogBaseURL + "/search?" + params.Encode() - req, err := http.NewRequest("GET", searchURL, nil) - if err != nil { - return nil, fmt.Errorf("failed to create apple music catalog request: %w", err) - } - - req.Header.Set("Authorization", "Bearer "+token) - req.Header.Set("Origin", "https://music.apple.com") - req.Header.Set("Referer", "https://music.apple.com/") - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0") - req.Header.Set("Accept", "application/json") - req.Header.Set("Accept-Language", "en-US,en;q=0.5") - req.Header.Set("x-apple-renewal", "true") - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("apple music catalog search failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusUnauthorized { - return nil, errAppleMusicUnauthorized - } - if resp.StatusCode != http.StatusOK { - return nil, lyricsHTTPStatusError(resp.StatusCode, "apple music catalog search returned HTTP %d", resp.StatusCode) - } - - var searchResp appleMusicCatalogSearchResponse - if err := json.NewDecoder(resp.Body).Decode(&searchResp); err != nil { - return nil, fmt.Errorf("failed to decode apple music catalog response: %w", err) - } - - if searchResp.Results.Songs == nil || searchResp.Resources == nil { - return nil, nil - } - - results := make([]appleMusicSearchResult, 0, len(searchResp.Results.Songs.Data)) - for _, item := range searchResp.Results.Songs.Data { - detail, ok := searchResp.Resources.Songs[item.ID] - if !ok { - continue - } - attr := detail.Attributes - results = append(results, appleMusicSearchResult{ - ID: item.ID, - SongName: attr.Name, - ArtistName: attr.ArtistName, - AlbumName: attr.AlbumName, - Duration: attr.DurationInMillis, - }) - } - - return results, nil -} - -func (c *AppleMusicClient) SearchSong(trackName, artistName string, durationSec float64) (string, error) { - query := trackName + " " + artistName - if strings.TrimSpace(query) == "" { - return "", lyricsNotFoundErrorf("empty search query") - } - - token, err := c.getAppleMusicToken() - if err != nil { - return "", err - } - - searchResp, err := c.searchSongWithToken(token, strings.TrimSpace(query)) - if errors.Is(err, errAppleMusicUnauthorized) { - clearAppleMusicToken() - token, tokenErr := c.getAppleMusicToken() - if tokenErr != nil { - return "", tokenErr - } - searchResp, err = c.searchSongWithToken(token, strings.TrimSpace(query)) - } - if err != nil { - return "", err - } - - best := selectBestAppleMusicSearchResult(searchResp, trackName, artistName, durationSec) - if best == nil || strings.TrimSpace(best.ID) == "" { - return "", lyricsNotFoundErrorf("no songs found on apple music") - } - - return strings.TrimSpace(best.ID), nil -} - -func (c *AppleMusicClient) FetchLyricsByID(songID string) (string, error) { - lyricsURL := fmt.Sprintf("https://lyrics.paxsenix.org/apple-music/lyrics?id=%s", songID) - - req, err := http.NewRequest("GET", lyricsURL, nil) - if err != nil { - return "", fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("User-Agent", appUserAgent()) - req.Header.Set("Accept", "application/json") - - resp, err := c.httpClient.Do(req) - if err != nil { - return "", fmt.Errorf("apple music lyrics fetch failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return "", lyricsHTTPStatusError(resp.StatusCode, "apple music lyrics proxy returned HTTP %d", resp.StatusCode) - } - - bodyBytes, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read lyrics response: %w", err) - } - - bodyStr := strings.TrimSpace(string(bodyBytes)) - if bodyStr == "" { - return "", fmt.Errorf("empty lyrics response from apple music") - } - - return bodyStr, nil -} - -func formatPaxLyricsToLRC(rawJSON string, multiPersonWordByWord bool, preserveWordTiming bool) (string, error) { - var stringPayload string - if err := json.Unmarshal([]byte(rawJSON), &stringPayload); err == nil { - stringPayload = strings.TrimSpace(stringPayload) - if stringPayload != "" { - return stringPayload, nil - } - } - - var paxResp paxResponse - if err := json.Unmarshal([]byte(rawJSON), &paxResp); err == nil && - (paxResp.Content != nil || - strings.TrimSpace(paxResp.ELRCMultiPerson) != "" || - strings.TrimSpace(paxResp.ELRC) != "" || - strings.TrimSpace(paxResp.Plain) != "" || - strings.TrimSpace(paxResp.TTMLContent) != "") { - if preserveWordTiming && multiPersonWordByWord && strings.TrimSpace(paxResp.ELRCMultiPerson) != "" { - return strings.TrimSpace(paxResp.ELRCMultiPerson), nil - } - if preserveWordTiming && strings.TrimSpace(paxResp.ELRC) != "" { - return strings.TrimSpace(paxResp.ELRC), nil - } - if strings.TrimSpace(paxResp.Plain) != "" && len(paxResp.Content) == 0 { - return strings.TrimSpace(paxResp.Plain), nil - } - if len(paxResp.Content) == 0 { - return "", fmt.Errorf("unsupported apple music lyrics payload") - } - return formatPaxContent(paxResp.Type, paxResp.Content, multiPersonWordByWord, preserveWordTiming), nil - } - - var directLyrics []paxLyrics - if err := json.Unmarshal([]byte(rawJSON), &directLyrics); err == nil && len(directLyrics) > 0 { - return formatPaxContent("Syllable", directLyrics, multiPersonWordByWord, preserveWordTiming), nil - } - - return "", fmt.Errorf("failed to parse pax lyrics response") -} - -func appendPaxLyricDetail(builder *strings.Builder, details []paxLyricDetail, preserveWordTiming bool) { - lastStart := "" - - for _, syllable := range details { - if preserveWordTiming && syllable.Timestamp != nil { - start := fmt.Sprintf("<%s>", msToLRCTimestampInline(int64(*syllable.Timestamp))) - if start != lastStart { - builder.WriteString(start) - lastStart = start - } - } - - builder.WriteString(syllable.Text) - if !syllable.Part { - builder.WriteString(" ") - } - - if preserveWordTiming && syllable.EndTime != nil { - builder.WriteString(fmt.Sprintf("<%s>", msToLRCTimestampInline(int64(*syllable.EndTime)))) - } - } -} - -func formatPaxContent(lyricsType string, content []paxLyrics, multiPersonWordByWord bool, preserveWordTiming bool) string { - var sb strings.Builder - - for i, line := range content { - if i > 0 { - sb.WriteString("\n") - } - - timestamp := msToLRCTimestamp(int64(line.Timestamp)) - - if strings.EqualFold(lyricsType, "Syllable") { - sb.WriteString(timestamp) - if multiPersonWordByWord { - if line.OppositeTurn { - sb.WriteString("v2:") - } else { - sb.WriteString("v1:") - } - } - - appendPaxLyricDetail(&sb, line.Text, preserveWordTiming) - - if line.Background && multiPersonWordByWord && len(line.BackgroundText) > 0 { - sb.WriteString("\n[bg:") - appendPaxLyricDetail(&sb, line.BackgroundText, preserveWordTiming) - sb.WriteString("]") - } - } else { - if len(line.Text) > 0 { - sb.WriteString(timestamp) - sb.WriteString(line.Text[0].Text) - } - } - } - - return strings.TrimSpace(sb.String()) -} - -func (c *AppleMusicClient) FetchLyrics( - trackName, - artistName string, - durationSec float64, - multiPersonWordByWord bool, - preserveWordTiming bool, -) (*LyricsResponse, error) { - songID, err := c.SearchSong(trackName, artistName, durationSec) - if err != nil { - return nil, err - } - - rawLyrics, err := c.FetchLyricsByID(songID) - if err != nil { - return nil, err - } - if errMsg, isErrorPayload := detectLyricsErrorPayload(rawLyrics); isErrorPayload { - return nil, classifyLyricsPayloadError(0, errMsg, "apple music proxy returned non-lyric payload: %s", errMsg) - } - - lrcText, err := formatPaxLyricsToLRC(rawLyrics, multiPersonWordByWord, preserveWordTiming) - if err != nil { - trimmedRaw := strings.TrimSpace(rawLyrics) - if strings.HasPrefix(trimmedRaw, "{") || strings.HasPrefix(trimmedRaw, "[") { - return nil, err - } - lrcText = rawLyrics - } - - if resp := lyricsResponseFromLRCText(lrcText, "Apple Music", "Apple Music"); resp != nil { - return resp, nil - } - return nil, lyricsNotFoundErrorf("no lyrics found on apple music") -} diff --git a/go_backend/lyrics_apple_test.go b/go_backend/lyrics_apple_test.go deleted file mode 100644 index b74d208f..00000000 --- a/go_backend/lyrics_apple_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package gobackend - -import "testing" - -func TestSelectBestAppleMusicSearchResultRejectsWrongSongWithMatchingArtistAndDuration(t *testing.T) { - results := []appleMusicSearchResult{ - { - ID: "azul", - SongName: "Azul", - ArtistName: "Guru Randhawa", - Duration: 186000, - }, - } - - best := selectBestAppleMusicSearchResult( - results, - "SIX", - "Guru Randhawa, Kiran Bajwa, Gurjit Gill & Lavish Dhiman", - 186, - ) - if best != nil { - t.Fatalf("expected the unrelated Azul result to be rejected, got %#v", best) - } -} - -func TestSelectBestAppleMusicSearchResultRequiresAvailableIdentitySignals(t *testing.T) { - tests := []struct { - name string - result appleMusicSearchResult - }{ - { - name: "wrong artist", - result: appleMusicSearchResult{ - ID: "wrong-artist", - SongName: "SIX", - ArtistName: "Different Artist", - Duration: 186000, - }, - }, - { - name: "wrong duration", - result: appleMusicSearchResult{ - ID: "wrong-duration", - SongName: "SIX", - ArtistName: "Guru Randhawa", - Duration: 240000, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - best := selectBestAppleMusicSearchResult( - []appleMusicSearchResult{test.result}, - "SIX", - "Guru Randhawa", - 186, - ) - if best != nil { - t.Fatalf("expected mismatched result to be rejected, got %#v", best) - } - }) - } -} - -func TestSelectBestAppleMusicSearchResultAcceptsCompatibleMetadata(t *testing.T) { - results := []appleMusicSearchResult{ - { - ID: "azul", - SongName: "Azul", - ArtistName: "Guru Randhawa", - Duration: 186000, - }, - { - ID: "six", - SongName: "SIX (feat. Kiran Bajwa)", - ArtistName: "Guru Randhawa & Kiran Bajwa", - Duration: 187000, - }, - } - - best := selectBestAppleMusicSearchResult( - results, - "SIX", - "Guru Randhawa, Kiran Bajwa, Gurjit Gill & Lavish Dhiman", - 186, - ) - if best == nil || best.ID != "six" { - t.Fatalf("expected the compatible SIX result, got %#v", best) - } -} diff --git a/go_backend/lyrics_config.go b/go_backend/lyrics_config.go deleted file mode 100644 index cf0de0a2..00000000 --- a/go_backend/lyrics_config.go +++ /dev/null @@ -1,562 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "errors" - "os" - "path/filepath" - "regexp" - "strings" - "sync" - "time" -) - -const ( - lyricsProviderUnavailableCooldown = 10 * time.Minute - lyricsProviderParallelism = 3 - lyricsProviderPriorityGrace = 5000 * time.Millisecond -) - -const ( - LyricsProviderLRCLIB = "lrclib" - LyricsProviderNetease = "netease" - LyricsProviderMusixmatch = "musixmatch" - LyricsProviderAppleMusic = "apple_music" - LyricsProviderQQMusic = "qqmusic" - LyricsProviderSpotify = "spotify" - LyricsProviderDeezer = "deezer" - LyricsProviderYouTube = "youtube" - LyricsProviderKugou = "kugou" - LyricsProviderGenius = "genius" - LyricsProviderLyricsPlus = "lyricsplus" -) - -var DefaultLyricsProviders = []string{ - LyricsProviderLRCLIB, - LyricsProviderAppleMusic, -} - -var ( - lyricsProvidersMu sync.RWMutex - lyricsProviders []string // ordered list of enabled providers - appVersionMu sync.RWMutex - appVersion string -) - -type lyricsProviderHealthEntry struct { - unavailableUntil time.Time - reason string -} - -type lyricsProviderSearchRequest struct { - spotifyID string - trackName string - artistName string - primaryArtist string - simplifiedTrack string - durationSec float64 - fetchOptions LyricsFetchOptions -} - -type lyricsProviderSearchResult struct { - index int - providerName string - lyrics *LyricsResponse - err error -} - -var ( - lyricsProviderHealthMu sync.RWMutex - lyricsProviderHealth = make(map[string]lyricsProviderHealthEntry) -) - -func SetAppVersion(version string) { - normalized := strings.TrimSpace(version) - - appVersionMu.Lock() - defer appVersionMu.Unlock() - appVersion = normalized -} - -func GetAppVersion() string { - appVersionMu.RLock() - defer appVersionMu.RUnlock() - return appVersion -} - -func appUserAgent() string { - version := GetAppVersion() - - if version == "" { - return "SpotiFLAC-Mobile" - } - - return "SpotiFLAC-Mobile/" + version -} - -type LyricsFetchOptions struct { - IncludeTranslationNetease bool `json:"include_translation_netease"` - IncludeRomanizationNetease bool `json:"include_romanization_netease"` - MultiPersonWordByWord bool `json:"multi_person_word_by_word"` - AppleElrcWordSync bool `json:"apple_elrc_word_sync"` - MusixmatchLanguage string `json:"musixmatch_language,omitempty"` -} - -var defaultLyricsFetchOptions = LyricsFetchOptions{ - IncludeTranslationNetease: false, - IncludeRomanizationNetease: false, - MultiPersonWordByWord: true, - AppleElrcWordSync: false, - MusixmatchLanguage: "", -} - -var instrumentalTrackPattern = regexp.MustCompile(`(?i)(?:^|[\s\[(\-])(?:instrumental|inst\.?)(?:[\s\])]|$)`) - -var ( - lyricsFetchOptionsMu sync.RWMutex - lyricsFetchOptions = defaultLyricsFetchOptions -) - -func SetLyricsProviderOrder(providers []string) { - lyricsProvidersMu.Lock() - - if len(providers) == 0 { - changed := len(lyricsProviders) != 0 - lyricsProviders = nil - lyricsProvidersMu.Unlock() - clearLyricsProviderHealth() - if changed { - globalLyricsCache.ClearAll() - } - return - } - - validNames := map[string]bool{ - LyricsProviderLRCLIB: true, - LyricsProviderNetease: true, - LyricsProviderMusixmatch: true, - LyricsProviderAppleMusic: true, - LyricsProviderQQMusic: true, - LyricsProviderSpotify: true, - LyricsProviderDeezer: true, - LyricsProviderYouTube: true, - LyricsProviderKugou: true, - LyricsProviderGenius: true, - LyricsProviderLyricsPlus: true, - } - - valid := make([]string, 0, len(providers)) - seen := make(map[string]struct{}, len(providers)) - for _, p := range providers { - normalized := strings.ToLower(strings.TrimSpace(p)) - isExtension := strings.HasPrefix(normalized, "extension:") && - strings.TrimSpace(strings.TrimPrefix(normalized, "extension:")) != "" - if !validNames[normalized] && !isExtension { - continue - } - if _, exists := seen[normalized]; exists { - continue - } - seen[normalized] = struct{}{} - valid = append(valid, normalized) - } - - changed := !equalLyricsProviderOrders(lyricsProviders, valid) - lyricsProviders = valid - lyricsProvidersMu.Unlock() - clearLyricsProviderHealth() - if changed { - globalLyricsCache.ClearAll() - } - GoLog("[Lyrics] Provider order set to: %v\n", valid) -} - -func equalLyricsProviderOrders(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} - -func clearLyricsProviderHealth() { - lyricsProviderHealthMu.Lock() - defer lyricsProviderHealthMu.Unlock() - lyricsProviderHealth = make(map[string]lyricsProviderHealthEntry) -} - -func lyricsProviderHealthKey(providerName string) string { - return strings.ToLower(strings.TrimSpace(providerName)) -} - -func shouldSkipLyricsProvider(providerName string) (bool, time.Duration, string) { - key := lyricsProviderHealthKey(providerName) - if key == "" { - return false, 0, "" - } - - now := time.Now() - lyricsProviderHealthMu.RLock() - entry, ok := lyricsProviderHealth[key] - lyricsProviderHealthMu.RUnlock() - if !ok { - return false, 0, "" - } - if !now.Before(entry.unavailableUntil) { - lyricsProviderHealthMu.Lock() - if current, exists := lyricsProviderHealth[key]; exists && !now.Before(current.unavailableUntil) { - delete(lyricsProviderHealth, key) - } - lyricsProviderHealthMu.Unlock() - return false, 0, "" - } - return true, time.Until(entry.unavailableUntil), entry.reason -} - -func markLyricsProviderAvailable(providerName string) { - key := lyricsProviderHealthKey(providerName) - if key == "" { - return - } - lyricsProviderHealthMu.Lock() - delete(lyricsProviderHealth, key) - lyricsProviderHealthMu.Unlock() -} - -func markLyricsProviderUnavailable(providerName string, err error) { - if err == nil || !isLyricsProviderUnavailableError(err) { - return - } - key := lyricsProviderHealthKey(providerName) - if key == "" { - return - } - reason := strings.TrimSpace(err.Error()) - if len(reason) > 160 { - reason = reason[:160] - } - unavailableUntil := time.Now().Add(lyricsProviderUnavailableCooldown) - - lyricsProviderHealthMu.Lock() - lyricsProviderHealth[key] = lyricsProviderHealthEntry{ - unavailableUntil: unavailableUntil, - reason: reason, - } - lyricsProviderHealthMu.Unlock() - GoLog("[Lyrics] Provider %s marked unavailable for %s: %s\n", providerName, lyricsProviderUnavailableCooldown, reason) -} - -// isLyricsProviderUnavailableError reports whether err is a provider/API-level -// failure that should temporarily disable a lyrics source. Providers classify -// their failures with the typed errors in lyrics_errors.go at the point of -// origin; transport failures are handled by isConnectivityFailure. -func isLyricsProviderUnavailableError(err error) bool { - if err == nil { - return false - } - if errors.Is(err, errLyricsNotFound) { - return false - } - if errors.Is(err, errLyricsServiceUnavailable) { - return true - } - return isConnectivityFailure(err) -} - -func GetLyricsProviderOrder() []string { - lyricsProvidersMu.RLock() - defer lyricsProvidersMu.RUnlock() - - if len(lyricsProviders) == 0 { - result := make([]string, len(DefaultLyricsProviders)) - copy(result, DefaultLyricsProviders) - return result - } - - result := make([]string, len(lyricsProviders)) - copy(result, lyricsProviders) - return result -} - -func GetAvailableLyricsProviders() []map[string]any { - return []map[string]any{ - {"id": LyricsProviderLRCLIB, "name": "LRCLIB", "has_proxy_dependency": false, "description": "Open-source synced lyrics database"}, - {"id": LyricsProviderNetease, "name": "Netease", "has_proxy_dependency": true, "description": "NetEase Cloud Music lyrics"}, - {"id": LyricsProviderMusixmatch, "name": "Musixmatch", "has_proxy_dependency": true, "description": "Musixmatch lyrics"}, - {"id": LyricsProviderAppleMusic, "name": "Apple Music", "has_proxy_dependency": true, "description": "Apple Music synced lyrics"}, - {"id": LyricsProviderQQMusic, "name": "QQ Music", "has_proxy_dependency": false, "description": "Direct QQ Music line-synced lyrics"}, - {"id": LyricsProviderSpotify, "name": "Spotify", "has_proxy_dependency": true, "description": "Spotify synced lyrics"}, - {"id": LyricsProviderDeezer, "name": "Deezer", "has_proxy_dependency": true, "description": "Deezer lyrics"}, - {"id": LyricsProviderYouTube, "name": "YouTube", "has_proxy_dependency": true, "description": "YouTube lyrics"}, - {"id": LyricsProviderKugou, "name": "Kugou", "has_proxy_dependency": false, "description": "Direct Kugou synced lyrics"}, - {"id": LyricsProviderGenius, "name": "Genius", "has_proxy_dependency": false, "description": "Direct Genius lyrics"}, - {"id": LyricsProviderLyricsPlus, "name": "LyricsPlus", "has_proxy_dependency": true, "description": "Word-by-word karaoke lyrics (Apple/Musixmatch/Spotify/QQ)"}, - } -} - -func normalizeLyricsFetchOptions(opts LyricsFetchOptions) LyricsFetchOptions { - opts.MusixmatchLanguage = strings.ToLower(strings.TrimSpace(opts.MusixmatchLanguage)) - opts.MusixmatchLanguage = regexp.MustCompile(`[^a-z0-9\-_]`).ReplaceAllString(opts.MusixmatchLanguage, "") - if len(opts.MusixmatchLanguage) > 16 { - opts.MusixmatchLanguage = opts.MusixmatchLanguage[:16] - } - return opts -} - -func SetLyricsFetchOptions(opts LyricsFetchOptions) { - normalized := normalizeLyricsFetchOptions(opts) - - lyricsFetchOptionsMu.Lock() - changed := lyricsFetchOptions != normalized - lyricsFetchOptions = normalized - lyricsFetchOptionsMu.Unlock() - - if changed { - globalLyricsCache.ClearAll() - } - - GoLog("[Lyrics] Fetch options set: translation=%v romanization=%v multi_person=%v apple_elrc=%v musixmatch_lang=%q\n", - normalized.IncludeTranslationNetease, - normalized.IncludeRomanizationNetease, - normalized.MultiPersonWordByWord, - normalized.AppleElrcWordSync, - normalized.MusixmatchLanguage, - ) -} - -func GetLyricsFetchOptions() LyricsFetchOptions { - lyricsFetchOptionsMu.RLock() - defer lyricsFetchOptionsMu.RUnlock() - return lyricsFetchOptions -} - -type lyricsCacheEntry struct { - response *LyricsResponse - expiresAt time.Time -} - -type lyricsCache struct { - mu sync.RWMutex - cache map[string]*lyricsCacheEntry - persistencePath string - persistGeneration uint64 - persistencePending bool -} - -var globalLyricsCache = &lyricsCache{ - cache: make(map[string]*lyricsCacheEntry), -} - -func (c *lyricsCache) generateKey(artist, track string, durationSec float64) string { - return lyricsFetchCacheKey("", track, artist, durationSec) -} - -func (c *lyricsCache) Get(artist, track string, durationSec float64) (*LyricsResponse, bool) { - key := c.generateKey(artist, track, durationSec) - c.mu.RLock() - defer c.mu.RUnlock() - - entry, exists := c.cache[key] - if !exists { - return nil, false - } - - if time.Now().After(entry.expiresAt) { - return nil, false - } - - responseCopy := *entry.response - responseCopy.Lines = append([]LyricsLine(nil), entry.response.Lines...) - return &responseCopy, true -} - -const lyricsCacheMaxEntries = 500 - -func (c *lyricsCache) Set(artist, track string, durationSec float64, response *LyricsResponse) { - key := c.generateKey(artist, track, durationSec) - c.mu.Lock() - defer c.mu.Unlock() - - // Bound the cache: without eviction a long session accumulates every - // looked-up track's full lyrics forever. - if len(c.cache) >= lyricsCacheMaxEntries { - now := time.Now() - for key, entry := range c.cache { - if now.After(entry.expiresAt) { - delete(c.cache, key) - } - } - for len(c.cache) >= lyricsCacheMaxEntries { - var oldestKey string - var oldestAt time.Time - for key, entry := range c.cache { - if oldestKey == "" || entry.expiresAt.Before(oldestAt) { - oldestKey = key - oldestAt = entry.expiresAt - } - } - delete(c.cache, oldestKey) - } - } - - c.cache[key] = &lyricsCacheEntry{ - response: cloneLyricsResponse(response), - expiresAt: time.Now().Add(lyricsCacheTTL), - } - c.schedulePersistenceLocked() -} - -func (c *lyricsCache) CleanExpired() int { - c.mu.Lock() - defer c.mu.Unlock() - - now := time.Now() - cleaned := 0 - for key, entry := range c.cache { - if now.After(entry.expiresAt) { - delete(c.cache, key) - cleaned++ - } - } - return cleaned -} - -func (c *lyricsCache) Size() int { - c.mu.RLock() - defer c.mu.RUnlock() - return len(c.cache) -} - -func (c *lyricsCache) ClearAll() int { - c.mu.Lock() - defer c.mu.Unlock() - - cleared := len(c.cache) - c.cache = make(map[string]*lyricsCacheEntry) - c.schedulePersistenceLocked() - return cleared -} - -// DropMemory releases the in-memory snapshot without deleting the persistent -// cache. It is used for OS memory-pressure handling; a later app start can -// still restore successful lyrics lookups from disk. -func (c *lyricsCache) DropMemory() int { - c.mu.Lock() - defer c.mu.Unlock() - cleared := len(c.cache) - c.cache = make(map[string]*lyricsCacheEntry) - return cleared -} - -type persistedLyricsCacheEntry struct { - Response *LyricsResponse `json:"response"` - ExpiresAt int64 `json:"expires_at"` -} - -type persistedLyricsCache struct { - Version int `json:"version"` - Entries map[string]persistedLyricsCacheEntry `json:"entries"` -} - -func cloneLyricsResponse(response *LyricsResponse) *LyricsResponse { - if response == nil { - return nil - } - copy := *response - copy.Lines = append([]LyricsLine(nil), response.Lines...) - return © -} - -func (c *lyricsCache) SetPersistencePath(path string) { - path = filepath.Clean(strings.TrimSpace(path)) - if path == "." || path == "" { - return - } - - data, err := os.ReadFile(path) - loaded := make(map[string]*lyricsCacheEntry) - if err == nil { - var persisted persistedLyricsCache - if json.Unmarshal(data, &persisted) == nil && persisted.Version == 1 { - now := time.Now() - for key, entry := range persisted.Entries { - expiresAt := time.Unix(entry.ExpiresAt, 0) - if entry.Response == nil || !now.Before(expiresAt) { - continue - } - loaded[key] = &lyricsCacheEntry{ - response: cloneLyricsResponse(entry.Response), - expiresAt: expiresAt, - } - if len(loaded) >= lyricsCacheMaxEntries { - break - } - } - } - } - - c.mu.Lock() - c.persistencePath = path - for key, entry := range loaded { - if _, exists := c.cache[key]; !exists { - c.cache[key] = entry - } - } - c.mu.Unlock() -} - -func (c *lyricsCache) schedulePersistenceLocked() { - if c.persistencePath == "" { - return - } - c.persistGeneration++ - if c.persistencePending { - return - } - c.persistencePending = true - go c.persistAfterDebounce() -} - -func (c *lyricsCache) persistAfterDebounce() { - time.Sleep(500 * time.Millisecond) - for { - c.mu.RLock() - path := c.persistencePath - generation := c.persistGeneration - snapshot := persistedLyricsCache{ - Version: 1, - Entries: make(map[string]persistedLyricsCacheEntry, len(c.cache)), - } - for key, entry := range c.cache { - snapshot.Entries[key] = persistedLyricsCacheEntry{ - Response: cloneLyricsResponse(entry.response), - ExpiresAt: entry.expiresAt.Unix(), - } - } - c.mu.RUnlock() - - if data, err := json.Marshal(snapshot); err == nil { - if err := os.MkdirAll(filepath.Dir(path), 0700); err == nil { - tempPath := path + ".tmp" - if os.WriteFile(tempPath, data, 0600) == nil { - if err := os.Rename(tempPath, path); err != nil { - _ = os.Remove(tempPath) - } - } - } - } - - c.mu.Lock() - if generation == c.persistGeneration { - c.persistencePending = false - c.mu.Unlock() - return - } - c.mu.Unlock() - time.Sleep(100 * time.Millisecond) - } -} diff --git a/go_backend/lyrics_errors.go b/go_backend/lyrics_errors.go deleted file mode 100644 index 45ccd88c..00000000 --- a/go_backend/lyrics_errors.go +++ /dev/null @@ -1,100 +0,0 @@ -package gobackend - -import ( - "errors" - "fmt" - "net/http" - "strings" -) - -// Sentinel classifications for lyrics provider failures. Providers wrap their -// errors with the typed constructors below at the point where the failure kind -// is known, so isLyricsProviderUnavailableError can classify with errors.Is -// instead of parsing error text. -var ( - errLyricsNotFound = errors.New("lyrics not found") - errLyricsServiceUnavailable = errors.New("lyrics service unavailable") -) - -// lyricsNotFoundError: the provider responded but has nothing for this track -// (including "nothing to search for" inputs). Never disables the provider. -type lyricsNotFoundError struct{ err error } - -func (e *lyricsNotFoundError) Error() string { return e.err.Error() } -func (e *lyricsNotFoundError) Unwrap() error { return e.err } -func (e *lyricsNotFoundError) Is(target error) bool { return target == errLyricsNotFound } - -func lyricsNotFoundErrorf(format string, args ...any) error { - return &lyricsNotFoundError{err: fmt.Errorf(format, args...)} -} - -// lyricsServiceUnavailableError: a provider/API-level failure that should -// temporarily disable the lyrics source (see markLyricsProviderUnavailable). -type lyricsServiceUnavailableError struct{ err error } - -func (e *lyricsServiceUnavailableError) Error() string { return e.err.Error() } -func (e *lyricsServiceUnavailableError) Unwrap() error { return e.err } -func (e *lyricsServiceUnavailableError) Is(target error) bool { - return target == errLyricsServiceUnavailable -} - -func lyricsServiceUnavailableErrorf(format string, args ...any) error { - return &lyricsServiceUnavailableError{err: fmt.Errorf(format, args...)} -} - -// lyricsHTTPStatusError classifies an upstream HTTP status at the response -// site: 429 and 5xx mean the service is temporarily unusable; anything else -// stays a plain error. -func lyricsHTTPStatusError(statusCode int, format string, args ...any) error { - if statusCode == http.StatusTooManyRequests || statusCode >= 500 { - return lyricsServiceUnavailableErrorf(format, args...) - } - return fmt.Errorf(format, args...) -} - -// Error payloads from third-party lyrics proxies arrive as free text, often -// with HTTP 200 (see detectLyricsErrorPayload). Their meaning can only be -// recognized from the message, so this is the one place lyrics failures are -// still classified by substring. Errors generated by our own code must use -// the typed constructors above instead. -var ( - lyricsPayloadNotFoundSignals = []string{ - "lyrics not found", - "no lyrics found", - "no songs found", - "not found", - } - lyricsPayloadUnavailableSignals = []string{ - "rate limit", - "too many requests", - "operation too frequent", - "操作频繁", - "missing required parameters", - } -) - -func lyricsPayloadIndicatesNotFound(payloadMsg string) bool { - msg := strings.ToLower(payloadMsg) - for _, signal := range lyricsPayloadNotFoundSignals { - if strings.Contains(msg, signal) { - return true - } - } - return false -} - -// classifyLyricsPayloadError builds a provider error from an upstream error -// payload, typed by what the payload message describes. statusCode is the -// HTTP status of the response (0 when it arrived with HTTP 200). -func classifyLyricsPayloadError(statusCode int, payloadMsg string, format string, args ...any) error { - if lyricsPayloadIndicatesNotFound(payloadMsg) { - return lyricsNotFoundErrorf(format, args...) - } - msg := strings.ToLower(payloadMsg) - for _, signal := range lyricsPayloadUnavailableSignals { - if strings.Contains(msg, signal) { - return lyricsServiceUnavailableErrorf(format, args...) - } - } - return lyricsHTTPStatusError(statusCode, format, args...) -} diff --git a/go_backend/lyrics_genius.go b/go_backend/lyrics_genius.go deleted file mode 100644 index 784b99e5..00000000 --- a/go_backend/lyrics_genius.go +++ /dev/null @@ -1,123 +0,0 @@ -package gobackend - -import ( - "fmt" - "io" - "net/http" - "strings" - - "golang.org/x/net/html" -) - -const maxGeniusPageBytes = 8 << 20 - -func geniusLyricsContainer(node *html.Node) bool { - if node.Type != html.ElementNode || node.Data != "div" { - return false - } - for _, attr := range node.Attr { - if attr.Key == "data-lyrics-container" && attr.Val == "true" { - return true - } - } - return false -} - -func geniusExcludedNode(node *html.Node) bool { - for _, attr := range node.Attr { - if attr.Key == "data-exclude-from-selection" && attr.Val == "true" { - return true - } - } - return false -} - -func appendGeniusText(builder *strings.Builder, node *html.Node) { - if geniusExcludedNode(node) { - return - } - if node.Type == html.TextNode { - builder.WriteString(node.Data) - return - } - if node.Type == html.ElementNode && node.Data == "br" { - builder.WriteByte('\n') - return - } - for child := node.FirstChild; child != nil; child = child.NextSibling { - appendGeniusText(builder, child) - } -} - -func geniusLyricsFromHTML(body io.Reader) (string, error) { - document, err := html.Parse(body) - if err != nil { - return "", fmt.Errorf("failed to parse Genius page: %w", err) - } - var containers []*html.Node - var walk func(*html.Node) - walk = func(node *html.Node) { - if geniusLyricsContainer(node) { - containers = append(containers, node) - } - for child := node.FirstChild; child != nil; child = child.NextSibling { - walk(child) - } - } - walk(document) - if len(containers) == 0 { - return "", lyricsNotFoundErrorf("Genius page has no lyrics container") - } - - // Genius renders each verse/chorus as a separate lyrics container. Preserve - // document order and join all usable containers into one LRC/plain payload. - var sections []string - for _, container := range containers { - var builder strings.Builder - appendGeniusText(&builder, container) - candidate := strings.TrimSpace(strings.ReplaceAll(builder.String(), "\u00a0", " ")) - if rawLyricsHasUsableContent(candidate) { - sections = append(sections, candidate) - } - } - if len(sections) == 0 { - return "", lyricsNotFoundErrorf("Genius page returned empty lyrics") - } - return strings.Join(sections, "\n"), nil -} - -func (c *GeniusLyricsClient) fetchLyricsFromPage(pageURL string) (*LyricsResponse, error) { - pageURL = strings.TrimSpace(pageURL) - if pageURL == "" { - return nil, lyricsNotFoundErrorf("empty Genius lyrics URL") - } - req, err := http.NewRequest(http.MethodGet, pageURL, nil) - if err != nil { - return nil, fmt.Errorf("failed to create Genius page request: %w", err) - } - req.Header.Set("Accept", "text/html,application/xhtml+xml") - req.Header.Set("Accept-Language", "en-US,en;q=0.9") - req.Header.Set("User-Agent", getRandomUserAgent()) - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("Genius page request failed: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, lyricsServiceUnavailableErrorf("Genius page returned HTTP %d", resp.StatusCode) - } - data, err := io.ReadAll(io.LimitReader(resp.Body, maxGeniusPageBytes+1)) - if err != nil { - return nil, fmt.Errorf("failed to read Genius page: %w", err) - } - if len(data) > maxGeniusPageBytes { - return nil, lyricsServiceUnavailableErrorf("Genius page exceeds %d bytes", maxGeniusPageBytes) - } - lrc, err := geniusLyricsFromHTML(strings.NewReader(string(data))) - if err != nil { - return nil, err - } - lyrics := lyricsResponseFromText(lrc, "Genius") - lyrics.Source = "Genius Direct" - return lyrics, nil -} diff --git a/go_backend/lyrics_kugou.go b/go_backend/lyrics_kugou.go deleted file mode 100644 index 429f7b85..00000000 --- a/go_backend/lyrics_kugou.go +++ /dev/null @@ -1,227 +0,0 @@ -package gobackend - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "io" - "math" - "net/http" - "net/url" - "strconv" - "strings" - "time" -) - -const maxKugouLyricsResponseBytes = 2 << 20 - -type KugouLyricsClient struct { - httpClient *http.Client -} - -type kugouLyricsSearchResult struct { - ID string `json:"id"` - AccessKey string `json:"accesskey"` - Title string `json:"song"` - Artist string `json:"singer"` - Duration float64 `json:"duration"` -} - -type kugouLyricsSearchResponse struct { - Status int `json:"status"` - ErrorCode int `json:"errcode"` - Error string `json:"errmsg"` - Candidates []kugouLyricsSearchResult `json:"candidates"` -} - -type kugouLyricsDownloadResponse struct { - Status int `json:"status"` - ErrorCode int `json:"error_code"` - Info string `json:"info"` - Content string `json:"content"` -} - -func NewKugouLyricsClient() *KugouLyricsClient { - return &KugouLyricsClient{httpClient: NewMetadataHTTPClient(15 * time.Second)} -} - -func fetchKugouLyricsBody(httpClient *http.Client, endpoint string, params url.Values) ([]byte, error) { - req, err := http.NewRequest(http.MethodGet, endpoint+"?"+params.Encode(), nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", appUserAgent()) - - resp, err := httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, lyricsServiceUnavailableErrorf("HTTP %d", resp.StatusCode) - } - - body, err := io.ReadAll(io.LimitReader(resp.Body, maxKugouLyricsResponseBytes+1)) - if err != nil { - return nil, fmt.Errorf("failed to read response: %w", err) - } - if len(body) > maxKugouLyricsResponseBytes { - return nil, lyricsServiceUnavailableErrorf( - "response exceeds %d bytes", - maxKugouLyricsResponseBytes, - ) - } - if strings.TrimSpace(string(body)) == "" { - return nil, lyricsServiceUnavailableErrorf("empty response") - } - return body, nil -} - -func (c *KugouLyricsClient) searchSong( - trackName, - artistName string, - durationSec float64, -) (*kugouLyricsSearchResult, error) { - query := strings.TrimSpace(artistName + " - " + trackName) - if query == "" { - return nil, lyricsNotFoundErrorf("empty search query") - } - - params := url.Values{ - "ver": {"1"}, - "man": {"yes"}, - "client": {"pc"}, - "keyword": {query}, - "duration": {strconv.FormatInt(int64(math.Round(durationSec*1000)), 10)}, - "hash": {""}, - } - raw, err := fetchKugouLyricsBody( - c.httpClient, - "https://lyrics.kugou.com/search", - params, - ) - if err != nil { - return nil, fmt.Errorf("kugou search failed: %w", err) - } - - var response kugouLyricsSearchResponse - if err := json.Unmarshal(raw, &response); err != nil { - return nil, fmt.Errorf("failed to decode kugou search: %w", err) - } - // KuGou uses errcode=200 for a successful search response, while some - // mirrors omit the field (or return zero). Treat both success forms as - // valid and only reject explicit non-success codes. - if response.Status != http.StatusOK || - (response.ErrorCode != 0 && response.ErrorCode != http.StatusOK) { - message := strings.TrimSpace(response.Error) - if message == "" { - message = fmt.Sprintf( - "status %d/error %d", - response.Status, - response.ErrorCode, - ) - } - return nil, lyricsServiceUnavailableErrorf("%s", message) - } - - best := selectBestKugouLyricsSearchResult( - response.Candidates, - trackName, - artistName, - durationSec, - ) - if best == nil || - strings.TrimSpace(best.ID) == "" || - strings.TrimSpace(best.AccessKey) == "" { - return nil, lyricsNotFoundErrorf("no matching song found on kugou") - } - return best, nil -} - -func selectBestKugouLyricsSearchResult( - results []kugouLyricsSearchResult, - trackName, - artistName string, - durationSec float64, -) *kugouLyricsSearchResult { - best := selectBestLyricsCandidate( - len(results), - trackName, - artistName, - durationSec, - func(i int) (string, string, float64, bool) { - result := &results[i] - durationSeconds := result.Duration / 1000 - matches := lyricsSearchTitlesMatch(result.Title, trackName, false) && - lyricsSearchArtistsMatch(result.Artist, artistName) && - lyricsSearchDurationMatches(durationSeconds, durationSec) - return result.Title, result.Artist, durationSeconds, matches - }, - ) - if best < 0 { - return nil - } - return &results[best] -} - -func (c *KugouLyricsClient) FetchLyrics( - trackName, - artistName string, - durationSec float64, -) (*LyricsResponse, error) { - match, err := c.searchSong(trackName, artistName, durationSec) - if err != nil { - return nil, err - } - - params := url.Values{ - "ver": {"1"}, - "client": {"pc"}, - "id": {match.ID}, - "accesskey": {match.AccessKey}, - "fmt": {"lrc"}, - "charset": {"utf8"}, - } - raw, err := fetchKugouLyricsBody( - c.httpClient, - "https://lyrics.kugou.com/download", - params, - ) - if err != nil { - return nil, fmt.Errorf("kugou lyrics fetch failed: %w", err) - } - - var response kugouLyricsDownloadResponse - if err := json.Unmarshal(raw, &response); err != nil { - return nil, fmt.Errorf("failed to decode kugou lyrics: %w", err) - } - if response.Status != http.StatusOK || response.ErrorCode != 0 { - message := strings.TrimSpace(response.Info) - if message == "" { - message = fmt.Sprintf( - "status %d/error %d", - response.Status, - response.ErrorCode, - ) - } - return nil, lyricsServiceUnavailableErrorf("%s", message) - } - - decoded, err := base64.StdEncoding.DecodeString(response.Content) - if err != nil { - return nil, lyricsServiceUnavailableErrorf( - "invalid base64 lyrics: %v", - err, - ) - } - lyrics := lyricsResponseFromLRCText( - string(decoded), - "Kugou", - "Kugou Direct", - ) - if !lyricsHasUsableText(lyrics) { - return nil, lyricsNotFoundErrorf("kugou returned empty lyrics") - } - return lyrics, nil -} diff --git a/go_backend/lyrics_lrc.go b/go_backend/lyrics_lrc.go deleted file mode 100644 index 432361fd..00000000 --- a/go_backend/lyrics_lrc.go +++ /dev/null @@ -1,317 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" -) - -var lrcLinePattern = regexp.MustCompile(`\[(\d{2}):(\d{2})\.(\d{2,3})\](.*)`) - -var ( - rawLyricsMetadataLinePattern = regexp.MustCompile(`(?i)^\[[a-z][a-z0-9_]*:.*\]$`) - rawLyricsBackgroundPattern = regexp.MustCompile(`(?i)^\[bg:(.*)\]$`) - rawLyricsTimestampPattern = regexp.MustCompile(`^\[\d{1,3}:\d{1,2}(?:[.:]\d{1,3})?\]`) - rawLyricsInlineTimePattern = regexp.MustCompile(`<\d{1,3}:\d{1,2}(?:[.:]\d{1,3})?>`) -) - -func isInstrumentalLyricsMarker(raw string) bool { - return strings.EqualFold(strings.TrimSpace(raw), "[instrumental:true]") -} - -// rawLyricsHasUsableContent rejects LRC payloads that contain only metadata -// headers. Those payloads are common in partially tagged files and must not be -// exposed as a blank "Embedded" lyrics result. -func rawLyricsHasUsableContent(raw string) bool { - if isInstrumentalLyricsMarker(raw) { - return true - } - - for _, line := range strings.Split(raw, "\n") { - cleaned := strings.TrimSpace(line) - if cleaned == "" { - continue - } - - if match := rawLyricsBackgroundPattern.FindStringSubmatch(cleaned); len(match) == 2 { - cleaned = strings.TrimSpace(match[1]) - } else if rawLyricsMetadataLinePattern.MatchString(cleaned) { - continue - } - - for rawLyricsTimestampPattern.MatchString(cleaned) { - cleaned = strings.TrimSpace(rawLyricsTimestampPattern.ReplaceAllString(cleaned, "")) - } - cleaned = strings.TrimSpace(rawLyricsInlineTimePattern.ReplaceAllString(cleaned, "")) - lower := strings.ToLower(cleaned) - if strings.HasPrefix(lower, "v1:") || strings.HasPrefix(lower, "v2:") { - cleaned = strings.TrimSpace(cleaned[3:]) - } - if cleaned != "" { - return true - } - } - - return false -} - -func parseSyncedLyrics(syncedLyrics string) []LyricsLine { - var lines []LyricsLine - - for _, line := range strings.Split(syncedLyrics, "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - - // Preserve Apple/QQ background vocal tags by attaching them to - // the previous timed line. This keeps [bg:...] in final exported LRC. - if strings.HasPrefix(line, "[bg:") && len(lines) > 0 { - lines[len(lines)-1].Words = strings.TrimSpace(lines[len(lines)-1].Words + "\n" + line) - continue - } - - matches := lrcLinePattern.FindStringSubmatch(line) - if len(matches) == 5 { - startMs := lrcTimestampToMs(matches[1], matches[2], matches[3]) - words := strings.TrimSpace(matches[4]) - if words == "" { - continue - } - - lines = append(lines, LyricsLine{ - StartTimeMs: startMs, - Words: words, - EndTimeMs: 0, - }) - } - } - - for i := 0; i < len(lines)-1; i++ { - lines[i].EndTimeMs = lines[i+1].StartTimeMs - } - - if len(lines) > 0 { - lines[len(lines)-1].EndTimeMs = lines[len(lines)-1].StartTimeMs + 5000 - } - - return lines -} - -func plainTextLyricsLines(rawLyrics string) []LyricsLine { - var lines []LyricsLine - for _, line := range strings.Split(rawLyrics, "\n") { - trimmed := strings.TrimSpace(line) - if trimmed == "" { - continue - } - lines = append(lines, LyricsLine{ - StartTimeMs: 0, - Words: trimmed, - EndTimeMs: 0, - }) - } - return lines -} - -func lyricsHasUsableText(lyrics *LyricsResponse) bool { - if lyrics == nil { - return false - } - if lyrics.Instrumental { - return true - } - if strings.TrimSpace(lyrics.PlainLyrics) != "" { - return true - } - for _, line := range lyrics.Lines { - if strings.TrimSpace(line.Words) != "" { - return true - } - } - return false -} - -func detectLyricsErrorPayload(raw string) (string, bool) { - trimmed := strings.TrimSpace(raw) - if trimmed == "" || !strings.HasPrefix(trimmed, "{") { - return "", false - } - - var payload map[string]any - if err := json.Unmarshal([]byte(trimmed), &payload); err != nil { - return "", false - } - - lyricsKeys := []string{"lyrics", "lyric", "lrc", "content", "lines", "syncedLyrics", "unsyncedLyrics"} - hasLyricsKey := false - for _, key := range lyricsKeys { - if _, ok := payload[key]; ok { - hasLyricsKey = true - break - } - } - - errorKeys := []string{"message", "error", "detail", "reason"} - for _, key := range errorKeys { - if msg, ok := payload[key].(string); ok { - msg = strings.TrimSpace(msg) - if msg != "" && !hasLyricsKey { - return msg, true - } - } - } - - if success, ok := payload["success"].(bool); ok && !success && !hasLyricsKey { - return "request unsuccessful", true - } - if isError, ok := payload["isError"].(bool); ok && isError && !hasLyricsKey { - return "request unsuccessful", true - } - if code, ok := payload["code"].(float64); ok && code != 0 && code != 200 && !hasLyricsKey { - if msg, ok := payload["message"].(string); ok && strings.TrimSpace(msg) != "" { - return strings.TrimSpace(msg), true - } - if msg, ok := payload["msg"].(string); ok && strings.TrimSpace(msg) != "" { - return strings.TrimSpace(msg), true - } - return fmt.Sprintf("unexpected response code %.0f", code), true - } - - return "", false -} - -func lrcTimestampToMs(minutes, seconds, centiseconds string) int64 { - min, _ := strconv.ParseInt(minutes, 10, 64) - sec, _ := strconv.ParseInt(seconds, 10, 64) - cs, _ := strconv.ParseInt(centiseconds, 10, 64) - - if len(centiseconds) == 2 { - cs *= 10 - } - - return min*60*1000 + sec*1000 + cs -} - -func msToLRCTimestamp(ms int64) string { - return fmt.Sprintf("[%s]", msToLRCTimestampInline(ms)) -} - -func msToLRCTimestampInline(ms int64) string { - totalSeconds := ms / 1000 - minutes := totalSeconds / 60 - seconds := totalSeconds % 60 - centiseconds := (ms % 1000) / 10 - - return fmt.Sprintf("%02d:%02d.%02d", minutes, seconds, centiseconds) -} - -// extractLyricsSourceFromLRC reads the provider recorded in the LRC [by:] tag, -// e.g. "[by:SpotiFLAC-Mobile (source: LRCLIB)]". Returns "" when absent. -const lrcSourceMarker = "(source: " - -func lyricsSourceUsesPaxsenix(source string) bool { - s := strings.ToLower(strings.TrimSpace(source)) - if s == "" { - return false - } - if strings.HasPrefix(s, "lrclib") || - strings.HasPrefix(s, "kugou direct") || - strings.HasPrefix(s, "qq music direct") || - strings.HasPrefix(s, "genius direct") || - strings.HasPrefix(s, "extension:") || - strings.HasPrefix(s, "heuristic") { - return false - } - return true -} - -func extractLyricsSourceFromLRC(lrc string) string { - for _, line := range strings.Split(lrc, "\n") { - trimmed := strings.TrimSpace(line) - if !strings.HasPrefix(strings.ToLower(trimmed), "[by:") { - continue - } - idx := strings.Index(trimmed, lrcSourceMarker) - if idx < 0 { - return "" - } - rest := strings.TrimSpace(trimmed[idx+len(lrcSourceMarker):]) - rest = strings.TrimSuffix(rest, "]") - rest = strings.TrimSuffix(rest, ")") - return strings.TrimSpace(rest) - } - return "" -} - -func convertToLRCWithMetadata(lyrics *LyricsResponse, trackName, artistName string) string { - if lyrics == nil || len(lyrics.Lines) == 0 { - return "" - } - - var builder strings.Builder - - builder.WriteString(fmt.Sprintf("[ti:%s]\n", trackName)) - builder.WriteString(fmt.Sprintf("[ar:%s]\n", artistName)) - source := strings.TrimSpace(lyrics.Source) - if source == "" { - source = strings.TrimSpace(lyrics.Provider) - } - credit := "SpotiFLAC-Mobile" - if lyricsSourceUsesPaxsenix(source) { - credit = "SpotiFLAC-Mobile via Paxsenix API" - } - if source == "" { - builder.WriteString(fmt.Sprintf("[by:%s]\n", credit)) - } else { - builder.WriteString( - fmt.Sprintf("[by:%s %s%s)]\n", credit, lrcSourceMarker, source), - ) - } - builder.WriteString("\n") - - if lyrics.SyncType == "LINE_SYNCED" { - for _, line := range lyrics.Lines { - if line.Words == "" { - continue - } - timestamp := msToLRCTimestamp(line.StartTimeMs) - builder.WriteString(timestamp) - builder.WriteString(line.Words) - builder.WriteString("\n") - } - } else { - for _, line := range lyrics.Lines { - if line.Words == "" { - continue - } - builder.WriteString(line.Words) - builder.WriteString("\n") - } - } - - return builder.String() -} - -func SaveLRCFile(audioFilePath, lrcContent string) (string, error) { - if lrcContent == "" { - return "", fmt.Errorf("empty LRC content") - } - - dir := filepath.Dir(audioFilePath) - ext := filepath.Ext(audioFilePath) - baseName := strings.TrimSuffix(filepath.Base(audioFilePath), ext) - - lrcFilePath := filepath.Join(dir, baseName+".lrc") - - if err := os.WriteFile(lrcFilePath, []byte(lrcContent), 0644); err != nil { - return "", fmt.Errorf("failed to write LRC file: %w", err) - } - - GoLog("[Lyrics] Saved LRC file: %s\n", lrcFilePath) - return lrcFilePath, nil -} diff --git a/go_backend/lyrics_lyricsplus.go b/go_backend/lyrics_lyricsplus.go deleted file mode 100644 index 9233b261..00000000 --- a/go_backend/lyrics_lyricsplus.go +++ /dev/null @@ -1,239 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "net/http" - "net/url" - "strconv" - "strings" - "time" -) - -// LyricsPlus (KPOE) provider. -// -// LyricsPlus aggregates word-by-word ("karaoke") synced lyrics from Apple -// Music, Musixmatch, Spotify and QQ Music via a community-run backend. It -// frequently has word-level timing for tracks that other providers only offer -// line-synced or not at all. -// -// API: GET {server}/v2/lyrics/get?title=&artist=&album=&duration=&isrc= -// The response is the KPOE JSON format which we convert into the same enhanced -// LRC text the Apple/QQ providers emit, so embedding/export behaves identically. - -// Public LyricsPlus / KPOE servers (mirrors). Tried in order with failover. -// Sourced from the upstream YouLy+ client server list. -var lyricsPlusServers = []string{ - "https://lyricsplus.prjktla.workers.dev", - "https://lyricsplus.binimum.org", -} - -type LyricsPlusClient struct { - httpClient *http.Client -} - -func NewLyricsPlusClient() *LyricsPlusClient { - return &LyricsPlusClient{httpClient: NewMetadataHTTPClient(15 * time.Second)} -} - -type lyricsPlusSyllable struct { - Text string `json:"text"` - Time float64 `json:"time"` // absolute ms - Duration float64 `json:"duration"` // ms - IsBackground bool `json:"isBackground"` -} - -type lyricsPlusLine struct { - Time float64 `json:"time"` // absolute ms - Duration float64 `json:"duration"` // ms - Text string `json:"text"` - Syllabus []lyricsPlusSyllable `json:"syllabus"` -} - -type lyricsPlusResponse struct { - Type string `json:"type"` // "Word" | "Line" | "Syllable" | "None" - Lyrics []lyricsPlusLine `json:"lyrics"` -} - -// FetchLyrics tries each LyricsPlus server in order until one returns usable -// lyrics. multiPersonWordByWord and preserveWordTiming mirror the Apple/QQ -// options so word/background timing is only emitted when the user enabled it. -func (c *LyricsPlusClient) FetchLyrics( - trackName, - artistName, - isrc string, - durationSec float64, - multiPersonWordByWord bool, - preserveWordTiming bool, -) (*LyricsResponse, error) { - if strings.TrimSpace(trackName) == "" || strings.TrimSpace(artistName) == "" { - return nil, fmt.Errorf("lyricsplus: missing track or artist") - } - - var lastErr error - for _, server := range lyricsPlusServers { - lyrics, err := c.fetchFromServer(server, trackName, artistName, isrc, durationSec, multiPersonWordByWord, preserveWordTiming) - if err == nil && lyricsHasUsableText(lyrics) { - return lyrics, nil - } - if err != nil { - lastErr = err - GoLog("[Lyrics] LyricsPlus server %s failed: %v\n", server, err) - } - } - - if lastErr != nil { - return nil, lastErr - } - return nil, lyricsNotFoundErrorf("lyricsplus: no lyrics found") -} - -func (c *LyricsPlusClient) fetchFromServer( - server, - trackName, - artistName, - isrc string, - durationSec float64, - multiPersonWordByWord bool, - preserveWordTiming bool, -) (*LyricsResponse, error) { - base := strings.TrimRight(strings.TrimSpace(server), "/") - if base == "" { - return nil, fmt.Errorf("empty server") - } - - params := url.Values{} - params.Set("title", trackName) - params.Set("artist", artistName) - if durationSec > 0 { - params.Set("duration", strconv.FormatFloat(durationSec, 'f', 3, 64)) - } - if strings.TrimSpace(isrc) != "" { - params.Set("isrc", strings.TrimSpace(isrc)) - } - - fullURL := base + "/v2/lyrics/get?" + params.Encode() - - req, err := http.NewRequest("GET", fullURL, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", appUserAgent()) - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusNotFound { - // Retry without the ISRC filter, which can be too strict. - if strings.TrimSpace(isrc) != "" { - return c.fetchFromServer(server, trackName, artistName, "", durationSec, multiPersonWordByWord, preserveWordTiming) - } - return nil, lyricsNotFoundErrorf("lyrics not found") - } - if resp.StatusCode != http.StatusOK { - return nil, lyricsHTTPStatusError(resp.StatusCode, "HTTP %d", resp.StatusCode) - } - - var payload lyricsPlusResponse - if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - return nil, fmt.Errorf("failed to decode lyricsplus response: %w", err) - } - if len(payload.Lyrics) == 0 { - return nil, fmt.Errorf("lyricsplus returned no lines") - } - - lrcText := buildLyricsPlusLRC(&payload, multiPersonWordByWord, preserveWordTiming) - if strings.TrimSpace(lrcText) == "" { - return nil, fmt.Errorf("lyricsplus produced empty lyrics") - } - - lyrics := lyricsResponseFromText(lrcText, "LyricsPlus") - return lyrics, nil -} - -// buildLyricsPlusLRC converts the KPOE JSON into enhanced LRC text. When word -// timing is available and enabled, each syllable is emitted as an inline -// <mm:ss.xx> tag (matching the Apple/QQ output); otherwise a line-synced LRC -// is produced from the full line text. -func buildLyricsPlusLRC(resp *lyricsPlusResponse, multiPersonWordByWord bool, preserveWordTiming bool) string { - isWordType := strings.EqualFold(resp.Type, "Word") || strings.EqualFold(resp.Type, "Syllable") - - var sb strings.Builder - first := true - for _, line := range resp.Lyrics { - lineText := line.Text - hasSyllables := len(line.Syllabus) > 0 - - timestamp := msToLRCTimestamp(int64(line.Time)) - - if isWordType && preserveWordTiming && hasSyllables { - mainSyllables := make([]lyricsPlusSyllable, 0, len(line.Syllabus)) - bgSyllables := make([]lyricsPlusSyllable, 0) - for _, syl := range line.Syllabus { - if syl.IsBackground { - bgSyllables = append(bgSyllables, syl) - } else { - mainSyllables = append(mainSyllables, syl) - } - } - if len(mainSyllables) == 0 { - mainSyllables = line.Syllabus - bgSyllables = nil - } - - if !first { - sb.WriteString("\n") - } - first = false - - sb.WriteString(timestamp) - appendLyricsPlusSyllables(&sb, mainSyllables) - - if multiPersonWordByWord && len(bgSyllables) > 0 { - sb.WriteString("\n[bg:") - appendLyricsPlusSyllables(&sb, bgSyllables) - sb.WriteString("]") - } - continue - } - - // Line-synced fallback. Reconstruct text from syllables if needed. - if strings.TrimSpace(lineText) == "" && hasSyllables { - var lineBuilder strings.Builder - for _, syl := range line.Syllabus { - lineBuilder.WriteString(syl.Text) - } - lineText = lineBuilder.String() - } - - lineText = strings.TrimSpace(lineText) - if lineText == "" { - continue - } - - if !first { - sb.WriteString("\n") - } - first = false - - sb.WriteString(timestamp) - sb.WriteString(lineText) - } - - return strings.TrimSpace(sb.String()) -} - -// appendLyricsPlusSyllables writes each syllable as "<mm:ss.xx>text". KPOE -// already embeds spacing inside the syllable text, so no extra spaces are added. -func appendLyricsPlusSyllables(sb *strings.Builder, syllables []lyricsPlusSyllable) { - for _, syl := range syllables { - sb.WriteString("<") - sb.WriteString(msToLRCTimestampInline(int64(syl.Time))) - sb.WriteString(">") - sb.WriteString(syl.Text) - } -} diff --git a/go_backend/lyrics_matching.go b/go_backend/lyrics_matching.go deleted file mode 100644 index 0002330b..00000000 --- a/go_backend/lyrics_matching.go +++ /dev/null @@ -1,135 +0,0 @@ -package gobackend - -import ( - "math" - "regexp" - "strings" -) - -var simplifyTrackNamePatterns = func() []*regexp.Regexp { - patterns := []string{ - `\s*\(feat\.?.*?\)`, - `\s*\(ft\.?.*?\)`, - `\s*\(featuring.*?\)`, - `\s*\(with.*?\)`, - `\s*-\s*Remaster(ed)?.*$`, - `\s*-\s*\d{4}\s*Remaster.*$`, - `\s*\(Remaster(ed)?.*?\)`, - `\s*\(Deluxe.*?\)`, - `\s*\(Bonus.*?\)`, - `\s*\(Live.*?\)`, - `\s*\(Acoustic.*?\)`, - `\s*\(Radio Edit\)`, - `\s*\(Single Version\)`, - } - compiled := make([]*regexp.Regexp, len(patterns)) - for i, pattern := range patterns { - compiled[i] = regexp.MustCompile("(?i)" + pattern) - } - return compiled -}() - -func simplifyTrackName(name string) string { - result := name - for _, re := range simplifyTrackNamePatterns { - result = re.ReplaceAllString(result, "") - } - result = strings.TrimSpace(result) - if result == "" { - return result - } - - if loose := normalizeLooseTitle(result); loose != "" { - return loose - } - - return result -} - -func normalizedLyricsSearchTitle(name string) string { - return strings.ToLower(strings.TrimSpace(simplifyTrackName(name))) -} - -func containsWordSequence(value, sequence string) bool { - valueWords := strings.Fields(value) - sequenceWords := strings.Fields(sequence) - if len(valueWords) == 0 || len(sequenceWords) == 0 || len(sequenceWords) > len(valueWords) { - return false - } - - for start := 0; start <= len(valueWords)-len(sequenceWords); start++ { - matches := true - for offset := range sequenceWords { - if valueWords[start+offset] != sequenceWords[offset] { - matches = false - break - } - } - if matches { - return true - } - } - return false -} - -func lyricsSearchTitlesMatch(candidateTrack, trackName string, allowDecoratedCandidate bool) bool { - expected := normalizedLyricsSearchTitle(trackName) - candidate := normalizedLyricsSearchTitle(candidateTrack) - if expected == "" || candidate == "" { - return false - } - if candidate == expected { - return true - } - return allowDecoratedCandidate && containsWordSequence(candidate, expected) -} - -func lyricsSearchArtistsMatch(candidateArtist, artistName string) bool { - expected := normalizeLooseArtistName(normalizeArtistName(artistName)) - if expected == "" { - return true - } - candidate := normalizeLooseArtistName(normalizeArtistName(candidateArtist)) - if candidate == "" { - return false - } - return candidate == expected || sameWordsUnordered(candidate, expected) -} - -func lyricsSearchDurationMatches(candidateDuration, durationSec float64) bool { - if candidateDuration <= 0 || durationSec <= 0 { - return true - } - return math.Abs(candidateDuration-durationSec) <= durationToleranceSec -} - -func lyricsSearchArtistAppearsInTitle(candidateTrack, artistName string) bool { - expectedArtist := normalizeLooseArtistName(normalizeArtistName(artistName)) - candidateTitle := normalizeLooseArtistName(candidateTrack) - return expectedArtist != "" && - candidateTitle != "" && - containsWordSequence(candidateTitle, expectedArtist) -} - -func normalizeArtistName(name string) string { - separators := []string{", ", "; ", " & ", " feat. ", " ft. ", " featuring ", " with "} - - result := name - for _, sep := range separators { - if idx := strings.Index(strings.ToLower(result), strings.ToLower(sep)); idx > 0 { - result = result[:idx] - break - } - } - - return strings.TrimSpace(result) -} - -func isLikelyInstrumentalTrack(name string) bool { - trimmed := strings.TrimSpace(name) - if trimmed == "" { - return false - } - - return instrumentalTrackPattern.MatchString(trimmed) -} diff --git a/go_backend/lyrics_musixmatch.go b/go_backend/lyrics_musixmatch.go deleted file mode 100644 index bf15f524..00000000 --- a/go_backend/lyrics_musixmatch.go +++ /dev/null @@ -1,124 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "io" - "math" - "net/http" - "net/url" - "strings" - "time" -) - -type MusixmatchClient struct { - httpClient *http.Client - baseURL string -} - -func NewMusixmatchClient() *MusixmatchClient { - return &MusixmatchClient{ - httpClient: NewMetadataHTTPClient(15 * time.Second), - baseURL: "https://lyrics.paxsenix.org/musixmatch/lyrics", - } -} - -func (c *MusixmatchClient) fetchLyricsPayload(trackName, artistName string, durationSec float64, lyricsType, language string) (string, error) { - if strings.TrimSpace(trackName) == "" || strings.TrimSpace(artistName) == "" { - return "", lyricsNotFoundErrorf("empty track or artist name") - } - - params := url.Values{} - params.Set("t", trackName) - params.Set("a", artistName) - params.Set("type", lyricsType) - params.Set("format", "lrc") - if durationSec > 0 { - params.Set("d", fmt.Sprintf("%d", int(math.Round(durationSec)))) - } - if strings.TrimSpace(language) != "" { - params.Set("l", strings.ToLower(strings.TrimSpace(language))) - } - fullURL := c.baseURL + "?" + params.Encode() - - req, err := http.NewRequest("GET", fullURL, nil) - if err != nil { - return "", fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", appUserAgent()) - - resp, err := c.httpClient.Do(req) - if err != nil { - return "", fmt.Errorf("musixmatch request failed: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read musixmatch response: %w", err) - } - - if resp.StatusCode != 200 { - trimmed := strings.TrimSpace(string(body)) - if errMsg, isErrorPayload := detectLyricsErrorPayload(trimmed); isErrorPayload { - return "", classifyLyricsPayloadError(resp.StatusCode, errMsg, "musixmatch proxy returned HTTP %d: %s", resp.StatusCode, errMsg) - } - return "", lyricsHTTPStatusError(resp.StatusCode, "musixmatch proxy returned HTTP %d", resp.StatusCode) - } - - var lrcPayload string - if err := json.Unmarshal(body, &lrcPayload); err == nil { - lrcPayload = strings.TrimSpace(lrcPayload) - if lrcPayload == "" { - return "", fmt.Errorf("empty musixmatch lyrics payload") - } - return lrcPayload, nil - } - - trimmed := strings.TrimSpace(string(body)) - if errMsg, isErrorPayload := detectLyricsErrorPayload(trimmed); isErrorPayload { - return "", classifyLyricsPayloadError(0, errMsg, "%s", errMsg) - } - if trimmed != "" && !strings.HasPrefix(trimmed, "{") { - return trimmed, nil - } - return "", fmt.Errorf("failed to decode musixmatch response") -} - -func (c *MusixmatchClient) FetchLyricsInLanguage(trackName, artistName string, durationSec float64, language string) (*LyricsResponse, error) { - lang := strings.ToLower(strings.TrimSpace(language)) - if lang == "" { - return nil, fmt.Errorf("invalid language") - } - - lrcText, err := c.fetchLyricsPayload(trackName, artistName, durationSec, "translate", lang) - if err != nil { - return nil, err - } - - if resp := lyricsResponseFromLRCText(lrcText, "Musixmatch", fmt.Sprintf("Musixmatch (%s)", lang)); resp != nil { - return resp, nil - } - return nil, lyricsNotFoundErrorf("no lyrics found on musixmatch for language %s", lang) -} - -func (c *MusixmatchClient) FetchLyrics(trackName, artistName string, durationSec float64, preferredLanguage string) (*LyricsResponse, error) { - if preferred := strings.ToLower(strings.TrimSpace(preferredLanguage)); preferred != "" { - localized, localizedErr := c.FetchLyricsInLanguage(trackName, artistName, durationSec, preferred) - if localizedErr == nil { - return localized, nil - } - GoLog("[Musixmatch] Language override '%s' failed: %v\n", preferred, localizedErr) - } - - lrcText, err := c.fetchLyricsPayload(trackName, artistName, durationSec, "word", "") - if err != nil { - return nil, err - } - - if resp := lyricsResponseFromLRCText(lrcText, "Musixmatch", "Musixmatch"); resp != nil { - return resp, nil - } - return nil, lyricsNotFoundErrorf("no lyrics found on musixmatch") -} diff --git a/go_backend/lyrics_netease.go b/go_backend/lyrics_netease.go deleted file mode 100644 index e2d44031..00000000 --- a/go_backend/lyrics_netease.go +++ /dev/null @@ -1,206 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "net/http" - "net/url" - "strings" - "time" -) - -type NeteaseClient struct { - httpClient *http.Client -} - -type neteaseSearchSong struct { - Name string `json:"name"` - ID int64 `json:"id"` - Artists []struct { - Name string `json:"name"` - } `json:"artists"` -} - -type neteaseSearchResponse struct { - Result struct { - Songs []neteaseSearchSong `json:"songs"` - SongCount int `json:"songCount"` - } `json:"result"` - Code int `json:"code"` - Message string `json:"message"` - Msg string `json:"msg"` -} - -type neteaseLyricsResponse struct { - LRC *neteaseLyricField `json:"lrc"` - TLyric *neteaseLyricField `json:"tlyric"` - RomaLRC *neteaseLyricField `json:"romalrc"` - Code int `json:"code"` -} - -type neteaseLyricField struct { - Lyric string `json:"lyric"` -} - -var neteaseHeaders = map[string]string{ - "Accept": "application/json", - "Accept-Language": "en-US,en;q=0.9", - "Cache-Control": "max-age=0", -} - -func NewNeteaseClient() *NeteaseClient { - return &NeteaseClient{ - httpClient: NewMetadataHTTPClient(15 * time.Second), - } -} - -func (c *NeteaseClient) SearchSong(trackName, artistName string) (int64, error) { - query := trackName + " " + artistName - if strings.TrimSpace(query) == "" { - return 0, lyricsNotFoundErrorf("empty search query") - } - - searchURL := "https://lyrics.paxsenix.org/netease/search" - params := url.Values{} - params.Set("q", query) - - fullURL := searchURL + "?" + params.Encode() - - req, err := http.NewRequest("GET", fullURL, nil) - if err != nil { - return 0, fmt.Errorf("failed to create request: %w", err) - } - - for k, v := range neteaseHeaders { - req.Header.Set(k, v) - } - req.Header.Set("User-Agent", appUserAgent()) - - resp, err := c.httpClient.Do(req) - if err != nil { - return 0, fmt.Errorf("netease search failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return 0, lyricsHTTPStatusError(resp.StatusCode, "netease search returned HTTP %d", resp.StatusCode) - } - - var searchResp neteaseSearchResponse - if err := json.NewDecoder(resp.Body).Decode(&searchResp); err != nil { - return 0, fmt.Errorf("failed to decode netease search: %w", err) - } - - if searchResp.Code != 0 && searchResp.Code != 200 { - message := strings.TrimSpace(searchResp.Message) - if message == "" { - message = strings.TrimSpace(searchResp.Msg) - } - if message == "" { - message = "unexpected response code" - } - return 0, lyricsServiceUnavailableErrorf("netease search unavailable: code %d: %s", searchResp.Code, message) - } - - if searchResp.Result.SongCount == 0 || len(searchResp.Result.Songs) == 0 { - return 0, lyricsNotFoundErrorf("no songs found on netease") - } - - best := selectBestNeteaseSearchResult(searchResp.Result.Songs, trackName, artistName) - if best == nil || best.ID == 0 { - return 0, lyricsNotFoundErrorf("no matching songs found on netease") - } - return best.ID, nil -} - -func selectBestNeteaseSearchResult(results []neteaseSearchSong, trackName, artistName string) *neteaseSearchSong { - best := selectBestLyricsCandidate(len(results), trackName, artistName, 0, func(i int) (string, string, float64, bool) { - result := &results[i] - artists := make([]string, 0, len(result.Artists)) - for _, artist := range result.Artists { - if name := strings.TrimSpace(artist.Name); name != "" { - artists = append(artists, name) - } - } - candidateArtist := strings.Join(artists, ", ") - ok := lyricsSearchTitlesMatch(result.Name, trackName, false) && - lyricsSearchArtistsMatch(candidateArtist, artistName) - return result.Name, candidateArtist, 0, ok - }) - if best < 0 { - return nil - } - return &results[best] -} - -func (c *NeteaseClient) FetchLyricsByID(songID int64, includeTranslation, includeRomanization bool) (string, error) { - lyricsURL := "https://lyrics.paxsenix.org/netease/lyrics" - params := url.Values{} - params.Set("id", fmt.Sprintf("%d", songID)) - - fullURL := lyricsURL + "?" + params.Encode() - - req, err := http.NewRequest("GET", fullURL, nil) - if err != nil { - return "", fmt.Errorf("failed to create request: %w", err) - } - - for k, v := range neteaseHeaders { - req.Header.Set(k, v) - } - req.Header.Set("User-Agent", appUserAgent()) - - resp, err := c.httpClient.Do(req) - if err != nil { - return "", fmt.Errorf("netease lyrics fetch failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return "", lyricsHTTPStatusError(resp.StatusCode, "netease lyrics returned HTTP %d", resp.StatusCode) - } - - var lyricsResp neteaseLyricsResponse - if err := json.NewDecoder(resp.Body).Decode(&lyricsResp); err != nil { - return "", fmt.Errorf("failed to decode netease lyrics: %w", err) - } - - if lyricsResp.LRC == nil || strings.TrimSpace(lyricsResp.LRC.Lyric) == "" { - return "", lyricsNotFoundErrorf("no lyrics available on netease") - } - - lyric := lyricsResp.LRC.Lyric - - if includeTranslation && lyricsResp.TLyric != nil && strings.TrimSpace(lyricsResp.TLyric.Lyric) != "" { - lyric += "\n\n" + lyricsResp.TLyric.Lyric - } - - if includeRomanization && lyricsResp.RomaLRC != nil && strings.TrimSpace(lyricsResp.RomaLRC.Lyric) != "" { - lyric += "\n\n" + lyricsResp.RomaLRC.Lyric - } - - return lyric, nil -} - -func (c *NeteaseClient) FetchLyrics( - trackName, - artistName string, - durationSec float64, - includeTranslation, - includeRomanization bool, -) (*LyricsResponse, error) { - songID, err := c.SearchSong(trackName, artistName) - if err != nil { - return nil, err - } - - lrcText, err := c.FetchLyricsByID(songID, includeTranslation, includeRomanization) - if err != nil { - return nil, err - } - - if resp := lyricsResponseFromLRCText(lrcText, "Netease", "Netease"); resp != nil { - return resp, nil - } - return nil, fmt.Errorf("netease returned empty lyrics") -} diff --git a/go_backend/lyrics_paxsenix.go b/go_backend/lyrics_paxsenix.go deleted file mode 100644 index c73b4b9a..00000000 --- a/go_backend/lyrics_paxsenix.go +++ /dev/null @@ -1,529 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "io" - "math" - "net/http" - "net/url" - "regexp" - "strconv" - "strings" - "time" -) - -type SpotifyLyricsClient struct { - httpClient *http.Client -} - -type DeezerLyricsClient struct { - httpClient *http.Client -} - -type YouTubeLyricsClient struct { - httpClient *http.Client -} - -type GeniusLyricsClient struct { - httpClient *http.Client -} - -type spotifyLyricsSearchResult struct { - TrackID string `json:"trackId"` - Name string `json:"name"` - ArtistName string `json:"artistName"` - Duration string `json:"duration"` -} - -type youtubeLyricsSearchResult struct { - VideoID string `json:"videoId"` - Title string `json:"title"` - Author string `json:"author"` - Duration string `json:"duration"` -} - -type geniusSearchResponse struct { - Response struct { - Sections []struct { - Hits []struct { - Type string `json:"type"` - Result struct { - Title string `json:"title"` - ArtistNames string `json:"artist_names"` - PrimaryArtistNames string `json:"primary_artist_names"` - URL string `json:"url"` - } `json:"result"` - } `json:"hits"` - } `json:"sections"` - } `json:"response"` -} - -type paxsenixLyricsObject struct { - Type string `json:"type"` - Content []paxLyrics `json:"content"` - Lyrics []paxLyrics `json:"lyrics"` - LyricsText string `json:"lyrics_text"` - PlainLyrics string `json:"plain_lyrics"` -} - -func NewSpotifyLyricsClient() *SpotifyLyricsClient { - return &SpotifyLyricsClient{httpClient: NewMetadataHTTPClient(15 * time.Second)} -} - -func NewDeezerLyricsClient() *DeezerLyricsClient { - return &DeezerLyricsClient{httpClient: NewMetadataHTTPClient(15 * time.Second)} -} - -func NewYouTubeLyricsClient() *YouTubeLyricsClient { - return &YouTubeLyricsClient{httpClient: NewMetadataHTTPClient(15 * time.Second)} -} - -func NewGeniusLyricsClient() *GeniusLyricsClient { - return &GeniusLyricsClient{httpClient: NewMetadataHTTPClient(15 * time.Second)} -} - -func fetchPaxsenixBody(httpClient *http.Client, endpoint string, params url.Values) (string, error) { - fullURL := endpoint - if len(params) > 0 { - fullURL += "?" + params.Encode() - } - - req, err := http.NewRequest("GET", fullURL, nil) - if err != nil { - return "", fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", appUserAgent()) - - resp, err := httpClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read response: %w", err) - } - - // Any failure of the shared proxy fetch disables the source until the - // cooldown expires, except when the payload says the track has no lyrics. - trimmed := strings.TrimSpace(string(body)) - if resp.StatusCode != http.StatusOK { - if errMsg, isErrorPayload := detectLyricsErrorPayload(trimmed); isErrorPayload { - if lyricsPayloadIndicatesNotFound(errMsg) { - return "", lyricsNotFoundErrorf("HTTP %d: %s", resp.StatusCode, errMsg) - } - return "", lyricsServiceUnavailableErrorf("HTTP %d: %s", resp.StatusCode, errMsg) - } - return "", lyricsServiceUnavailableErrorf("HTTP %d", resp.StatusCode) - } - if errMsg, isErrorPayload := detectLyricsErrorPayload(trimmed); isErrorPayload { - if lyricsPayloadIndicatesNotFound(errMsg) { - return "", lyricsNotFoundErrorf("%s", errMsg) - } - return "", lyricsServiceUnavailableErrorf("%s", errMsg) - } - if trimmed == "" { - return "", lyricsServiceUnavailableErrorf("empty response") - } - return trimmed, nil -} - -func parsePaxsenixLyricsPayload(raw, provider string, multiPersonWordByWord bool) (*LyricsResponse, error) { - var lrcPayload string - if err := json.Unmarshal([]byte(raw), &lrcPayload); err == nil { - lrcPayload = strings.TrimSpace(lrcPayload) - if lrcPayload == "" { - return nil, lyricsServiceUnavailableErrorf("%s returned empty lyrics", provider) - } - return lyricsResponseFromText(lrcPayload, provider), nil - } - - var rawObject map[string]json.RawMessage - if err := json.Unmarshal([]byte(raw), &rawObject); err == nil { - for _, key := range []string{"lyrics", "lyric", "lyrics_text", "plain_lyrics"} { - var value string - if rawValue, ok := rawObject[key]; ok && json.Unmarshal(rawValue, &value) == nil { - value = strings.TrimSpace(value) - if value != "" { - return lyricsResponseFromText(value, provider), nil - } - } - } - } - - var payload paxsenixLyricsObject - if err := json.Unmarshal([]byte(raw), &payload); err == nil { - switch { - case strings.TrimSpace(payload.LyricsText) != "": - return lyricsResponseFromText(payload.LyricsText, provider), nil - case len(payload.Lyrics) > 0: - return lyricsResponseFromText(formatPaxContent("Syllable", payload.Lyrics, multiPersonWordByWord, true), provider), nil - case len(payload.Content) > 0: - lyricsType := payload.Type - if lyricsType == "" { - lyricsType = "Syllable" - } - return lyricsResponseFromText(formatPaxContent(lyricsType, payload.Content, multiPersonWordByWord, true), provider), nil - case strings.TrimSpace(payload.PlainLyrics) != "": - return lyricsResponseFromText(payload.PlainLyrics, provider), nil - } - } - - trimmed := strings.TrimSpace(raw) - if trimmed != "" && !strings.HasPrefix(trimmed, "{") && !strings.HasPrefix(trimmed, "[") { - return lyricsResponseFromText(trimmed, provider), nil - } - if json.Valid([]byte(trimmed)) { - return nil, lyricsServiceUnavailableErrorf( - "%s returned a response without usable lyrics", - provider, - ) - } - return nil, lyricsServiceUnavailableErrorf("failed to decode %s lyrics response", provider) -} - -// lyricsResponseFromLRCText parses LRC-or-plain text into a response, or nil -// when the text contains no usable lines. -func lyricsResponseFromLRCText(text, provider, source string) *LyricsResponse { - if lines := parseSyncedLyrics(text); len(lines) > 0 { - return &LyricsResponse{ - Lines: lines, - SyncType: "LINE_SYNCED", - PlainLyrics: plainLyricsFromTimedLines(lines), - Provider: provider, - Source: source, - } - } - if lines := plainTextLyricsLines(text); len(lines) > 0 { - return &LyricsResponse{ - Lines: lines, - SyncType: "UNSYNCED", - PlainLyrics: text, - Provider: provider, - Source: source, - } - } - return nil -} - -func lyricsResponseFromText(text, provider string) *LyricsResponse { - if resp := lyricsResponseFromLRCText(text, provider, provider); resp != nil { - return resp - } - return &LyricsResponse{Provider: provider, Source: provider} -} - -func normalizeSpotifyLyricsID(raw string) string { - raw = strings.TrimSpace(raw) - if raw == "" || strings.HasPrefix(strings.ToLower(raw), "deezer:") { - return "" - } - if strings.HasPrefix(strings.ToLower(raw), "spotify:") { - parts := strings.Split(raw, ":") - raw = parts[len(parts)-1] - } - if strings.Contains(raw, "spotify.com/track/") { - raw = extractSpotifyIDFromURL(raw) - } - raw = strings.TrimSpace(strings.Split(raw, "?")[0]) - if regexpSpotifyTrackID.MatchString(raw) { - return raw - } - return "" -} - -var regexpSpotifyTrackID = regexp.MustCompile(`^[A-Za-z0-9]{22}$`) - -func (c *SpotifyLyricsClient) SearchSong(trackName, artistName string, durationSec float64) (string, error) { - query := strings.TrimSpace(trackName + " " + artistName) - if query == "" { - return "", lyricsNotFoundErrorf("empty search query") - } - - params := url.Values{} - params.Set("q", query) - raw, err := fetchPaxsenixBody(c.httpClient, "https://lyrics.paxsenix.org/spotify/search", params) - if err != nil { - return "", fmt.Errorf("spotify search failed: %w", err) - } - - var results []spotifyLyricsSearchResult - if err := json.Unmarshal([]byte(raw), &results); err != nil { - return "", fmt.Errorf("failed to decode spotify search: %w", err) - } - best := selectBestSpotifyLyricsSearchResult(results, trackName, artistName, durationSec) - if best == nil || strings.TrimSpace(best.TrackID) == "" { - return "", lyricsNotFoundErrorf("no songs found on spotify") - } - return strings.TrimSpace(best.TrackID), nil -} - -// selectBestLyricsCandidate returns the index of the highest-scoring candidate -// whose provider-specific gate passes; candidate returns -// (name, artist, durationSec, gatePassed). Returns -1 when nothing matches. -func selectBestLyricsCandidate(n int, trackName, artistName string, durationSec float64, candidate func(i int) (string, string, float64, bool)) int { - bestIndex := -1 - bestScore := -1 - for i := 0; i < n; i++ { - name, artist, duration, ok := candidate(i) - if !ok { - continue - } - score := scoreLyricsSearchCandidate(name, artist, duration, trackName, artistName, durationSec) - if score > bestScore { - bestIndex = i - bestScore = score - } - } - return bestIndex -} - -func selectBestSpotifyLyricsSearchResult(results []spotifyLyricsSearchResult, trackName, artistName string, durationSec float64) *spotifyLyricsSearchResult { - best := selectBestLyricsCandidate(len(results), trackName, artistName, durationSec, func(i int) (string, string, float64, bool) { - result := &results[i] - duration := parseClockDuration(result.Duration) - ok := lyricsSearchTitlesMatch(result.Name, trackName, false) && - lyricsSearchArtistsMatch(result.ArtistName, artistName) && - lyricsSearchDurationMatches(duration, durationSec) - return result.Name, result.ArtistName, duration, ok - }) - if best < 0 { - return nil - } - return &results[best] -} - -func (c *SpotifyLyricsClient) FetchLyricsByID(trackID string) (*LyricsResponse, error) { - params := url.Values{} - params.Set("id", trackID) - raw, err := fetchPaxsenixBody(c.httpClient, "https://lyrics.paxsenix.org/spotify/lyrics", params) - if err != nil { - return nil, fmt.Errorf("spotify lyrics fetch failed: %w", err) - } - return parsePaxsenixLyricsPayload(raw, "Spotify", false) -} - -func (c *SpotifyLyricsClient) FetchLyrics(spotifyID, trackName, artistName string, durationSec float64) (*LyricsResponse, error) { - trackID := normalizeSpotifyLyricsID(spotifyID) - if trackID == "" { - var err error - trackID, err = c.SearchSong(trackName, artistName, durationSec) - if err != nil { - return nil, err - } - } - return c.FetchLyricsByID(trackID) -} - -func normalizeDeezerLyricsID(raw string) string { - raw = strings.TrimSpace(raw) - if strings.HasPrefix(strings.ToLower(raw), "deezer:") { - raw = strings.TrimSpace(raw[len("deezer:"):]) - } - if strings.Contains(raw, "deezer.com/") { - raw = extractDeezerIDFromURL(raw) - } - raw = strings.TrimSpace(strings.Split(raw, "?")[0]) - if _, err := strconv.ParseInt(raw, 10, 64); err == nil { - return raw - } - return "" -} - -func (c *DeezerLyricsClient) FetchLyricsByID(trackID string, multiPersonWordByWord bool) (*LyricsResponse, error) { - params := url.Values{} - params.Set("id", trackID) - raw, err := fetchPaxsenixBody(c.httpClient, "https://lyrics.paxsenix.org/deezer/lyrics", params) - if err != nil { - return nil, fmt.Errorf("deezer lyrics fetch failed: %w", err) - } - return parsePaxsenixLyricsPayload(raw, "Deezer", multiPersonWordByWord) -} - -func (c *DeezerLyricsClient) FetchLyrics(spotifyID, trackName, artistName string, durationSec float64) (*LyricsResponse, error) { - deezerID := normalizeDeezerLyricsID(spotifyID) - if deezerID == "" { - spotifyTrackID := normalizeSpotifyLyricsID(spotifyID) - if spotifyTrackID == "" { - return nil, lyricsNotFoundErrorf("deezer provider needs a deezer id or spotify id") - } - resolvedID, err := NewSongLinkClient().GetDeezerIDFromSpotify(spotifyTrackID) - if err != nil { - return nil, fmt.Errorf("failed to resolve deezer id: %w", err) - } - deezerID = normalizeDeezerLyricsID(resolvedID) - } - if deezerID == "" { - return nil, fmt.Errorf("deezer id unavailable") - } - return c.FetchLyricsByID(deezerID, true) -} - -func (c *YouTubeLyricsClient) SearchSong(trackName, artistName string, durationSec float64) (string, error) { - query := strings.TrimSpace(trackName + " " + artistName) - if query == "" { - return "", lyricsNotFoundErrorf("empty search query") - } - - params := url.Values{} - params.Set("q", query) - raw, err := fetchPaxsenixBody(c.httpClient, "https://lyrics.paxsenix.org/youtube/search", params) - if err != nil { - return "", fmt.Errorf("youtube search failed: %w", err) - } - - var results []youtubeLyricsSearchResult - if err := json.Unmarshal([]byte(raw), &results); err != nil { - return "", fmt.Errorf("failed to decode youtube search: %w", err) - } - best := selectBestYouTubeLyricsSearchResult(results, trackName, artistName, durationSec) - if best == nil || strings.TrimSpace(best.VideoID) == "" { - return "", lyricsNotFoundErrorf("no songs found on youtube") - } - return strings.TrimSpace(best.VideoID), nil -} - -func selectBestYouTubeLyricsSearchResult(results []youtubeLyricsSearchResult, trackName, artistName string, durationSec float64) *youtubeLyricsSearchResult { - best := selectBestLyricsCandidate(len(results), trackName, artistName, durationSec, func(i int) (string, string, float64, bool) { - result := &results[i] - duration := parseClockDuration(result.Duration) - artistMatches := lyricsSearchArtistsMatch(result.Author, artistName) || - lyricsSearchArtistAppearsInTitle(result.Title, artistName) - ok := lyricsSearchTitlesMatch(result.Title, trackName, true) && - artistMatches && - lyricsSearchDurationMatches(duration, durationSec) - return result.Title, result.Author, duration, ok - }) - if best < 0 { - return nil - } - return &results[best] -} - -func (c *YouTubeLyricsClient) FetchLyrics(trackName, artistName string, durationSec float64) (*LyricsResponse, error) { - videoID, err := c.SearchSong(trackName, artistName, durationSec) - if err != nil { - return nil, err - } - - params := url.Values{} - params.Set("id", videoID) - raw, err := fetchPaxsenixBody(c.httpClient, "https://lyrics.paxsenix.org/youtube/lyrics", params) - if err != nil { - return nil, fmt.Errorf("youtube lyrics fetch failed: %w", err) - } - return parsePaxsenixLyricsPayload(raw, "YouTube", false) -} - -func (c *GeniusLyricsClient) SearchSong(trackName, artistName string, durationSec float64) (string, error) { - query := strings.TrimSpace(trackName + " " + artistName) - if query == "" { - return "", lyricsNotFoundErrorf("empty search query") - } - - params := url.Values{} - params.Set("q", query) - params.Set("per_page", "5") - raw, err := fetchPaxsenixBody(c.httpClient, "https://genius.com/api/search/multi", params) - if err != nil { - return "", fmt.Errorf("genius search failed: %w", err) - } - - var results geniusSearchResponse - if err := json.Unmarshal([]byte(raw), &results); err != nil { - return "", fmt.Errorf("failed to decode genius search: %w", err) - } - - bestURL := selectBestGeniusLyricsSearchResult(results, trackName, artistName, durationSec) - if bestURL == "" { - return "", lyricsNotFoundErrorf("no songs found on genius") - } - return bestURL, nil -} - -func selectBestGeniusLyricsSearchResult(results geniusSearchResponse, trackName, artistName string, durationSec float64) string { - bestURL := "" - bestScore := -1 - for _, section := range results.Response.Sections { - for _, hit := range section.Hits { - if hit.Type != "song" || strings.TrimSpace(hit.Result.URL) == "" { - continue - } - - artist := hit.Result.PrimaryArtistNames - if strings.TrimSpace(artist) == "" { - artist = hit.Result.ArtistNames - } - if !lyricsSearchTitlesMatch(hit.Result.Title, trackName, false) || - !lyricsSearchArtistsMatch(artist, artistName) { - continue - } - score := scoreLyricsSearchCandidate(hit.Result.Title, artist, 0, trackName, artistName, durationSec) - if score > bestScore { - bestScore = score - bestURL = strings.TrimSpace(hit.Result.URL) - } - } - } - return bestURL -} - -func (c *GeniusLyricsClient) FetchLyrics(trackName, artistName string, durationSec float64) (*LyricsResponse, error) { - geniusURL, err := c.SearchSong(trackName, artistName, durationSec) - if err != nil { - return nil, err - } - return c.fetchLyricsFromPage(geniusURL) -} - -func scoreLyricsSearchCandidate(candidateTrack, candidateArtist string, candidateDuration float64, trackName, artistName string, durationSec float64) int { - normalizedTrack := strings.ToLower(strings.TrimSpace(simplifyTrackName(trackName))) - normalizedArtist := strings.ToLower(strings.TrimSpace(normalizeArtistName(artistName))) - candidateTrack = strings.ToLower(strings.TrimSpace(simplifyTrackName(candidateTrack))) - candidateArtist = strings.ToLower(strings.TrimSpace(normalizeArtistName(candidateArtist))) - - score := 0 - switch { - case candidateTrack == normalizedTrack: - score += 50 - case strings.Contains(candidateTrack, normalizedTrack) || strings.Contains(normalizedTrack, candidateTrack): - score += 25 - } - - switch { - case candidateArtist == normalizedArtist: - score += 60 - case strings.Contains(candidateArtist, normalizedArtist) || strings.Contains(normalizedArtist, candidateArtist): - score += 30 - } - - if durationSec > 0 && candidateDuration > 0 { - diff := math.Abs(candidateDuration - durationSec) - if diff <= durationToleranceSec { - score += 20 - } - } - - return score -} - -func parseClockDuration(value string) float64 { - value = strings.TrimSpace(value) - if value == "" { - return 0 - } - - parts := strings.Split(value, ":") - total := 0 - for _, part := range parts { - n, err := strconv.Atoi(strings.TrimSpace(part)) - if err != nil { - return 0 - } - total = total*60 + n - } - return float64(total) -} diff --git a/go_backend/lyrics_qqmusic.go b/go_backend/lyrics_qqmusic.go deleted file mode 100644 index defc503f..00000000 --- a/go_backend/lyrics_qqmusic.go +++ /dev/null @@ -1,188 +0,0 @@ -package gobackend - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "strings" - "time" -) - -const ( - qqMusicSearchURL = "https://c.y.qq.com/soso/fcgi-bin/client_search_cp" - qqMusicLyricsURL = "https://c.y.qq.com/lyric/fcgi-bin/fcg_query_lyric_new.fcg" - maxQQMusicResponseSize = 2 << 20 -) - -// QQMusicClient fetches line-synchronised lyrics from QQ Music's public web -// endpoints. Word-level timing remains available through lyrics extensions. -type QQMusicClient struct { - httpClient *http.Client -} - -type qqMusicSearchResult struct { - Mid string `json:"mid"` - ID int64 `json:"id"` - Name string `json:"name"` - Interval int `json:"interval"` - Singer []struct { - Name string `json:"name"` - } `json:"singer"` -} - -type qqMusicSearchResponse struct { - Code int `json:"code"` - Data struct { - Song struct { - List []qqMusicSearchResult `json:"list"` - } `json:"song"` - } `json:"data"` -} - -type qqMusicLyricsResponse struct { - RetCode int `json:"retcode"` - Code int `json:"code"` - Lyric string `json:"lyric"` -} - -func NewQQMusicClient() *QQMusicClient { - return &QQMusicClient{httpClient: NewMetadataHTTPClient(15 * time.Second)} -} - -func fetchQQMusicBody(client *http.Client, endpoint string, params url.Values) ([]byte, error) { - req, err := http.NewRequest(http.MethodGet, endpoint+"?"+params.Encode(), nil) - if err != nil { - return nil, fmt.Errorf("failed to create QQ Music request: %w", err) - } - req.Header.Set("Accept", "application/json") - req.Header.Set("Referer", "https://y.qq.com/") - req.Header.Set("User-Agent", getRandomUserAgent()) - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, lyricsServiceUnavailableErrorf("QQ Music returned HTTP %d", resp.StatusCode) - } - - body, err := io.ReadAll(io.LimitReader(resp.Body, maxQQMusicResponseSize+1)) - if err != nil { - return nil, fmt.Errorf("failed to read QQ Music response: %w", err) - } - if len(body) > maxQQMusicResponseSize { - return nil, lyricsServiceUnavailableErrorf("QQ Music response exceeds %d bytes", maxQQMusicResponseSize) - } - if len(strings.TrimSpace(string(body))) == 0 { - return nil, lyricsServiceUnavailableErrorf("QQ Music returned an empty response") - } - return body, nil -} - -func (c *QQMusicClient) searchSong(trackName, artistName string, durationSec float64) (*qqMusicSearchResult, error) { - query := strings.TrimSpace(trackName + " " + artistName) - if query == "" { - return nil, lyricsNotFoundErrorf("empty search query") - } - - params := url.Values{ - "format": {"json"}, "inCharset": {"utf8"}, "outCharset": {"utf8"}, - "platform": {"yqq.json"}, "new_json": {"1"}, "w": {query}, - "p": {"1"}, "n": {"20"}, "t": {"0"}, "aggr": {"1"}, - "cr": {"1"}, "catZhida": {"1"}, "lossless": {"1"}, - "flag_qc": {"0"}, "remoteplace": {"txt.yqq.center"}, "needNewCode": {"0"}, - } - raw, err := fetchQQMusicBody(c.httpClient, qqMusicSearchURL, params) - if err != nil { - return nil, fmt.Errorf("QQ Music search failed: %w", err) - } - var response qqMusicSearchResponse - if err := json.Unmarshal(raw, &response); err != nil { - return nil, fmt.Errorf("failed to decode QQ Music search: %w", err) - } - if response.Code != 0 { - return nil, lyricsServiceUnavailableErrorf("QQ Music search returned code %d", response.Code) - } - best := selectBestQQMusicSearchResult(response.Data.Song.List, trackName, artistName, durationSec) - if best == nil || strings.TrimSpace(best.Mid) == "" { - return nil, lyricsNotFoundErrorf("no matching song found on QQ Music") - } - return best, nil -} - -func selectBestQQMusicSearchResult(results []qqMusicSearchResult, trackName, artistName string, durationSec float64) *qqMusicSearchResult { - best := selectBestLyricsCandidate(len(results), trackName, artistName, durationSec, func(i int) (string, string, float64, bool) { - result := &results[i] - artists := make([]string, 0, len(result.Singer)) - for _, singer := range result.Singer { - if name := strings.TrimSpace(singer.Name); name != "" { - artists = append(artists, name) - } - } - candidateArtist := strings.Join(artists, ", ") - duration := float64(result.Interval) - matches := lyricsSearchTitlesMatch(result.Name, trackName, false) && - lyricsSearchArtistsMatch(candidateArtist, artistName) && - lyricsSearchDurationMatches(duration, durationSec) - return result.Name, candidateArtist, duration, matches - }) - if best < 0 { - return nil - } - return &results[best] -} - -func decodeQQMusicLyric(raw string) (string, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return "", lyricsNotFoundErrorf("QQ Music returned empty lyrics") - } - if strings.HasPrefix(raw, "[") { - return raw, nil - } - decoded, err := base64.StdEncoding.DecodeString(raw) - if err != nil { - if decoded, rawErr := base64.RawStdEncoding.DecodeString(raw); rawErr == nil { - return string(decoded), nil - } - return "", lyricsServiceUnavailableErrorf("invalid QQ Music lyrics encoding") - } - return string(decoded), nil -} - -func (c *QQMusicClient) FetchLyrics(trackName, artistName string, durationSec float64, _ bool) (*LyricsResponse, error) { - match, err := c.searchSong(trackName, artistName, durationSec) - if err != nil { - return nil, err - } - params := url.Values{ - "format": {"json"}, "inCharset": {"utf8"}, "outCharset": {"utf-8"}, - "notice": {"0"}, "platform": {"yqq.json"}, "needNewCode": {"0"}, - "songmid": {match.Mid}, "songid": {strconv.FormatInt(match.ID, 10)}, - } - raw, err := fetchQQMusicBody(c.httpClient, qqMusicLyricsURL, params) - if err != nil { - return nil, fmt.Errorf("QQ Music lyrics fetch failed: %w", err) - } - var response qqMusicLyricsResponse - if err := json.Unmarshal(raw, &response); err != nil { - return nil, fmt.Errorf("failed to decode QQ Music lyrics: %w", err) - } - if response.RetCode != 0 || response.Code != 0 { - return nil, lyricsServiceUnavailableErrorf("QQ Music lyrics returned code %d", response.Code) - } - lrc, err := decodeQQMusicLyric(response.Lyric) - if err != nil { - return nil, err - } - lyrics := lyricsResponseFromLRCText(lrc, "QQ Music", "QQ Music Direct") - if !lyricsHasUsableText(lyrics) { - return nil, lyricsNotFoundErrorf("no lyrics found on QQ Music") - } - return lyrics, nil -} diff --git a/go_backend/lyrics_search_matching_test.go b/go_backend/lyrics_search_matching_test.go deleted file mode 100644 index 907d4e5f..00000000 --- a/go_backend/lyrics_search_matching_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "testing" -) - -func TestLyricsSearchSelectorsRejectUnrelatedSongWithMatchingArtistAndDuration(t *testing.T) { - const ( - trackName = "SIX" - artistName = "Guru Randhawa" - durationSec = 186 - ) - - if best := selectBestSpotifyLyricsSearchResult( - []spotifyLyricsSearchResult{{ - TrackID: "azul", - Name: "Azul", - ArtistName: artistName, - Duration: "3:06", - }}, - trackName, - artistName, - durationSec, - ); best != nil { - t.Fatalf("Spotify accepted unrelated result: %#v", best) - } - - if best := selectBestYouTubeLyricsSearchResult( - []youtubeLyricsSearchResult{{ - VideoID: "azul", - Title: "Azul", - Author: artistName, - Duration: "3:06", - }}, - trackName, - artistName, - durationSec, - ); best != nil { - t.Fatalf("YouTube accepted unrelated result: %#v", best) - } - - if best := selectBestKugouLyricsSearchResult( - []kugouLyricsSearchResult{{ - ID: "azul", - AccessKey: "key", - Title: "Azul", - Artist: artistName, - Duration: durationSec * 1000, - }}, - trackName, - artistName, - durationSec, - ); best != nil { - t.Fatalf("Kugou accepted unrelated result: %#v", best) - } - - var geniusResults geniusSearchResponse - if err := json.Unmarshal([]byte(`{ - "response": { - "sections": [{ - "hits": [{ - "type": "song", - "result": { - "title": "Azul", - "primary_artist_names": "Guru Randhawa", - "url": "https://genius.com/guru-randhawa-azul-lyrics" - } - }] - }] - } - }`), &geniusResults); err != nil { - t.Fatalf("decode Genius fixture: %v", err) - } - if bestURL := selectBestGeniusLyricsSearchResult( - geniusResults, - trackName, - artistName, - durationSec, - ); bestURL != "" { - t.Fatalf("Genius accepted unrelated result: %q", bestURL) - } - - var neteaseResults neteaseSearchResponse - if err := json.Unmarshal([]byte(`{ - "result": { - "songCount": 1, - "songs": [{ - "name": "Azul", - "id": 123, - "artists": [{"name": "Guru Randhawa"}] - }] - }, - "code": 200 - }`), &neteaseResults); err != nil { - t.Fatalf("decode Netease fixture: %v", err) - } - if best := selectBestNeteaseSearchResult( - neteaseResults.Result.Songs, - trackName, - artistName, - ); best != nil { - t.Fatalf("Netease accepted unrelated result: %#v", best) - } - - lrclibResult := &LRCLibResponse{ - TrackName: "Azul", - ArtistName: artistName, - Duration: durationSec, - SyncedLyrics: "[00:01.00]Wrong", - } - if lrclibSearchResultMatches( - lrclibResult, - "Guru Randhawa SIX", - trackName, - artistName, - durationSec, - ) { - t.Fatalf("LRCLIB accepted unrelated result: %#v", lrclibResult) - } -} - -func TestYouTubeLyricsSearchAllowsDecoratedTitleWithArtistSignal(t *testing.T) { - results := []youtubeLyricsSearchResult{{ - VideoID: "six", - Title: "Guru Randhawa - SIX (Official Music Video)", - Author: "T-Series", - Duration: "3:06", - }} - - best := selectBestYouTubeLyricsSearchResult( - results, - "SIX", - "Guru Randhawa", - 186, - ) - if best == nil || best.VideoID != "six" { - t.Fatalf("expected decorated YouTube result to match, got %#v", best) - } -} - -func TestDecoratedLyricsTitleMatchingUsesWholeWords(t *testing.T) { - if !lyricsSearchTitlesMatch( - "Guru Randhawa - SIX (Official Music Video)", - "SIX", - true, - ) { - t.Fatal("expected decorated SIX title to match") - } - if lyricsSearchTitlesMatch("SIXTEEN", "SIX", true) { - t.Fatal("SIX must not match SIXTEEN") - } - if lyricsSearchArtistsMatch("Guru Randhawa Tribute", "Guru Randhawa") { - t.Fatal("artist matching must not accept a longer unrelated name") - } -} diff --git a/go_backend/lyrics_supplement_test.go b/go_backend/lyrics_supplement_test.go deleted file mode 100644 index fcc3181a..00000000 --- a/go_backend/lyrics_supplement_test.go +++ /dev/null @@ -1,584 +0,0 @@ -package gobackend - -import ( - "encoding/base64" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "testing" - "time" -) - -func TestLyricsLookupSingleflightAndPersistentCache(t *testing.T) { - SetLyricsProviderOrder([]string{LyricsProviderLRCLIB}) - defer SetLyricsProviderOrder(nil) - clearLyricsProviderHealth() - globalLyricsCache.ClearAll() - - var calls atomic.Int32 - client := &LyricsClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls.Add(1) - time.Sleep(25 * time.Millisecond) - return &http.Response{ - StatusCode: 200, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"id":1,"trackName":"Singleflight Song","artistName":"Cache Artist","duration":180,"plainLyrics":"Cached lyric"}`, - )), - Request: req, - }, nil - })}} - - var wait sync.WaitGroup - for range 8 { - wait.Add(1) - go func() { - defer wait.Done() - lyrics, err := client.FetchLyricsAllSources("", "Singleflight Song", "Cache Artist", 180) - if err != nil || lyrics == nil { - t.Errorf("FetchLyricsAllSources = %#v/%v", lyrics, err) - } - }() - } - wait.Wait() - if got := calls.Load(); got != 1 { - t.Fatalf("identical lyrics lookups made %d HTTP calls, want 1", got) - } - - path := filepath.Join(t.TempDir(), "lyrics-cache.json") - cache := &lyricsCache{cache: make(map[string]*lyricsCacheEntry)} - cache.SetPersistencePath(path) - cache.Set("Cache Artist", "Persistent Song", 180, &LyricsResponse{PlainLyrics: "Persisted", Source: "test"}) - deadline := time.Now().Add(2 * time.Second) - for { - if _, err := os.Stat(path); err == nil { - break - } - if time.Now().After(deadline) { - t.Fatal("persistent lyrics cache was not written") - } - time.Sleep(20 * time.Millisecond) - } - - reloaded := &lyricsCache{cache: make(map[string]*lyricsCacheEntry)} - reloaded.SetPersistencePath(path) - lyrics, ok := reloaded.Get("Cache Artist", "Persistent Song", 180) - if !ok || lyrics.PlainLyrics != "Persisted" { - t.Fatalf("reloaded persistent lyrics = %#v/%v", lyrics, ok) - } -} - -func TestLyricsCacheParsingAndLRCLibClient(t *testing.T) { - SetAppVersion("4.5.0") - if ua := appUserAgent(); !strings.Contains(ua, "4.5.0") { - t.Fatalf("user agent = %q", ua) - } - SetLyricsProviderOrder([]string{"LRCLIB", "bad", "extension:Lyrics-Fixture", "netease", "extension:lyrics-fixture"}) - if providers := GetLyricsProviderOrder(); len(providers) != 3 || providers[0] != LyricsProviderLRCLIB || providers[1] != "extension:lyrics-fixture" { - t.Fatalf("providers = %#v", providers) - } - SetLyricsProviderOrder(nil) - SetLyricsFetchOptions(LyricsFetchOptions{MusixmatchLanguage: " EN_us!!too-long-value ", MultiPersonWordByWord: true}) - if opts := GetLyricsFetchOptions(); !strings.HasPrefix(opts.MusixmatchLanguage, "en_us") || len(opts.MusixmatchLanguage) > 16 { - t.Fatalf("options = %#v", opts) - } - - cache := &lyricsCache{cache: map[string]*lyricsCacheEntry{}} - response := &LyricsResponse{PlainLyrics: "Hello", Source: "test"} - cache.Set(" Artist ", " Song ", 184, response) - if got, ok := cache.Get("artist", "song", 180); !ok || got.PlainLyrics != "Hello" { - t.Fatalf("cache get = %#v/%v", got, ok) - } - cache.cache["expired"] = &lyricsCacheEntry{response: response, expiresAt: time.Now().Add(-time.Hour)} - if cleaned := cache.CleanExpired(); cleaned != 1 { - t.Fatalf("cleaned = %d", cleaned) - } - if cache.Size() != 1 || cache.ClearAll() != 1 || cache.Size() != 0 { - t.Fatalf("cache size after clear = %d", cache.Size()) - } - - lines := parseSyncedLyrics("[00:01.20]Hello\n[bg:Harmony]\n[00:02.300]World\n[00:03.00]\n") - if len(lines) != 2 || !strings.Contains(lines[0].Words, "[bg:Harmony]") || lines[0].EndTimeMs != lines[1].StartTimeMs { - t.Fatalf("synced lines = %#v", lines) - } - if plain := plainLyricsFromTimedLines(lines); !strings.Contains(plain, "Hello") { - t.Fatalf("plain = %q", plain) - } - if unsynced := plainTextLyricsLines("A\n\n B "); len(unsynced) != 2 { - t.Fatalf("unsynced = %#v", unsynced) - } - if !lyricsHasUsableText(&LyricsResponse{Instrumental: true}) || lyricsHasUsableText(&LyricsResponse{}) { - t.Fatal("unexpected usable lyrics result") - } - if msg, ok := detectLyricsErrorPayload(`{"success":false,"message":"nope"}`); !ok || msg != "nope" { - t.Fatalf("error payload = %q/%v", msg, ok) - } - if msg, ok := detectLyricsErrorPayload(`{"isError":true,"error":"Missing required parameters"}`); !ok || msg != "Missing required parameters" { - t.Fatalf("isError payload = %q/%v", msg, ok) - } - if msg, ok := detectLyricsErrorPayload(`{"code":405,"message":"rate limited"}`); !ok || msg != "rate limited" { - t.Fatalf("coded error payload = %q/%v", msg, ok) - } - if !isLyricsProviderUnavailableError(classifyLyricsPayloadError(0, "rate limit", "proxy error: %s", "rate limit")) { - t.Fatal("expected rate-limit payloads to mark provider unavailable") - } - if !isLyricsProviderUnavailableError(fmt.Errorf("spotify search failed: %w", lyricsServiceUnavailableErrorf("HTTP 503"))) { - t.Fatal("expected wrapped unavailable errors to keep their classification") - } - if !isLyricsProviderUnavailableError(lyricsHTTPStatusError(503, "proxy returned HTTP 503")) { - t.Fatal("expected 5xx statuses to mark provider unavailable") - } - if isLyricsProviderUnavailableError(lyricsHTTPStatusError(403, "proxy returned HTTP 403")) { - t.Fatal("4xx statuses other than 429 must not mark provider unavailable") - } - if isLyricsProviderUnavailableError(classifyLyricsPayloadError(500, "lyrics not found", "HTTP 500: lyrics not found")) { - t.Fatal("not-found payloads must never mark provider unavailable") - } - if isLyricsProviderUnavailableError(errors.New("rate limit")) { - t.Fatal("untyped errors must not be classified by message text") - } - if lrcTimestampToMs("01", "02", "345") != 62345 || msToLRCTimestamp(62340) != "[01:02.34]" { - t.Fatal("unexpected LRC timestamp conversion") - } - lrc := convertToLRCWithMetadata(&LyricsResponse{SyncType: "LINE_SYNCED", Lines: lines}, "Song", "Artist") - if !strings.Contains(lrc, "[ti:Song]") || !strings.Contains(lrc, "Hello") { - t.Fatalf("lrc = %q", lrc) - } - if got := simplifyTrackName("Song (feat. Guest) - 2020 Remaster"); got != "song" { - t.Fatalf("simplified = %q", got) - } - if got := normalizeArtistName("Artist feat. Guest"); got != "Artist" { - t.Fatalf("artist = %q", got) - } - if !isLikelyInstrumentalTrack("Song (Instrumental)") || isLikelyInstrumentalTrack("Song") { - t.Fatal("instrumental heuristic mismatch") - } - - dir := t.TempDir() - lrcPath, err := SaveLRCFile(filepath.Join(dir, "song.flac"), lrc) - if err != nil { - t.Fatalf("SaveLRCFile: %v", err) - } - if !strings.HasSuffix(lrcPath, ".lrc") { - t.Fatalf("lrc path = %q", lrcPath) - } - if _, err := SaveLRCFile(filepath.Join(dir, "empty.flac"), ""); err == nil { - t.Fatal("expected empty LRC error") - } - - client := &LyricsClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch req.URL.Path { - case "/api/get": - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"id":1,"trackName":"Song","artistName":"Artist","duration":180,"syncedLyrics":"[00:01.00]Hello"}`)), Request: req}, nil - case "/api/search": - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`[{"id":2,"trackName":"Song","artistName":"Artist","duration":180,"plainLyrics":"Plain\nLyric"},{"id":3,"trackName":"Song","artistName":"Artist","duration":180,"syncedLyrics":"[00:02.00]Synced"}]`)), Request: req}, nil - default: - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}} - got, err := client.FetchLyricsWithMetadata("Artist", "Song") - if err != nil || got.SyncType != "LINE_SYNCED" || len(got.Lines) != 1 { - t.Fatalf("FetchLyricsWithMetadata = %#v/%v", got, err) - } - search, err := client.fetchLyricsFromLRCLibSearch("Artist Song", "", "", 180) - if err != nil || len(search.Lines) == 0 { - t.Fatalf("fetchLyricsFromLRCLibSearch = %#v/%v", search, err) - } - if best := client.findBestLRCLibSearchMatch( - []LRCLibResponse{ - {TrackName: "Other", ArtistName: "Artist", Duration: 180, PlainLyrics: "A"}, - {TrackName: "Song", ArtistName: "Artist", Duration: 180, SyncedLyrics: "[00:01.00]B"}, - }, - "Artist Song", - "Song", - "Artist", - 180, - ); best == nil || best.SyncedLyrics == "" { - t.Fatalf("best = %#v", best) - } - if !client.durationMatches(181, 180) || client.durationMatches(300, 180) { - t.Fatal("duration match mismatch") - } - parsed := client.parseLRCLibResponse(&LRCLibResponse{PlainLyrics: "A\nB"}) - if parsed.SyncType != "UNSYNCED" || len(parsed.Lines) != 2 { - t.Fatalf("parsed plain = %#v", parsed) - } - - allSources := &LyricsClient{httpClient: client.httpClient} - SetLyricsProviderOrder([]string{LyricsProviderLRCLIB}) - globalLyricsCache.ClearAll() - all, err := allSources.FetchLyricsAllSources("", "Song (Instrumental)", "Artist", 180) - if err != nil || !all.Instrumental { - t.Fatalf("instrumental all sources = %#v/%v", all, err) - } - globalLyricsCache.ClearAll() - all, err = allSources.FetchLyricsAllSources("", "Song", "Artist", 180) - if err != nil || len(all.Lines) == 0 { - t.Fatalf("all sources = %#v/%v", all, err) - } - cached, err := allSources.FetchLyricsAllSources("", "Song", "Artist", 180) - if err != nil || !strings.Contains(cached.Source, "cached") { - t.Fatalf("cached all sources = %#v/%v", cached, err) - } -} - -func TestPaxsenixEmptyLyricsPayloadMarksProviderUnavailable(t *testing.T) { - lyrics, err := parsePaxsenixLyricsPayload( - `{"error":false,"lyrics":""}`, - "Genius", - false, - ) - if lyrics != nil || err == nil { - t.Fatalf("empty PAX Senix payload = %#v/%v", lyrics, err) - } - if !isLyricsProviderUnavailableError(err) { - t.Fatalf("empty PAX Senix payload was not marked unavailable: %v", err) - } - if strings.Contains(strings.ToLower(err.Error()), "decode") { - t.Fatalf("empty JSON payload was misreported as a decode failure: %v", err) - } -} - -func TestLyricsProviderHealthSkipsUnavailableProvider(t *testing.T) { - SetLyricsProviderOrder([]string{LyricsProviderLRCLIB}) - defer SetLyricsProviderOrder(nil) - globalLyricsCache.ClearAll() - clearLyricsProviderHealth() - defer clearLyricsProviderHealth() - - calls := 0 - downClient := &LyricsClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - calls++ - return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`service unavailable`)), Request: req}, nil - })}} - - if lyrics, err := downClient.FetchLyricsAllSources("", "Down Song", "Artist", 180); err == nil || lyrics != nil { - t.Fatalf("expected unavailable provider error, got %#v/%v", lyrics, err) - } - if calls != 1 { - t.Fatalf("expected one HTTP call before cooldown, got %d", calls) - } - if skip, _, _ := shouldSkipLyricsProvider(LyricsProviderLRCLIB); !skip { - t.Fatal("expected LRCLIB to be marked unavailable") - } - if lyrics, err := downClient.FetchLyricsAllSources("", "Another Song", "Artist", 180); err == nil || lyrics != nil { - t.Fatalf("expected skipped provider error, got %#v/%v", lyrics, err) - } - if calls != 1 { - t.Fatalf("provider was called while in cooldown, calls=%d", calls) - } - - clearLyricsProviderHealth() - globalLyricsCache.ClearAll() - notFoundCalls := 0 - notFoundClient := &LyricsClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - notFoundCalls++ - switch req.URL.Path { - case "/api/get": - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - case "/api/search": - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`[]`)), Request: req}, nil - default: - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}} - - if lyrics, err := notFoundClient.FetchLyricsAllSources("", "missing song", "Artist", 180); err == nil || lyrics != nil { - t.Fatalf("expected not found error, got %#v/%v", lyrics, err) - } - if skip, _, _ := shouldSkipLyricsProvider(LyricsProviderLRCLIB); skip { - t.Fatal("not-found result must not mark provider unavailable") - } - if lyrics, err := notFoundClient.FetchLyricsAllSources("", "missing song 2", "Artist", 180); err == nil || lyrics != nil { - t.Fatalf("expected second not found error, got %#v/%v", lyrics, err) - } - if notFoundCalls != 4 { - t.Fatalf("expected not-found provider to be retried, calls=%d", notFoundCalls) - } -} - -func TestConcurrentLyricsProvidersReturnFastFallback(t *testing.T) { - clearLyricsProviderHealth() - defer clearLyricsProviderHealth() - - start := time.Now() - lyrics, err := fetchLyricsProviders( - []string{LyricsProviderLRCLIB, LyricsProviderAppleMusic}, - lyricsProviderSearchRequest{}, - func(providerName string, _ lyricsProviderSearchRequest) (*LyricsResponse, error, bool) { - if providerName == LyricsProviderLRCLIB { - time.Sleep(lyricsProviderPriorityGrace + 800*time.Millisecond) - return &LyricsResponse{Provider: "LRCLIB", PlainLyrics: "slow"}, nil, true - } - return &LyricsResponse{Provider: "Apple Music", PlainLyrics: "fast"}, nil, true - }, - ) - if err != nil { - t.Fatalf("concurrent providers returned error: %v", err) - } - if lyrics == nil || lyrics.Provider != "Apple Music" { - t.Fatalf("expected fast fallback lyrics, got %#v", lyrics) - } - if elapsed := time.Since(start); elapsed >= lyricsProviderPriorityGrace+700*time.Millisecond { - t.Fatalf("fallback waited too long: %s", elapsed) - } -} - -func TestResolveLyricsProviderOrderOnlyIncludesSelectedAvailableExtensions(t *testing.T) { - availableExtensions := map[string]*extensionProviderWrapper{ - "extension:lyrics-fixture": nil, - "extension:future-provider": nil, - } - providers := resolveLyricsProviderOrder( - []string{ - LyricsProviderLRCLIB, - "extension:future-provider", - "extension:not-installed", - LyricsProviderNetease, - }, - availableExtensions, - ) - - want := []string{ - LyricsProviderLRCLIB, - "extension:future-provider", - LyricsProviderNetease, - } - if !equalLyricsProviderOrders(providers, want) { - t.Fatalf("providers = %#v, want %#v", providers, want) - } -} - -func TestConcurrentLyricsProvidersPreferEarlierProviderWithinGrace(t *testing.T) { - clearLyricsProviderHealth() - defer clearLyricsProviderHealth() - - lyrics, err := fetchLyricsProviders( - []string{LyricsProviderLRCLIB, LyricsProviderAppleMusic}, - lyricsProviderSearchRequest{}, - func(providerName string, _ lyricsProviderSearchRequest) (*LyricsResponse, error, bool) { - if providerName == LyricsProviderLRCLIB { - time.Sleep(50 * time.Millisecond) - return &LyricsResponse{Provider: "LRCLIB", PlainLyrics: "preferred"}, nil, true - } - return &LyricsResponse{Provider: "Apple Music", PlainLyrics: "fast"}, nil, true - }, - ) - if err != nil { - t.Fatalf("concurrent providers returned error: %v", err) - } - if lyrics == nil || lyrics.Provider != "LRCLIB" { - t.Fatalf("expected preferred provider lyrics, got %#v", lyrics) - } -} - -func TestExternalLyricsProvidersWithFakeHTTP(t *testing.T) { - clearAppleMusicToken() - defer clearAppleMusicToken() - if len(lyricsPlusServers) == 0 || lyricsPlusServers[0] != "https://lyricsplus.prjktla.workers.dev" { - t.Fatalf("unexpected LyricsPlus server order = %#v", lyricsPlusServers) - } - - paxJSON := `{"type":"Syllable","content":[{"timestamp":1000,"oppositeTurn":true,"background":true,"text":[{"text":"Hel","part":true,"timestamp":1000},{"text":"lo","part":false,"timestamp":1200,"endtime":1500}],"backgroundText":[{"text":"bg","part":false,"timestamp":900}]}]}` - apple := &AppleMusicClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch { - case req.URL.Host == "beta.music.apple.com" && (req.URL.Path == "" || req.URL.Path == "/"): - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`<script src="/assets/index~test.js"></script>`)), Request: req}, nil - case req.URL.Host == "beta.music.apple.com" && req.URL.Path == "/assets/index~test.js": - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`const token="eyJ0eXAiOiJKV1Q.eyJpc3MiOiJ0ZXN0.c2ln";`)), Request: req}, nil - case req.URL.Host == "amp-api.music.apple.com" && strings.Contains(req.URL.Path, "/v1/catalog/us/search"): - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"results":{"songs":{"data":[{"id":"apple-2"},{"id":"apple-1"}]}},"resources":{"songs":{"apple-2":{"attributes":{"name":"Other","artistName":"Other","durationInMillis":1000}},"apple-1":{"attributes":{"name":"Song","artistName":"Artist","albumName":"Album","durationInMillis":180000}}}}}`)), Request: req}, nil - case strings.Contains(req.URL.Path, "/apple-music/lyrics"): - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(paxJSON)), Request: req}, nil - default: - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}} - if best := selectBestAppleMusicSearchResult([]appleMusicSearchResult{{ID: "1", SongName: "Song", ArtistName: "Artist", Duration: 180000}}, "Song", "Artist", 180); best == nil || best.ID != "1" { - t.Fatalf("best apple result = %#v", best) - } - appleID, err := apple.SearchSong("Song", "Artist", 180) - if err != nil || appleID != "apple-1" { - t.Fatalf("apple SearchSong = %q/%v", appleID, err) - } - rawApple, err := apple.FetchLyricsByID(appleID) - if err != nil || !strings.Contains(rawApple, "Syllable") { - t.Fatalf("apple raw = %q/%v", rawApple, err) - } - appleLyrics, err := apple.FetchLyrics("Song", "Artist", 180, true, true) - if err != nil || appleLyrics.SyncType != "LINE_SYNCED" || appleLyrics.Provider != "Apple Music" { - t.Fatalf("apple lyrics = %#v/%v", appleLyrics, err) - } - if plain, err := formatPaxLyricsToLRC(`[{"timestamp":2000,"text":[{"text":"Plain","part":false}]}]`, false, false); err != nil || !strings.Contains(plain, "Plain") { - t.Fatalf("direct pax = %q/%v", plain, err) - } - lineOnly, err := formatPaxLyricsToLRC(paxJSON, true, false) - if err != nil { - t.Fatalf("line-only pax = %v", err) - } - if strings.Contains(lineOnly, "<00:") { - t.Fatalf("line-only pax should not include inline word timing: %q", lineOnly) - } - elrc, err := formatPaxLyricsToLRC(paxJSON, true, true) - if err != nil { - t.Fatalf("elrc pax = %v", err) - } - if !strings.Contains(elrc, "<00:") { - t.Fatalf("elrc pax should include inline word timing: %q", elrc) - } - if preferred, err := formatPaxLyricsToLRC(`{"elrcMultiPerson":"[00:01.00]v1:<00:01.00>Hello","content":[{"timestamp":1000,"text":[{"text":"Fallback","part":false}]}]}`, true, true); err != nil || !strings.Contains(preferred, "Hello") { - t.Fatalf("preferred apple elrc = %q/%v", preferred, err) - } - if _, err := apple.SearchSong("", "", 0); err == nil { - t.Fatal("expected empty apple search error") - } - - musixmatch := &MusixmatchClient{ - httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - lyricsType := req.URL.Query().Get("type") - lang := req.URL.Query().Get("l") - if req.URL.Query().Get("t") == "bad" { - return &http.Response{StatusCode: 429, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":"rate limited"}`)), Request: req}, nil - } - if lyricsType == "translate" && lang == "id" { - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`"[00:01.00]Halo"`)), Request: req}, nil - } - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`[00:01.00]Hello`)), Request: req}, nil - })}, - baseURL: "https://lyrics.paxsenix.org/musixmatch/lyrics", - } - if localized, err := musixmatch.FetchLyricsInLanguage("Song", "Artist", 180, "id"); err != nil || localized.Source != "Musixmatch (id)" { - t.Fatalf("localized musixmatch = %#v/%v", localized, err) - } - if normal, err := musixmatch.FetchLyrics("Song", "Artist", 180, "xx"); err != nil || normal.Provider != "Musixmatch" { - t.Fatalf("musixmatch = %#v/%v", normal, err) - } - if _, err := musixmatch.FetchLyricsInLanguage("Song", "Artist", 180, " "); err == nil { - t.Fatal("expected invalid language error") - } - if _, err := musixmatch.fetchLyricsPayload("bad", "Artist", 0, "word", ""); err == nil { - t.Fatal("expected musixmatch proxy error") - } - - netease := &NeteaseClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch { - case strings.Contains(req.URL.Path, "/netease/search"): - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"code":200,"result":{"songCount":1,"songs":[{"name":"Song","id":123,"artists":[{"name":"Artist"}]}]}}`)), Request: req}, nil - case strings.Contains(req.URL.Path, "/netease/lyrics"): - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"code":200,"lrc":{"lyric":"[00:01.00]Hello"},"tlyric":{"lyric":"[00:01.00]Halo"},"romalrc":{"lyric":"[00:01.00]Romaji"}}`)), Request: req}, nil - default: - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}} - songID, err := netease.SearchSong("Song", "Artist") - if err != nil || songID != 123 { - t.Fatalf("netease search = %d/%v", songID, err) - } - netLyrics, err := netease.FetchLyrics("Song", "Artist", 180, true, true) - if err != nil || netLyrics.SyncType != "LINE_SYNCED" { - t.Fatalf("netease lyrics = %#v/%v", netLyrics, err) - } - if _, err := netease.SearchSong("", ""); err == nil { - t.Fatal("expected empty netease search error") - } - rateLimitedNetease := &NeteaseClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"msg":"操作频繁,请稍候再试","code":405,"message":"操作频繁,请稍候再试"}`)), Request: req}, nil - })}} - if _, err := rateLimitedNetease.SearchSong("Song", "Artist"); err == nil || !isLyricsProviderUnavailableError(err) { - t.Fatalf("expected unavailable netease rate-limit error, got %v", err) - } - - qq := &QQMusicClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch req.URL.Path { - case "/soso/fcgi-bin/client_search_cp": - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"code":0,"data":{"song":{"list":[{"mid":"qq-mid","id":123,"name":"Song","interval":180,"singer":[{"name":"Artist"}]}]}}}`)), Request: req}, nil - case "/lyric/fcgi-bin/fcg_query_lyric_new.fcg": - encoded := base64.StdEncoding.EncodeToString([]byte("[00:01.00]QQ Direct")) - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"retcode":0,"code":0,"lyric":"` + encoded + `"}`)), Request: req}, nil - default: - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}} - qqLyrics, err := qq.FetchLyrics("Song", "Artist", 180, false) - if err != nil || qqLyrics.Provider != "QQ Music" || qqLyrics.Source != "QQ Music Direct" || qqLyrics.SyncType != "LINE_SYNCED" { - t.Fatalf("qq lyrics = %#v/%v", qqLyrics, err) - } - - spotify := &SpotifyLyricsClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch { - case strings.Contains(req.URL.Path, "/spotify/search"): - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`[{"trackId":"spotify-1","name":"Song","artistName":"Artist","duration":"03:00"}]`)), Request: req}, nil - case strings.Contains(req.URL.Path, "/spotify/lyrics"): - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`"[00:01.00]Spotify"`)), Request: req}, nil - default: - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}} - spotifyLyrics, err := spotify.FetchLyrics("", "Song", "Artist", 180) - if err != nil || spotifyLyrics.Provider != "Spotify" || spotifyLyrics.SyncType != "LINE_SYNCED" { - t.Fatalf("spotify lyrics = %#v/%v", spotifyLyrics, err) - } - - deezer := &DeezerLyricsClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"lyrics":[{"timestamp":1000,"text":[{"text":"Deezer","part":false}]}]}`)), Request: req}, nil - })}} - deezerLyrics, err := deezer.FetchLyricsByID("123", false) - if err != nil || deezerLyrics.Provider != "Deezer" || deezerLyrics.SyncType != "LINE_SYNCED" { - t.Fatalf("deezer lyrics = %#v/%v", deezerLyrics, err) - } - - youtube := &YouTubeLyricsClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch { - case strings.Contains(req.URL.Path, "/youtube/search"): - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`[{"videoId":"yt-1","title":"Song","author":"Artist","duration":"3:00"}]`)), Request: req}, nil - case strings.Contains(req.URL.Path, "/youtube/lyrics"): - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`"[00:01.00]YouTube"`)), Request: req}, nil - default: - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}} - youtubeLyrics, err := youtube.FetchLyrics("Song", "Artist", 180) - if err != nil || youtubeLyrics.Provider != "YouTube" || youtubeLyrics.SyncType != "LINE_SYNCED" { - t.Fatalf("youtube lyrics = %#v/%v", youtubeLyrics, err) - } - - kugou := &KugouLyricsClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch { - case req.URL.Path == "/search": - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"status":200,"errcode":200,"candidates":[{"id":"kg-1","accesskey":"key","song":"Song","singer":"Artist","duration":180000}]}`)), Request: req}, nil - case req.URL.Path == "/download": - encoded := base64.StdEncoding.EncodeToString([]byte("[00:01.00]Kugou")) - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"status":200,"error_code":0,"content":"` + encoded + `"}`)), Request: req}, nil - default: - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}} - kugouLyrics, err := kugou.FetchLyrics("Song", "Artist", 180) - if err != nil || kugouLyrics.Provider != "Kugou" || kugouLyrics.Source != "Kugou Direct" || kugouLyrics.SyncType != "LINE_SYNCED" { - t.Fatalf("kugou lyrics = %#v/%v", kugouLyrics, err) - } - - genius := &GeniusLyricsClient{httpClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch { - case strings.Contains(req.URL.Path, "/api/search/multi"): - if got := req.URL.Query().Get("per_page"); got != "5" { - t.Fatalf("genius per_page = %q", got) - } - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"response":{"sections":[{"hits":[{"type":"song","result":{"title":"Song","primary_artist_names":"Artist","url":"https://genius.com/artist-song-lyrics"}}]}]}}`)), Request: req}, nil - case req.URL.Host == "genius.com" && req.URL.Path == "/artist-song-lyrics": - return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`<html><div data-lyrics-container="true"><div data-exclude-from-selection="true">Contributors</div>[00:01.00]Genius<br/>Direct line</div><div data-lyrics-container="true">[00:02.00]Second section</div></html>`)), Request: req}, nil - default: - return &http.Response{StatusCode: 404, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil - } - })}} - geniusLyrics, err := genius.FetchLyrics("Song", "Artist", 180) - if err != nil || geniusLyrics.Provider != "Genius" || geniusLyrics.Source != "Genius Direct" || geniusLyrics.SyncType != "LINE_SYNCED" || !strings.Contains(geniusLyrics.PlainLyrics, "Second section") { - t.Fatalf("genius lyrics = %#v/%v", geniusLyrics, err) - } -} diff --git a/go_backend/lyrics_usability_test.go b/go_backend/lyrics_usability_test.go deleted file mode 100644 index 04913a32..00000000 --- a/go_backend/lyrics_usability_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package gobackend - -import ( - "os" - "path/filepath" - "strconv" - "strings" - "testing" -) - -func TestSharedLyricsUsabilityCases(t *testing.T) { - data, err := os.ReadFile(filepath.Join("..", "android", "app", "src", "test", "resources", "lyrics_usability_cases.tsv")) - if err != nil { - t.Fatal(err) - } - decode := strings.NewReplacer(`\n`, "\n", `\r`, "\r", `\t`, "\t") - for _, line := range strings.Split(string(data), "\n") { - if line == "" || strings.HasPrefix(line, "#") { - continue - } - fields := strings.Split(line, "\t") - if len(fields) != 3 { - t.Fatalf("invalid shared fixture: %q", line) - } - t.Run(fields[0], func(t *testing.T) { - want, err := strconv.ParseBool(fields[1]) - if err != nil { - t.Fatal(err) - } - if got := rawLyricsHasUsableContent(decode.Replace(fields[2])); got != want { - t.Errorf("rawLyricsHasUsableContent(%q) = %v, want %v", fields[2], got, want) - } - }) - } -} diff --git a/go_backend/m4a_edit.go b/go_backend/m4a_edit.go deleted file mode 100644 index 489f5aa7..00000000 --- a/go_backend/m4a_edit.go +++ /dev/null @@ -1,414 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/binary" - "fmt" - "os" - "strings" -) - -// Native editor for the standard iTunes ilst atoms plus freeform tags, in one -// atomic pass. Unlike the ffmpeg remux path it preserves every atom it does -// not control, fixes stco/co64 chunk offsets when moov precedes mdat, and -// publishes via temp+fsync+rename. - -// m4aTextAtoms maps editor field keys to iTunes text atom types. -var m4aTextAtoms = map[string]string{ - "title": "\xa9nam", - "artist": "\xa9ART", - "album": "\xa9alb", - "album_artist": "aART", - "date": "\xa9day", - "genre": "\xa9gen", - "composer": "\xa9wrt", - "comment": "\xa9cmt", - "copyright": "cprt", - "lyrics": "\xa9lyr", -} - -func buildM4ADataAtom(dataType uint32, payload []byte) []byte { - buf := make([]byte, 8+len(payload)) - binary.BigEndian.PutUint32(buf[0:4], dataType) - copy(buf[8:], payload) - return buildM4AAtom("data", buf) -} - -func buildM4ATextAtom(typ, value string) []byte { - return buildM4AAtom(typ, buildM4ADataAtom(1, []byte(value))) -} - -func buildM4AIndexAtom(typ string, num, total int) []byte { - size := 8 // trkn: pad(2) num(2) total(2) pad(2) - if typ == "disk" { - size = 6 - } - payload := make([]byte, size) - binary.BigEndian.PutUint16(payload[2:4], uint16(num)) - binary.BigEndian.PutUint16(payload[4:6], uint16(total)) - return buildM4AAtom(typ, buildM4ADataAtom(0, payload)) -} - -func buildM4ACoverAtom(coverData []byte) []byte { - dataType := uint32(13) // JPEG - if bytes.HasPrefix(coverData, []byte("\x89PNG")) { - dataType = 14 - } - return buildM4AAtom("covr", buildM4ADataAtom(dataType, coverData)) -} - -// buildM4AFlagAtom writes an iTunes boolean atom (cpil-style, data type 22) -// whose payload is 1 when set. -func buildM4AFlagAtom(typ string, set bool) []byte { - payload := []byte{0} - if set { - payload[0] = 1 - } - return buildM4AAtom(typ, buildM4ADataAtom(22, payload)) -} - -// buildM4AInt8Atom writes an iTunes 8-bit integer atom (rtng-style, data -// type 21). -func buildM4AInt8Atom(typ string, value int) []byte { - payload := []byte{byte(value)} - return buildM4AAtom(typ, buildM4ADataAtom(21, payload)) -} - -type m4aIlstLocation struct { - moov mp4Box - ilst mp4Box - // ancestors whose size field must grow when ilst changes size; all start - // before ilst so their offsets survive the splice. - ancestors []mp4Box -} - -// locateM4AIlstInBuf finds moov > [udta >] meta > ilst. The meta box is a full -// box, so its children start 4 bytes into the body. -func locateM4AIlstInBuf(data []byte) (m4aIlstLocation, bool) { - n := int64(len(data)) - moov, ok := findChildMP4(data, 0, n, "moov") - if !ok { - return m4aIlstLocation{}, false - } - if udta, ok := findChildMP4(data, moov.body(), moov.end(), "udta"); ok { - if meta, ok := findChildMP4(data, udta.body(), udta.end(), "meta"); ok { - if ilst, ok := findChildMP4(data, meta.body()+4, meta.end(), "ilst"); ok { - return m4aIlstLocation{moov: moov, ilst: ilst, ancestors: []mp4Box{moov, udta, meta}}, true - } - } - } - if meta, ok := findChildMP4(data, moov.body(), moov.end(), "meta"); ok { - if ilst, ok := findChildMP4(data, meta.body()+4, meta.end(), "ilst"); ok { - return m4aIlstLocation{moov: moov, ilst: ilst, ancestors: []mp4Box{moov, meta}}, true - } - } - return m4aIlstLocation{}, false -} - -// buildM4AMetaBox returns a meta full box containing the iTunes mdir hdlr and -// an empty ilst. -func buildM4AMetaBox() []byte { - hdlrPayload := make([]byte, 25) // ver/flags + pre_defined + handler + 3x reserved + empty name - copy(hdlrPayload[8:12], "mdir") - copy(hdlrPayload[12:16], "appl") - body := append([]byte{0, 0, 0, 0}, buildM4AAtom("hdlr", hdlrPayload)...) - body = append(body, buildM4AAtom("ilst", nil)...) - return buildM4AAtom("meta", body) -} - -// ensureM4AIlstInBuf returns a buffer guaranteed to contain the -// moov>[udta>]meta>ilst chain, creating the missing tail at the end of the -// deepest existing ancestor. Chunk offsets are shifted for the inserted bytes; -// base is the buffer's absolute file offset (0 for a whole-file buffer) so the -// shift compares against the absolute positions stco/co64 entries hold. -func ensureM4AIlstInBuf(data []byte, base int64) ([]byte, m4aIlstLocation, error) { - if loc, ok := locateM4AIlstInBuf(data); ok { - return data, loc, nil - } - - n := int64(len(data)) - moov, ok := findChildMP4(data, 0, n, "moov") - if !ok { - return nil, m4aIlstLocation{}, fmt.Errorf("moov not found") - } - - var insertPos int64 - var insert []byte - var grow []mp4Box - - if udta, ok := findChildMP4(data, moov.body(), moov.end(), "udta"); ok { - if meta, ok := findChildMP4(data, udta.body(), udta.end(), "meta"); ok { - insertPos = meta.end() - insert = buildM4AAtom("ilst", nil) - grow = []mp4Box{moov, udta, meta} - } else { - insertPos = udta.end() - insert = buildM4AMetaBox() - grow = []mp4Box{moov, udta} - } - } else { - insertPos = moov.end() - insert = buildM4AAtom("udta", buildM4AMetaBox()) - grow = []mp4Box{moov} - } - - updated := make([]byte, 0, len(data)+len(insert)) - updated = append(updated, data[:insertPos]...) - updated = append(updated, insert...) - updated = append(updated, data[insertPos:]...) - - delta := int64(len(insert)) - for _, b := range grow { - growBoxSize(updated, b, delta) - } - if newMoov, ok := findChildMP4(updated, 0, int64(len(updated)), "moov"); ok { - shiftChunkOffsets(updated, newMoov, base+insertPos, delta) - } - - loc, ok := locateM4AIlstInBuf(updated) - if !ok { - return nil, m4aIlstLocation{}, fmt.Errorf("failed to create ilst") - } - return updated, loc, nil -} - -// m4aFreeformNameInBuf extracts the freeform ("----") atom's name from a -// buffer-backed child box. -func m4aFreeformNameInBuf(data []byte, box mp4Box) string { - pos := box.body() - for pos+8 <= box.end() { - child, ok := readMP4Box(data, pos) - if !ok { - return "" - } - if child.typ == "name" && child.size > child.hdr+4 { - raw := data[child.body()+4 : child.end()] - return strings.TrimSpace(strings.TrimRight(string(raw), "\x00")) - } - pos = child.end() - } - return "" -} - -// m4aIndexPairInBuf reads the (number, total) pair from a trkn/disk box. -func m4aIndexPairInBuf(data []byte, box mp4Box) (int, int) { - pos := box.body() - for pos+8 <= box.end() { - child, ok := readMP4Box(data, pos) - if !ok { - return 0, 0 - } - if child.typ == "data" && child.size >= child.hdr+8+6 { - payload := data[child.body()+8 : child.end()] - return int(binary.BigEndian.Uint16(payload[2:4])), int(binary.BigEndian.Uint16(payload[4:6])) - } - pos = child.end() - } - return 0, 0 -} - -// isTruthyTagValue reports whether a fields-map flag value means set/true. -func isTruthyTagValue(v string) bool { - switch strings.ToLower(strings.TrimSpace(v)) { - case "1", "true", "yes", "explicit": - return true - } - return false -} - -// EditM4AFields updates only the ilst entries whose keys are explicitly -// present in the fields map (set-or-clear semantics, mirroring EditFlacFields) -// while preserving every other atom. Standard atoms, freeform ISRC/LABEL, and -// ReplayGain freeform tags are all written in a single file rewrite. Only the -// moov box is held in memory; the audio bulk is streamed. -func EditM4AFields(filePath string, fields map[string]string) error { - f, err := os.Open(filePath) - if err != nil { - return err - } - info, err := f.Stat() - if err != nil { - f.Close() - return err - } - moovBuf, moovOffset, found, err := loadTopLevelMP4Box(f, info.Size(), "moov") - f.Close() - if err != nil { - return err - } - if !found { - return fmt.Errorf("moov not found") - } - moovLen := int64(len(moovBuf)) - - data, loc, err := ensureM4AIlstInBuf(moovBuf, moovOffset) - if err != nil { - return err - } - - dropStandard := map[string]bool{} - var appended []byte - - for fieldKey, atomType := range m4aTextAtoms { - v, ok := fields[fieldKey] - if !ok { - continue - } - dropStandard[atomType] = true - if fieldKey == "genre" { - dropStandard["gnre"] = true // legacy numeric genre - } - if strings.TrimSpace(v) != "" { - appended = append(appended, buildM4ATextAtom(atomType, v)...) - } - } - - // Freeform tags: ISRC, LABEL, ReplayGain (+iTunNORM). - removeFreeform := map[string]struct{}{} - var freeformTags []m4aFreeformTag - if _, ok := fields["isrc"]; ok { - removeFreeform["ISRC"] = struct{}{} - freeformTags = append(freeformTags, m4aFreeformTag{name: "ISRC", value: strings.TrimSpace(fields["isrc"])}) - } - if _, ok := fields["label"]; ok { - removeFreeform["LABEL"] = struct{}{} - removeFreeform["ORGANIZATION"] = struct{}{} - freeformTags = append(freeformTags, m4aFreeformTag{name: "LABEL", value: strings.TrimSpace(fields["label"])}) - } - if _, ok := fields["lyrics"]; ok { - // The canonical iTunes lyrics atom is written above. Remove custom - // aliases so an older synced value cannot survive a replace/clear edit. - removeFreeform["LYRICS"] = struct{}{} - removeFreeform["UNSYNCEDLYRICS"] = struct{}{} - removeFreeform["SYNCEDLYRICS"] = struct{}{} - } - if v, ok := fields["album_type"]; ok { - removeFreeform["RELEASETYPE"] = struct{}{} - freeformTags = append(freeformTags, m4aFreeformTag{name: "RELEASETYPE", value: strings.TrimSpace(v)}) - } - if v, ok := fields["upc"]; ok { - removeFreeform["BARCODE"] = struct{}{} - freeformTags = append(freeformTags, m4aFreeformTag{name: "BARCODE", value: strings.TrimSpace(v)}) - } - // Content advisory (rtng) and compilation (cpil) are integer/boolean - // atoms rather than text. - if v, ok := fields["explicit"]; ok { - dropStandard["rtng"] = true - if isTruthyTagValue(v) { - appended = append(appended, buildM4AInt8Atom("rtng", 1)...) - } - } - if v, ok := fields["compilation"]; ok { - dropStandard["cpil"] = true - if isTruthyTagValue(v) { - appended = append(appended, buildM4AFlagAtom("cpil", true)...) - } - } - replayGain := collectM4AReplayGainFields(fields) - if len(replayGain) > 0 { - for _, key := range []string{"replaygain_track_gain", "replaygain_track_peak", "replaygain_album_gain", "replaygain_album_peak"} { - removeFreeform[strings.ToUpper(key)] = struct{}{} - if value := replayGain[key]; value != "" { - freeformTags = append(freeformTags, m4aFreeformTag{name: key, value: value}) - } - } - removeFreeform["ITUNNORM"] = struct{}{} - if norm := replayGain["iTunNORM"]; norm != "" { - freeformTags = append(freeformTags, m4aFreeformTag{name: "iTunNORM", value: norm}) - } - } - for _, tag := range freeformTags { - if tag.value != "" { - appended = append(appended, buildM4AFreeformAtom(tag.name, tag.value)...) - } - } - - editTrack := hasMapKey(fields, "track_number") || hasMapKey(fields, "track_total") - editDisc := hasMapKey(fields, "disc_number") || hasMapKey(fields, "disc_total") - - coverPath := strings.TrimSpace(fields["cover_path"]) - var coverData []byte - if coverPath != "" { - if b, err := os.ReadFile(coverPath); err == nil && len(b) > 0 { - coverData = b - dropStandard["covr"] = true - } - } - - // Rebuild the ilst body, keeping untouched children verbatim. - var newBody []byte - curTrack, curTrackTotal := 0, 0 - curDisc, curDiscTotal := 0, 0 - for pos := loc.ilst.body(); pos+8 <= loc.ilst.end(); { - child, ok := readMP4Box(data, pos) - if !ok { - return fmt.Errorf("malformed ilst child at %d", pos) - } - keep := true - switch { - case dropStandard[child.typ]: - keep = false - case child.typ == "trkn": - curTrack, curTrackTotal = m4aIndexPairInBuf(data, child) - keep = !editTrack - case child.typ == "disk": - curDisc, curDiscTotal = m4aIndexPairInBuf(data, child) - keep = !editDisc - case child.typ == "----": - name := strings.ToUpper(m4aFreeformNameInBuf(data, child)) - if _, remove := removeFreeform[name]; remove { - keep = false - } - } - if keep { - newBody = append(newBody, data[child.offset:child.end()]...) - } - pos = child.end() - } - - if editTrack { - if v, ok := fields["track_number"]; ok { - curTrack = parsePositiveInt(v) - } - if v, ok := fields["track_total"]; ok { - curTrackTotal = parsePositiveInt(v) - } - if curTrack > 0 { - appended = append(appended, buildM4AIndexAtom("trkn", curTrack, curTrackTotal)...) - } - } - if editDisc { - if v, ok := fields["disc_number"]; ok { - curDisc = parsePositiveInt(v) - } - if v, ok := fields["disc_total"]; ok { - curDiscTotal = parsePositiveInt(v) - } - if curDisc > 0 { - appended = append(appended, buildM4AIndexAtom("disk", curDisc, curDiscTotal)...) - } - } - if len(coverData) > 0 { - appended = append(appended, buildM4ACoverAtom(coverData)...) - } - newBody = append(newBody, appended...) - - newIlst := buildM4AAtom("ilst", newBody) - delta := int64(len(newIlst)) - loc.ilst.size - - updated := make([]byte, 0, int64(len(data))+delta) - updated = append(updated, data[:loc.ilst.offset]...) - updated = append(updated, newIlst...) - updated = append(updated, data[loc.ilst.end():]...) - - for _, b := range loc.ancestors { - growBoxSize(updated, b, delta) - } - if moov, ok := findChildMP4(updated, 0, int64(len(updated)), "moov"); ok { - shiftChunkOffsets(updated, moov, moovOffset+loc.ilst.offset, delta) - } - - return replaceFileSectionsStreaming(filePath, []fileSection{ - {start: moovOffset, end: moovOffset + moovLen, data: updated}, - }) -} diff --git a/go_backend/m4a_freeform_write_test.go b/go_backend/m4a_freeform_write_test.go deleted file mode 100644 index 563f353a..00000000 --- a/go_backend/m4a_freeform_write_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package gobackend - -import ( - "os" - "path/filepath" - "testing" -) - -func TestEditM4AFreeformTextWritesISRCAndLabel(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "track.m4a") - - ilst := buildM4ATextTag("\xa9nam", "Title") - if err := os.WriteFile(path, buildM4AFileWithIlst(ilst, true), 0600); err != nil { - t.Fatal(err) - } - - if err := EditM4AFreeformText(path, map[string]string{ - "isrc": "USRC17607839", - "label": "Some Label", - }); err != nil { - t.Fatalf("EditM4AFreeformText: %v", err) - } - - meta, err := ReadM4ATags(path) - if err != nil { - t.Fatalf("ReadM4ATags: %v", err) - } - if meta.ISRC != "USRC17607839" { - t.Fatalf("ISRC = %q, want USRC17607839", meta.ISRC) - } - if meta.Label != "Some Label" { - t.Fatalf("Label = %q, want Some Label", meta.Label) - } - if meta.Title != "Title" { - t.Fatalf("Title = %q, want Title (existing tag must survive)", meta.Title) - } -} - -func TestEditM4AFreeformTextReplacesExisting(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "track.m4a") - - ilst := buildM4ATextTag("\xa9nam", "Title") - ilst = append(ilst, buildM4AFreeformAtom("ISRC", "OLDISRC00001")...) - ilst = append(ilst, buildM4AFreeformAtom("LABEL", "Old Label")...) - if err := os.WriteFile(path, buildM4AFileWithIlst(ilst, true), 0600); err != nil { - t.Fatal(err) - } - - if err := EditM4AFreeformText(path, map[string]string{ - "isrc": "NEWISRC00002", - "label": "", - }); err != nil { - t.Fatalf("EditM4AFreeformText: %v", err) - } - - meta, err := ReadM4ATags(path) - if err != nil { - t.Fatalf("ReadM4ATags: %v", err) - } - if meta.ISRC != "NEWISRC00002" { - t.Fatalf("ISRC = %q, want NEWISRC00002", meta.ISRC) - } - if meta.Label != "" { - t.Fatalf("Label = %q, want empty (cleared)", meta.Label) - } -} diff --git a/go_backend/m4a_stream.go b/go_backend/m4a_stream.go deleted file mode 100644 index 4c4928fe..00000000 --- a/go_backend/m4a_stream.go +++ /dev/null @@ -1,120 +0,0 @@ -package gobackend - -import ( - "fmt" - "io" - "os" - "path/filepath" - "sort" -) - -// Streaming support for MP4 tag rewrites: only the boxes being edited (moov, -// ftyp) are held in memory while the mdat bulk is streamed between file -// handles, so peak memory tracks the moov size instead of the file size. - -// loadTopLevelMP4Box scans f's top-level boxes and loads only the first box of -// the given type into memory, returning its bytes and absolute file offset. -// ok is false when the box does not exist; err reports read/parse failures. -func loadTopLevelMP4Box(f *os.File, fileSize int64, typ string) (buf []byte, offset int64, ok bool, err error) { - for pos := int64(0); pos+8 <= fileSize; { - header, err := readAtomHeaderAt(f, pos, fileSize) - if err != nil { - return nil, 0, false, err - } - size := header.size - if size == 0 { - size = fileSize - pos - } - if size < header.headerSize || pos+size > fileSize { - return nil, 0, false, fmt.Errorf("invalid atom size for %s", header.typ) - } - if header.typ == typ { - buf := make([]byte, size) - if _, err := f.ReadAt(buf, pos); err != nil { - return nil, 0, false, err - } - return buf, pos, true, nil - } - pos += size - } - return nil, 0, false, nil -} - -// fileSection is one byte range [start,end) to substitute during a streaming -// rewrite. -type fileSection struct { - start, end int64 - data []byte -} - -// replaceFileSectionsStreaming rewrites filePath with each section replaced by -// its data, streaming every byte outside the sections, and publishes via -// temp+fsync+rename so an interruption never leaves a truncated file under the -// final name. Sections must not overlap; they are sorted internally. -func replaceFileSectionsStreaming(filePath string, sections []fileSection) error { - sorted := append([]fileSection{}, sections...) - sort.Slice(sorted, func(i, j int) bool { return sorted[i].start < sorted[j].start }) - for i := 1; i < len(sorted); i++ { - if sorted[i].start < sorted[i-1].end { - return fmt.Errorf("overlapping file sections") - } - } - - src, err := os.Open(filePath) - if err != nil { - return err - } - - tmpPath := filePath + ".tag.partial" - os.Remove(tmpPath) - tmp, err := os.Create(tmpPath) - if err != nil { - src.Close() - return err - } - fail := func(err error) error { - tmp.Close() - src.Close() - os.Remove(tmpPath) - return err - } - - pos := int64(0) - for _, sec := range sorted { - if sec.start < pos { - return fail(fmt.Errorf("file section out of order")) - } - if _, err := io.CopyN(tmp, src, sec.start-pos); err != nil { - return fail(err) - } - if _, err := tmp.Write(sec.data); err != nil { - return fail(err) - } - if _, err := src.Seek(sec.end, io.SeekStart); err != nil { - return fail(err) - } - pos = sec.end - } - if _, err := io.Copy(tmp, src); err != nil { - return fail(err) - } - if err := tmp.Sync(); err != nil { - return fail(err) - } - if err := tmp.Close(); err != nil { - src.Close() - os.Remove(tmpPath) - return err - } - // Release the read handle before the rename (required on Windows). - if err := src.Close(); err != nil { - os.Remove(tmpPath) - return err - } - if err := os.Rename(tmpPath, filePath); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("failed to publish tagged file: %w", err) - } - syncDir(filepath.Dir(filePath)) - return nil -} diff --git a/go_backend/memory_release_test.go b/go_backend/memory_release_test.go deleted file mode 100644 index 32132d2c..00000000 --- a/go_backend/memory_release_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package gobackend - -import ( - "runtime" - "testing" - "time" - "weak" -) - -func TestDiscardedPooledRuntimeCanBeCollected(t *testing.T) { - ext := newTestLoadedExtension(t, ExtensionTypeDownloadProvider) - discarded := func() weak.Pointer[extensionRuntime] { - vm, rt, err := acquireIsolatedExtensionRuntime(ext) - if err != nil { - t.Fatal(err) - } - if err := vm.Set("temporaryDownloadBytes", vm.NewArrayBuffer(make([]byte, 16<<20))); err != nil { - t.Fatal(err) - } - releaseIsolatedExtensionRuntime(ext, vm, rt, true, true, nil) - vm, rt, err = acquireIsolatedExtensionRuntime(ext) - if err != nil { - t.Fatal(err) - } - pointer := weak.Make(rt) - // An operation error retires the borrowed VM instead of pooling it. - releaseIsolatedExtensionRuntime(ext, vm, rt, false, true, nil) - return pointer - }() - runtime.GC() - if discarded.Value() != nil { - t.Fatal("retired runtime and its download buffer remain reachable from the idle pool") - } - runtime.KeepAlive(ext) -} - -func TestReleaseMemoryUnderPressureClearsDisposableCaches(t *testing.T) { - clearCoverMemoryCache() - coverCachePut("https://example.com/cover.jpg", []byte("cover")) - globalLyricsCache.ClearAll() - globalLyricsCache.Set("artist", "track", 120, &LyricsResponse{PlainLyrics: "lyrics"}) - - privateIPCacheMu.Lock() - privateIPCache["example.com"] = privateIPCacheEntry{expiresAt: time.Now().Add(time.Hour)} - privateIPCacheMu.Unlock() - extensionHealthCacheMu.Lock() - extensionHealthCache["extension"] = cachedExtensionHealthResult{expiresAt: time.Now().Add(time.Hour)} - extensionHealthCacheMu.Unlock() - - ReleaseMemoryUnderPressure() - - coverMu.Lock() - coverEntries, coverBytes := len(coverCache), coverCacheBytes - coverMu.Unlock() - if coverEntries != 0 || coverBytes != 0 { - t.Fatalf("cover cache retained %d entries/%d bytes", coverEntries, coverBytes) - } - if globalLyricsCache.Size() != 0 { - t.Fatalf("lyrics cache retained %d entries", globalLyricsCache.Size()) - } - privateIPCacheMu.RLock() - privateEntries := len(privateIPCache) - privateIPCacheMu.RUnlock() - if privateEntries != 0 { - t.Fatalf("private IP cache retained %d entries", privateEntries) - } - extensionHealthCacheMu.Lock() - healthEntries := len(extensionHealthCache) - extensionHealthCacheMu.Unlock() - if healthEntries != 0 { - t.Fatalf("health cache retained %d entries", healthEntries) - } -} diff --git a/go_backend/metadata.go b/go_backend/metadata.go deleted file mode 100644 index c143ee65..00000000 --- a/go_backend/metadata.go +++ /dev/null @@ -1,1058 +0,0 @@ -package gobackend - -import ( - "bytes" - "fmt" - stdimage "image" - _ "image/gif" - "image/jpeg" - _ "image/png" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/go-flac/flacpicture/v2" - "github.com/go-flac/flacvorbis/v2" - "github.com/go-flac/go-flac/v2" -) - -const artistTagModeSplitVorbis = "split_vorbis" - -var artistTagSplitPattern = regexp.MustCompile(`\s*(?:,|&|\bx\b)\s*|\s+\b(?:feat(?:uring)?|ft|with)\.?\s*`) - -func detectCoverMIME(coverPath string, coverData []byte) string { - // Prefer magic-byte detection over file extension. - // Some providers return non-JPEG data behind .jpg URLs. - if len(coverData) >= 8 && - coverData[0] == 0x89 && - coverData[1] == 0x50 && - coverData[2] == 0x4E && - coverData[3] == 0x47 && - coverData[4] == 0x0D && - coverData[5] == 0x0A && - coverData[6] == 0x1A && - coverData[7] == 0x0A { - return "image/png" - } - if len(coverData) >= 3 && - coverData[0] == 0xFF && - coverData[1] == 0xD8 && - coverData[2] == 0xFF { - return "image/jpeg" - } - if len(coverData) >= 6 { - header := string(coverData[:6]) - if header == "GIF87a" || header == "GIF89a" { - return "image/gif" - } - } - if len(coverData) >= 12 && - string(coverData[:4]) == "RIFF" && - string(coverData[8:12]) == "WEBP" { - return "image/webp" - } - - switch strings.ToLower(filepath.Ext(strings.TrimSpace(coverPath))) { - case ".png": - return "image/png" - case ".jpg", ".jpeg": - return "image/jpeg" - case ".webp": - return "image/webp" - case ".gif": - return "image/gif" - } - - return "image/jpeg" -} - -// maxFlacPictureBytes keeps cover art below the 24-bit length field of a FLAC -// metadata block; go-flac silently truncates oversized blocks into a corrupt file. -const maxFlacPictureBytes = 16 * 1000 * 1000 - -// fitCoverForFlac returns cover bytes that fit inside a FLAC PICTURE block, -// re-encoding and downscaling when needed. Returns false if the data cannot be -// decoded as an image. -func fitCoverForFlac(coverData []byte) ([]byte, bool) { - if len(coverData) <= maxFlacPictureBytes { - return coverData, true - } - - img, _, err := stdimage.Decode(bytes.NewReader(coverData)) - if err != nil { - return nil, false - } - - for _, quality := range []int{90, 80, 70, 60} { - if encoded, ok := encodeJPEGUnder(img, quality, maxFlacPictureBytes); ok { - return encoded, true - } - } - - for _, maxDim := range []int{1500, 1200, 1000, 800} { - scaled := downscaleImage(img, maxDim) - if encoded, ok := encodeJPEGUnder(scaled, 85, maxFlacPictureBytes); ok { - return encoded, true - } - } - - return nil, false -} - -func encodeJPEGUnder(img stdimage.Image, quality, limit int) ([]byte, bool) { - var buf bytes.Buffer - if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: quality}); err != nil { - return nil, false - } - if buf.Len() > limit { - return nil, false - } - return buf.Bytes(), true -} - -func downscaleImage(img stdimage.Image, maxDim int) stdimage.Image { - bounds := img.Bounds() - width, height := bounds.Dx(), bounds.Dy() - if width <= maxDim && height <= maxDim { - return img - } - - scale := float64(maxDim) / float64(max(width, height)) - newWidth := max(1, int(float64(width)*scale)) - newHeight := max(1, int(float64(height)*scale)) - - dst := stdimage.NewRGBA(stdimage.Rect(0, 0, newWidth, newHeight)) - for y := 0; y < newHeight; y++ { - srcY := bounds.Min.Y + int(float64(y)/scale) - for x := 0; x < newWidth; x++ { - srcX := bounds.Min.X + int(float64(x)/scale) - dst.Set(x, y, img.At(srcX, srcY)) - } - } - return dst -} - -func buildPictureBlock(coverPath string, coverData []byte) (flac.MetaDataBlock, error) { - if len(coverData) == 0 { - return flac.MetaDataBlock{}, fmt.Errorf("empty cover data") - } - - fitted, ok := fitCoverForFlac(coverData) - if !ok { - return flac.MetaDataBlock{}, fmt.Errorf("cover too large for FLAC picture block and could not be resized") - } - coverData = fitted - - mime := detectCoverMIME(coverPath, coverData) - picture := &flacpicture.MetadataBlockPicture{ - PictureType: flacpicture.PictureTypeFrontCover, - MIME: mime, - Description: "Front Cover", - ImageData: coverData, - } - - // Width/height/depth are optional in practice; keep zero when decode fails. - if cfg, format, err := stdimage.DecodeConfig(bytes.NewReader(coverData)); err == nil { - picture.Width = uint32(cfg.Width) - picture.Height = uint32(cfg.Height) - switch format { - case "png": - picture.ColorDepth = 32 - case "jpeg": - picture.ColorDepth = 24 - default: - picture.ColorDepth = 0 - } - } - - return picture.Marshal(), nil -} - -type Metadata struct { - Title string - Artist string - Album string - AlbumArtist string - ArtistTagMode string - Date string - TrackNumber int - TotalTracks int - DiscNumber int - TotalDiscs int - ISRC string - Description string - Lyrics string - Genre string - Label string - Copyright string - Composer string - Comment string - Explicit bool - AlbumType string - UPC string - - // ReplayGain fields (stored as Vorbis Comments in FLAC) - ReplayGainTrackGain string // e.g. "-6.50 dB" - ReplayGainTrackPeak string // e.g. "0.988831" - ReplayGainAlbumGain string // e.g. "-7.20 dB" - ReplayGainAlbumPeak string // e.g. "1.000000" -} - -// parseFlacFile wraps flac.ParseFile but closes the file handle when parsing -// fails. flac.ParseFile leaks the *os.File on parse errors (no reference is -// returned to close it), which on Windows keeps the file locked until GC. -// Callers must Close() the returned file when done reading; File.Save also -// closes the underlying handle, and a second Close afterwards is harmless. -func parseFlacFile(filePath string) (*flac.File, error) { - handle, err := os.Open(filePath) - if err != nil { - return nil, err - } - f, err := flac.ParseBytes(flac.NewBufIOWithInner(handle)) - if err != nil { - handle.Close() - return nil, err - } - return f, nil -} - -// updateFlacVorbis parses a FLAC file, hands the parsed file and its Vorbis -// comment block (created if absent) to mutate, then marshals the block back -// into the file and saves it. Shared scaffold for all FLAC tag writers. -func updateFlacVorbis(filePath string, mutate func(f *flac.File, cmt *flacvorbis.MetaDataBlockVorbisComment) error) error { - f, err := parseFlacFile(filePath) - if err != nil { - return fmt.Errorf("failed to parse FLAC file: %w", err) - } - defer f.Close() - - var cmt *flacvorbis.MetaDataBlockVorbisComment - for _, meta := range f.Meta { - if meta.Type == flac.VorbisComment { - cmt, err = flacvorbis.ParseFromMetaDataBlock(*meta) - if err != nil { - return fmt.Errorf("failed to parse vorbis comment: %w", err) - } - break - } - } - if cmt == nil { - cmt = flacvorbis.New() - } - - if err := mutate(f, cmt); err != nil { - return err - } - - // Re-scan for the block index: mutate may have removed blocks. - cmtBlock := cmt.Marshal() - replaced := false - for idx, meta := range f.Meta { - if meta.Type == flac.VorbisComment { - f.Meta[idx] = &cmtBlock - replaced = true - break - } - } - if !replaced { - f.Meta = append(f.Meta, &cmtBlock) - } - - return saveFlacFile(f, filePath) -} - -// replaceFlacPictures strips all Picture blocks and appends a new front cover -// built from coverData. On error the pictures stay removed and no cover is added. -func replaceFlacPictures(f *flac.File, coverPath string, coverData []byte) error { - for i := len(f.Meta) - 1; i >= 0; i-- { - if f.Meta[i].Type == flac.Picture { - f.Meta = append(f.Meta[:i], f.Meta[i+1:]...) - } - } - - picBlock, err := buildPictureBlock(coverPath, coverData) - if err != nil { - return err - } - f.Meta = append(f.Meta, &picBlock) - return nil -} - -func EmbedMetadata(filePath string, metadata Metadata, coverPath string) error { - return updateFlacVorbis(filePath, func(f *flac.File, cmt *flacvorbis.MetaDataBlockVorbisComment) error { - writeVorbisMetadata(cmt, metadata) - - if coverPath != "" { - if fileExists(coverPath) { - coverData, err := os.ReadFile(coverPath) - if err != nil { - LogWarn("Metadata", "Failed to read cover file: %v", err) - } else if err := replaceFlacPictures(f, coverPath, coverData); err != nil { - LogWarn("Metadata", "Skipping cover art: %v", err) - } - } else { - LogWarn("Metadata", "Cover file does not exist") - } - } - return nil - }) -} - -func EmbedMetadataWithCoverData(filePath string, metadata Metadata, coverData []byte) error { - return updateFlacVorbis(filePath, func(f *flac.File, cmt *flacvorbis.MetaDataBlockVorbisComment) error { - writeVorbisMetadata(cmt, metadata) - - if len(coverData) > 0 { - if err := replaceFlacPictures(f, "", coverData); err != nil { - LogWarn("Metadata", "Skipping cover art: %v", err) - } - } - return nil - }) -} - -func ReadMetadata(filePath string) (*Metadata, error) { - f, err := parseFlacFile(filePath) - if err != nil { - return nil, fmt.Errorf("failed to parse FLAC file: %w", err) - } - defer f.Close() - return metadataFromParsedFlac(f), nil -} - -func metadataFromParsedFlac(f *flac.File) *Metadata { - metadata := &Metadata{} - - for _, meta := range f.Meta { - if meta.Type == flac.VorbisComment { - cmt, err := flacvorbis.ParseFromMetaDataBlock(*meta) - if err != nil { - continue - } - - metadata.Title = getComment(cmt, "TITLE") - metadata.Artist = getJoinedComment(cmt, "ARTIST") - metadata.Album = getComment(cmt, "ALBUM") - metadata.AlbumArtist = getJoinedComment(cmt, "ALBUMARTIST") - if metadata.AlbumArtist == "" { - metadata.AlbumArtist = getJoinedComment(cmt, "ALBUM ARTIST") - } - if metadata.AlbumArtist == "" { - metadata.AlbumArtist = getJoinedComment(cmt, "ALBUM_ARTIST") - } - metadata.Date = getComment(cmt, "DATE") - metadata.ISRC = getComment(cmt, "ISRC") - metadata.Description = getComment(cmt, "DESCRIPTION") - - metadata.Lyrics = getLyricsComment(cmt) - - trackNum := getComment(cmt, "TRACKNUMBER") - if trackNum != "" { - metadata.TrackNumber, metadata.TotalTracks = parseIndexPair(trackNum) - } - if metadata.TrackNumber == 0 { - trackNum = getComment(cmt, "TRACK") - if trackNum != "" { - metadata.TrackNumber, metadata.TotalTracks = parseIndexPair(trackNum) - } - } - - discNum := getComment(cmt, "DISCNUMBER") - if discNum != "" { - metadata.DiscNumber, metadata.TotalDiscs = parseIndexPair(discNum) - } - if metadata.DiscNumber == 0 { - discNum = getComment(cmt, "DISC") - if discNum != "" { - metadata.DiscNumber, metadata.TotalDiscs = parseIndexPair(discNum) - } - } - - if metadata.Date == "" { - metadata.Date = getComment(cmt, "YEAR") - } - - metadata.Genre = getComment(cmt, "GENRE") - metadata.Label = getComment(cmt, "ORGANIZATION") - if metadata.Label == "" { - metadata.Label = getComment(cmt, "LABEL") - } - if metadata.Label == "" { - metadata.Label = getComment(cmt, "PUBLISHER") - } - metadata.Copyright = getComment(cmt, "COPYRIGHT") - metadata.Composer = getComment(cmt, "COMPOSER") - metadata.Comment = getComment(cmt, "COMMENT") - metadata.Explicit = isTruthyTagValue(getComment(cmt, "ITUNESADVISORY")) - metadata.AlbumType = getComment(cmt, "RELEASETYPE") - if metadata.AlbumType == "" && isTruthyTagValue(getComment(cmt, "COMPILATION")) { - metadata.AlbumType = "compilation" - } - metadata.UPC = getComment(cmt, "BARCODE") - if metadata.UPC == "" { - metadata.UPC = getComment(cmt, "UPC") - } - - metadata.ReplayGainTrackGain = getComment(cmt, "REPLAYGAIN_TRACK_GAIN") - metadata.ReplayGainTrackPeak = getComment(cmt, "REPLAYGAIN_TRACK_PEAK") - metadata.ReplayGainAlbumGain = getComment(cmt, "REPLAYGAIN_ALBUM_GAIN") - metadata.ReplayGainAlbumPeak = getComment(cmt, "REPLAYGAIN_ALBUM_PEAK") - - break - } - } - - return metadata -} - -// EditFlacFields opens a FLAC file and updates only the Vorbis Comment keys -// that are explicitly present in the fields map. Keys present with a non-empty -// value are set; keys present with an empty value are removed (cleared). Keys -// absent from the map are left untouched. This is the correct function for -// partial edits (e.g. writing only ReplayGain tags) and full editor saves alike. -func EditFlacFields(filePath string, fields map[string]string) error { - return updateFlacVorbis(filePath, func(f *flac.File, cmt *flacvorbis.MetaDataBlockVorbisComment) error { - applyVorbisFieldEdits(cmt, fields) - - coverPath := strings.TrimSpace(fields["cover_path"]) - if coverPath != "" && fileExists(coverPath) { - if coverData, err := os.ReadFile(coverPath); err == nil && len(coverData) > 0 { - _ = replaceFlacPictures(f, "", coverData) - } - } - return nil - }) -} - -// applyVorbisFieldEdits applies the editor's set-or-clear field semantics to a -// Vorbis comment block. Shared by the FLAC and Ogg/Opus editors so both -// formats interpret the fields map identically. -func applyVorbisFieldEdits(cmt *flacvorbis.MetaDataBlockVorbisComment, fields map[string]string) { - artistMode := fields["artist_tag_mode"] - - // Mapping from fields-map key → one or more Vorbis Comment keys. - // Each entry is handled with set-or-clear semantics. - simpleKeys := map[string]string{ - "title": "TITLE", - "album": "ALBUM", - "date": "DATE", - "isrc": "ISRC", - "genre": "GENRE", - "label": "ORGANIZATION", - "copyright": "COPYRIGHT", - "composer": "COMPOSER", - "comment": "COMMENT", - "explicit": "ITUNESADVISORY", - "album_type": "RELEASETYPE", - "upc": "BARCODE", - "barcode": "BARCODE", - "compilation": "COMPILATION", - "replaygain_track_gain": "REPLAYGAIN_TRACK_GAIN", - "replaygain_track_peak": "REPLAYGAIN_TRACK_PEAK", - "replaygain_album_gain": "REPLAYGAIN_ALBUM_GAIN", - "replaygain_album_peak": "REPLAYGAIN_ALBUM_PEAK", - } - - for fieldKey, vorbisKey := range simpleKeys { - if v, ok := fields[fieldKey]; ok { - setOrClearComment(cmt, vorbisKey, v) - } - } - - // Remove known aliases for fields that were just written/cleared, so that - // tags from other taggers (e.g. LABEL, PUBLISHER, ALBUM ARTIST) don't - // conflict with the canonical keys we use. - aliasCleanup := map[string][]string{ - "label": {"LABEL", "PUBLISHER"}, // canonical: ORGANIZATION - "date": {"YEAR"}, // canonical: DATE - "genre": {}, // no common aliases - "copyright": {}, - } - for fieldKey, aliases := range aliasCleanup { - if _, ok := fields[fieldKey]; ok { - for _, alias := range aliases { - removeCommentKey(cmt, alias) - } - } - } - - // Artist fields: use split-artist logic when mode is set. - if v, ok := fields["artist"]; ok { - setOrClearArtistComments(cmt, "ARTIST", v, artistMode) - } - if v, ok := fields["album_artist"]; ok { - setOrClearArtistComments(cmt, "ALBUMARTIST", v, artistMode) - // Remove aliases from other taggers. - removeCommentKey(cmt, "ALBUM ARTIST") - removeCommentKey(cmt, "ALBUM_ARTIST") - } - - // Track/disc numbers: present + empty → clear; when only totals are edited, - // preserve the current index number and rewrite the combined value. - if _, ok := fields["track_number"]; ok || fields["track_total"] != "" || hasMapKey(fields, "track_total") { - currentTrackNum, currentTotalTracks := parseIndexPair(getComment(cmt, "TRACKNUMBER")) - if currentTrackNum == 0 && currentTotalTracks == 0 { - currentTrackNum, currentTotalTracks = parseIndexPair(getComment(cmt, "TRACK")) - } - if v, ok := fields["track_number"]; ok { - currentTrackNum = parsePositiveInt(v) - } - if v, ok := fields["track_total"]; ok { - currentTotalTracks = parsePositiveInt(v) - } - if currentTrackNum > 0 { - setOrClearComment(cmt, "TRACKNUMBER", formatIndexValue(currentTrackNum, currentTotalTracks)) - } else { - removeCommentKey(cmt, "TRACKNUMBER") - } - removeCommentKey(cmt, "TRACK") // alias - } - if _, ok := fields["disc_number"]; ok || fields["disc_total"] != "" || hasMapKey(fields, "disc_total") { - currentDiscNum, currentTotalDiscs := parseIndexPair(getComment(cmt, "DISCNUMBER")) - if currentDiscNum == 0 && currentTotalDiscs == 0 { - currentDiscNum, currentTotalDiscs = parseIndexPair(getComment(cmt, "DISC")) - } - if v, ok := fields["disc_number"]; ok { - currentDiscNum = parsePositiveInt(v) - } - if v, ok := fields["disc_total"]; ok { - currentTotalDiscs = parsePositiveInt(v) - } - if currentDiscNum > 0 { - setOrClearComment(cmt, "DISCNUMBER", formatIndexValue(currentDiscNum, currentTotalDiscs)) - } else { - removeCommentKey(cmt, "DISCNUMBER") - } - removeCommentKey(cmt, "DISC") // alias - } - - // Lyrics: set the broadly-supported plain aliases and remove any stale - // SYNCEDLYRICS value, or clear every alias. - if v, ok := fields["lyrics"]; ok { - removeCommentKey(cmt, "SYNCEDLYRICS") - if v != "" { - setOrClearComment(cmt, "LYRICS", v) - setOrClearComment(cmt, "UNSYNCEDLYRICS", v) - } else { - removeCommentKey(cmt, "LYRICS") - removeCommentKey(cmt, "UNSYNCEDLYRICS") - } - } -} - -// writeVorbisMetadata writes all metadata fields to a Vorbis Comment block. -// Empty/zero values are simply skipped (not written, not cleared). This is -// used by the download embedding path where absent fields should preserve any -// existing values. The editor path uses EditFlacFields() instead. -func writeVorbisMetadata(cmt *flacvorbis.MetaDataBlockVorbisComment, metadata Metadata) { - setComment(cmt, "TITLE", metadata.Title) - setArtistComments(cmt, "ARTIST", metadata.Artist, metadata.ArtistTagMode) - setComment(cmt, "ALBUM", metadata.Album) - setArtistComments(cmt, "ALBUMARTIST", metadata.AlbumArtist, metadata.ArtistTagMode) - setComment(cmt, "DATE", metadata.Date) - - if metadata.TrackNumber > 0 { - setComment(cmt, "TRACKNUMBER", formatIndexValue(metadata.TrackNumber, metadata.TotalTracks)) - } - - if metadata.DiscNumber > 0 { - setComment(cmt, "DISCNUMBER", formatIndexValue(metadata.DiscNumber, metadata.TotalDiscs)) - } - - if metadata.ISRC != "" { - setComment(cmt, "ISRC", metadata.ISRC) - } - - if metadata.Description != "" { - setComment(cmt, "DESCRIPTION", metadata.Description) - } - - if metadata.Lyrics != "" { - setComment(cmt, "LYRICS", metadata.Lyrics) - setComment(cmt, "UNSYNCEDLYRICS", metadata.Lyrics) - } - - if metadata.Genre != "" { - setComment(cmt, "GENRE", metadata.Genre) - } - - if metadata.Label != "" { - setComment(cmt, "ORGANIZATION", metadata.Label) - } - - if metadata.Copyright != "" { - setComment(cmt, "COPYRIGHT", metadata.Copyright) - } - - if metadata.Composer != "" { - setComment(cmt, "COMPOSER", metadata.Composer) - } - - if metadata.Comment != "" { - setComment(cmt, "COMMENT", metadata.Comment) - } - - if metadata.Explicit { - setComment(cmt, "ITUNESADVISORY", "1") - } - - if metadata.AlbumType != "" { - setComment(cmt, "RELEASETYPE", strings.ToLower(metadata.AlbumType)) - if strings.EqualFold(metadata.AlbumType, "compilation") { - setComment(cmt, "COMPILATION", "1") - } - } - - if metadata.UPC != "" { - setComment(cmt, "BARCODE", metadata.UPC) - } - - setComment(cmt, "REPLAYGAIN_TRACK_GAIN", metadata.ReplayGainTrackGain) - setComment(cmt, "REPLAYGAIN_TRACK_PEAK", metadata.ReplayGainTrackPeak) - setComment(cmt, "REPLAYGAIN_ALBUM_GAIN", metadata.ReplayGainAlbumGain) - setComment(cmt, "REPLAYGAIN_ALBUM_PEAK", metadata.ReplayGainAlbumPeak) -} - -func setComment(cmt *flacvorbis.MetaDataBlockVorbisComment, key, value string) { - if value == "" { - return - } - removeCommentKey(cmt, key) - cmt.Comments = append(cmt.Comments, key+"="+value) -} - -// setOrClearComment writes a Vorbis Comment, or removes the key if value is -// empty. Used by the metadata editor path where empty means "delete this tag". -func setOrClearComment(cmt *flacvorbis.MetaDataBlockVorbisComment, key, value string) { - if value == "" { - removeCommentKey(cmt, key) - return - } - removeCommentKey(cmt, key) - cmt.Comments = append(cmt.Comments, key+"="+value) -} - -func setArtistComments(cmt *flacvorbis.MetaDataBlockVorbisComment, key, value, mode string) { - if value == "" { - return - } - values := []string{value} - if shouldSplitVorbisArtistTags(mode) { - values = splitArtistTagValues(value) - } - if len(values) == 0 { - return - } - removeCommentKey(cmt, key) - for _, artist := range values { - if strings.TrimSpace(artist) == "" { - continue - } - cmt.Comments = append(cmt.Comments, key+"="+artist) - } -} - -// setOrClearArtistComments writes artist Vorbis Comments, or removes the key -// if value is empty. Used by the metadata editor path. -func setOrClearArtistComments(cmt *flacvorbis.MetaDataBlockVorbisComment, key, value, mode string) { - if value == "" { - removeCommentKey(cmt, key) - return - } - values := []string{value} - if shouldSplitVorbisArtistTags(mode) { - values = splitArtistTagValues(value) - } - if len(values) == 0 { - removeCommentKey(cmt, key) - return - } - removeCommentKey(cmt, key) - for _, artist := range values { - if strings.TrimSpace(artist) == "" { - continue - } - cmt.Comments = append(cmt.Comments, key+"="+artist) - } -} - -// RewriteSplitArtistTags opens a FLAC file and rewrites the ARTIST and -// ALBUMARTIST Vorbis comments as multiple separate entries (one per artist). -// This is needed because FFmpeg's -metadata flag deduplicates keys, so only -// the last value survives when multiple -metadata ARTIST=X flags are used. -// The native go-flac writer correctly handles multiple Vorbis comments. -func RewriteSplitArtistTags(filePath, artist, albumArtist string) error { - return updateFlacVorbis(filePath, func(_ *flac.File, cmt *flacvorbis.MetaDataBlockVorbisComment) error { - setArtistComments(cmt, "ARTIST", artist, artistTagModeSplitVorbis) - setArtistComments(cmt, "ALBUMARTIST", albumArtist, artistTagModeSplitVorbis) - return nil - }) -} - -func removeCommentKey(cmt *flacvorbis.MetaDataBlockVorbisComment, key string) { - keyUpper := strings.ToUpper(key) - for i := len(cmt.Comments) - 1; i >= 0; i-- { - comment := cmt.Comments[i] - eqIdx := strings.Index(comment, "=") - if eqIdx > 0 { - existingKey := strings.ToUpper(comment[:eqIdx]) - if existingKey == keyUpper { - cmt.Comments = append(cmt.Comments[:i], cmt.Comments[i+1:]...) - } - } - } -} - -func getComment(cmt *flacvorbis.MetaDataBlockVorbisComment, key string) string { - values := getCommentValues(cmt, key) - if len(values) == 0 { - return "" - } - return values[0] -} - -func getLyricsComment(cmt *flacvorbis.MetaDataBlockVorbisComment) string { - for _, key := range []string{"LYRICS", "UNSYNCEDLYRICS", "SYNCEDLYRICS"} { - if lyrics := getComment(cmt, key); strings.TrimSpace(lyrics) != "" { - return lyrics - } - } - return "" -} - -func getJoinedComment(cmt *flacvorbis.MetaDataBlockVorbisComment, key string) string { - return joinVorbisCommentValues(getCommentValues(cmt, key)) -} - -func getCommentValues(cmt *flacvorbis.MetaDataBlockVorbisComment, key string) []string { - keyUpper := strings.ToUpper(key) + "=" - values := make([]string, 0, 1) - for _, comment := range cmt.Comments { - if len(comment) > len(key) { - commentUpper := strings.ToUpper(comment[:len(key)+1]) - if commentUpper == keyUpper { - values = append(values, comment[len(key)+1:]) - } - } - } - return values -} - -func shouldSplitVorbisArtistTags(mode string) bool { - return strings.EqualFold(strings.TrimSpace(mode), artistTagModeSplitVorbis) -} - -func splitArtistTagValues(rawArtists string) []string { - trimmed := strings.TrimSpace(rawArtists) - if trimmed == "" { - return nil - } - - parts := artistTagSplitPattern.Split(trimmed, -1) - values := make([]string, 0, len(parts)) - seen := make(map[string]struct{}, len(parts)) - for _, part := range parts { - artist := strings.TrimSpace(part) - if artist == "" { - continue - } - key := strings.ToLower(artist) - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - values = append(values, artist) - } - if len(values) > 0 { - return values - } - return []string{trimmed} -} - -func joinVorbisCommentValues(values []string) string { - if len(values) == 0 { - return "" - } - - joined := make([]string, 0, len(values)) - seen := make(map[string]struct{}, len(values)) - for _, value := range values { - trimmed := strings.TrimSpace(value) - if trimmed == "" { - continue - } - key := strings.ToLower(trimmed) - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - joined = append(joined, trimmed) - } - return strings.Join(joined, ", ") -} - -func fileExists(path string) bool { - return CheckFileExists(path) -} - -func ExtractCoverArt(filePath string) ([]byte, error) { - f, err := parseFlacFile(filePath) - if err != nil { - return nil, fmt.Errorf("failed to parse FLAC file: %w", err) - } - defer f.Close() - return coverArtFromParsedFlac(f) -} - -func coverArtFromParsedFlac(f *flac.File) ([]byte, error) { - for _, meta := range f.Meta { - if meta.Type == flac.Picture { - pic, err := flacpicture.ParseFromMetaDataBlock(*meta) - if err != nil { - continue - } - if pic.PictureType == flacpicture.PictureTypeFrontCover && len(pic.ImageData) > 0 { - return pic.ImageData, nil - } - } - } - - for _, meta := range f.Meta { - if meta.Type == flac.Picture { - pic, err := flacpicture.ParseFromMetaDataBlock(*meta) - if err != nil { - continue - } - if len(pic.ImageData) > 0 { - return pic.ImageData, nil - } - } - } - - return nil, fmt.Errorf("no cover art found in file") -} - -func EmbedLyrics(filePath string, lyrics string) error { - return updateFlacVorbis(filePath, func(_ *flac.File, cmt *flacvorbis.MetaDataBlockVorbisComment) error { - setComment(cmt, "LYRICS", lyrics) - setComment(cmt, "UNSYNCEDLYRICS", lyrics) - return nil - }) -} - -func ExtractLyrics(filePath string) (string, error) { - lower := strings.ToLower(filePath) - - if strings.HasSuffix(lower, ".flac") { - lyrics, err := extractLyricsFromFlac(filePath) - if err == nil && strings.TrimSpace(lyrics) != "" { - return lyrics, nil - } - return extractLyricsFromSidecarLRC(filePath) - } - - if strings.HasSuffix(lower, ".m4a") || strings.HasSuffix(lower, ".mp4") || strings.HasSuffix(lower, ".aac") { - lyrics, err := extractLyricsFromM4A(filePath) - if err == nil && strings.TrimSpace(lyrics) != "" { - return lyrics, nil - } - return extractLyricsFromSidecarLRC(filePath) - } - - if strings.HasSuffix(lower, ".mp3") { - meta, err := ReadID3Tags(filePath) - if err == nil && meta != nil { - if strings.TrimSpace(meta.Lyrics) != "" { - return meta.Lyrics, nil - } - if looksLikeEmbeddedLyrics(meta.Comment) { - return meta.Comment, nil - } - } - return extractLyricsFromSidecarLRC(filePath) - } - - if strings.HasSuffix(lower, ".opus") || strings.HasSuffix(lower, ".ogg") { - meta, err := ReadOggVorbisComments(filePath) - if err == nil && meta != nil { - if strings.TrimSpace(meta.Lyrics) != "" { - return meta.Lyrics, nil - } - if looksLikeEmbeddedLyrics(meta.Comment) { - return meta.Comment, nil - } - } - return extractLyricsFromSidecarLRC(filePath) - } - - if strings.HasSuffix(lower, ".wav") { - meta, err := ReadWAVTags(filePath) - if err == nil && meta != nil { - if strings.TrimSpace(meta.Lyrics) != "" { - return meta.Lyrics, nil - } - if looksLikeEmbeddedLyrics(meta.Comment) { - return meta.Comment, nil - } - } - return extractLyricsFromSidecarLRC(filePath) - } - - if strings.HasSuffix(lower, ".aiff") || strings.HasSuffix(lower, ".aif") || strings.HasSuffix(lower, ".aifc") { - meta, err := ReadAIFFTags(filePath) - if err == nil && meta != nil { - if strings.TrimSpace(meta.Lyrics) != "" { - return meta.Lyrics, nil - } - if looksLikeEmbeddedLyrics(meta.Comment) { - return meta.Comment, nil - } - } - return extractLyricsFromSidecarLRC(filePath) - } - - return extractLyricsFromSidecarLRC(filePath) -} - -func extractLyricsFromSidecarLRC(filePath string) (string, error) { - ext := filepath.Ext(filePath) - base := strings.TrimSuffix(filePath, ext) - if strings.TrimSpace(base) == "" { - return "", fmt.Errorf("no lyrics found in file") - } - - lrcPath := base + ".lrc" - data, err := os.ReadFile(lrcPath) - if err != nil { - return "", fmt.Errorf("no lyrics found in file") - } - - lyrics := strings.TrimSpace(string(data)) - if lyrics == "" { - return "", fmt.Errorf("no lyrics found in file") - } - return lyrics, nil -} - -func extractLyricsFromFlac(filePath string) (string, error) { - f, err := parseFlacFile(filePath) - if err != nil { - return "", fmt.Errorf("failed to parse FLAC file: %w", err) - } - defer f.Close() - - for _, meta := range f.Meta { - if meta.Type != flac.VorbisComment { - continue - } - - cmt, err := flacvorbis.ParseFromMetaDataBlock(*meta) - if err != nil { - continue - } - - if lyrics := getLyricsComment(cmt); lyrics != "" { - return lyrics, nil - } - } - - return "", fmt.Errorf("no lyrics found in file") -} - -func looksLikeEmbeddedLyrics(value string) bool { - trimmed := strings.TrimSpace(value) - if trimmed == "" { - return false - } - - lower := strings.ToLower(trimmed) - if strings.Contains(lower, "[ar:") || strings.Contains(lower, "[ti:") { - return true - } - - if strings.Contains(trimmed, "\n") && strings.Contains(trimmed, "[") && strings.Contains(trimmed, "]") { - return true - } - - return false -} - -type AudioQuality struct { - BitDepth int `json:"bit_depth"` - SampleRate int `json:"sample_rate"` - TotalSamples int64 `json:"total_samples"` - Duration int `json:"duration"` - Bitrate int `json:"bitrate,omitempty"` // kbps, estimated for compressed MP4-family streams - Codec string `json:"codec,omitempty"` -} - -func flacAudioQualityFromStreamInfo(streamInfo []byte) AudioQuality { - bitDepth, sampleRate, totalSamples := parseFLACStreamInfoQuality(streamInfo) - duration := 0 - if sampleRate > 0 && totalSamples > 0 { - duration = int(totalSamples / int64(sampleRate)) - } - return AudioQuality{ - BitDepth: bitDepth, - SampleRate: sampleRate, - TotalSamples: totalSamples, - Duration: duration, - Codec: "flac", - } -} - -func audioQualityFromParsedFlac(f *flac.File) (AudioQuality, error) { - for _, meta := range f.Meta { - if meta.Type != flac.StreamInfo || len(meta.Data) < 18 { - continue - } - return flacAudioQualityFromStreamInfo(meta.Data), nil - } - return AudioQuality{}, fmt.Errorf("FLAC STREAMINFO block not found") -} - -func GetAudioQuality(filePath string) (AudioQuality, error) { - file, err := os.Open(filePath) - if err != nil { - return AudioQuality{}, fmt.Errorf("failed to open file: %w", err) - } - defer file.Close() - - marker := make([]byte, 4) - if _, err := file.Read(marker); err != nil { - return AudioQuality{}, fmt.Errorf("failed to read marker: %w", err) - } - - if string(marker) == "fLaC" { - header := make([]byte, 4) - if _, err := file.Read(header); err != nil { - return AudioQuality{}, fmt.Errorf("failed to read header: %w", err) - } - - blockType := header[0] & 0x7F - if blockType != 0 { - return AudioQuality{}, fmt.Errorf("first block is not STREAMINFO") - } - - streamInfo := make([]byte, 34) - if _, err := file.Read(streamInfo); err != nil { - return AudioQuality{}, fmt.Errorf("failed to read STREAMINFO: %w", err) - } - - return flacAudioQualityFromStreamInfo(streamInfo), nil - } - - file.Seek(0, 0) - header8 := make([]byte, 8) - if _, err := file.Read(header8); err != nil { - return AudioQuality{}, fmt.Errorf("failed to read header: %w", err) - } - - if string(header8[4:8]) == "ftyp" { - file.Close() - return GetM4AQuality(filePath) - } - - return AudioQuality{}, fmt.Errorf("unsupported file format (not FLAC or M4A)") -} diff --git a/go_backend/metadata_artist_tags_test.go b/go_backend/metadata_artist_tags_test.go deleted file mode 100644 index a829030c..00000000 --- a/go_backend/metadata_artist_tags_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/binary" - "slices" - "testing" - - "github.com/go-flac/flacvorbis/v2" -) - -func TestSplitArtistTagValues(t *testing.T) { - got := splitArtistTagValues("Artist A, Artist B feat. Artist C & Artist B") - want := []string{"Artist A", "Artist B", "Artist C"} - if !slices.Equal(got, want) { - t.Fatalf("splitArtistTagValues() = %#v, want %#v", got, want) - } -} - -func TestSetArtistCommentsSplitVorbis(t *testing.T) { - cmt := flacvorbis.New() - setArtistComments(cmt, "ARTIST", "Artist A, Artist B", artistTagModeSplitVorbis) - - got := getCommentValues(cmt, "ARTIST") - want := []string{"Artist A", "Artist B"} - if !slices.Equal(got, want) { - t.Fatalf("getCommentValues(ARTIST) = %#v, want %#v", got, want) - } -} - -func TestParseVorbisCommentsJoinsRepeatedArtists(t *testing.T) { - metadata := &AudioMetadata{} - parseVorbisComments( - buildVorbisCommentPayload( - []string{ - "TITLE=Song", - "ARTIST=Artist A", - "ARTIST=Artist B", - "ALBUMARTIST=Album Artist A", - "ALBUMARTIST=Album Artist B", - }, - ), - metadata, - ) - - if metadata.Title != "Song" { - t.Fatalf("title = %q", metadata.Title) - } - if metadata.Artist != "Artist A, Artist B" { - t.Fatalf("artist = %q", metadata.Artist) - } - if metadata.AlbumArtist != "Album Artist A, Album Artist B" { - t.Fatalf("album artist = %q", metadata.AlbumArtist) - } -} - -func TestGetLyricsCommentReadsAndClearsSyncedLyrics(t *testing.T) { - cmt := flacvorbis.New() - setComment(cmt, "SYNCEDLYRICS", "[00:01.00]Synced line") - - if got := getLyricsComment(cmt); got != "[00:01.00]Synced line" { - t.Fatalf("getLyricsComment() = %q", got) - } - - applyVorbisFieldEdits(cmt, map[string]string{"lyrics": ""}) - if got := getComment(cmt, "SYNCEDLYRICS"); got != "" { - t.Fatalf("cleared SYNCEDLYRICS = %q", got) - } -} - -func buildVorbisCommentPayload(comments []string) []byte { - var buf bytes.Buffer - _ = binary.Write(&buf, binary.LittleEndian, uint32(len("spotiflac"))) - buf.WriteString("spotiflac") - _ = binary.Write(&buf, binary.LittleEndian, uint32(len(comments))) - for _, comment := range comments { - _ = binary.Write(&buf, binary.LittleEndian, uint32(len(comment))) - buf.WriteString(comment) - } - return buf.Bytes() -} diff --git a/go_backend/metadata_descriptor_test.go b/go_backend/metadata_descriptor_test.go deleted file mode 100644 index 6ac2c6da..00000000 --- a/go_backend/metadata_descriptor_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "runtime" - "testing" -) - -func TestCompleteMetadataHintReturnsFileAccessErrors(t *testing.T) { - for _, format := range []string{"flac", "mp3", "m4a", "mp4", "aac", "opus", "ogg", "wav", "aiff", "aif", "aifc", "ape", "wv", "mpc"} { - t.Run(format, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "descriptor") - payload, err := ReadFileMetadataWithHint(path, "track."+format) - if !errors.Is(err, os.ErrNotExist) || payload != "" { - t.Fatalf("missing descriptor returned metadata=%s err=%v", payload, err) - } - if err := os.WriteFile(path, []byte("inaccessible"), 0000); err != nil { - t.Fatal(err) - } - if file, err := os.Open(path); err == nil { - file.Close() - t.Skip("host can bypass file permissions; missing-path check passed") - } - payload, err = ReadFileMetadataWithHint(path, "track."+format) - if !errors.Is(err, os.ErrPermission) || payload != "" { - t.Fatalf("unreadable descriptor returned metadata=%s err=%v", payload, err) - } - }) - } -} - -func TestCompleteMetadataHintAcceptsAudioWithoutTags(t *testing.T) { - for _, format := range []string{"wav", "aiff"} { - t.Run(format, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "descriptor") - if format == "wav" { - writeTestWAV(t, path) - } else { - writeTestAIFF(t, path) - } - payload, err := ReadFileMetadataWithHint(path, "track."+format) - if err != nil { - t.Fatal(err) - } - var metadata map[string]any - if err := json.Unmarshal([]byte(payload), &metadata); err != nil { - t.Fatal(err) - } - if metadata["title"] != "" || metadata["sample_rate"] != float64(44100) { - t.Fatalf("unexpected tagless audio metadata: %s", payload) - } - }) - } -} - -func TestCompleteMetadataHintMatchesNamedFileAndDescriptor(t *testing.T) { - for _, format := range []string{"mp3", "flac", "m4a", "wav"} { - t.Run(format, func(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "track."+format) - switch format { - case "mp3": - data := buildID3v23Tag(id3TextFrame("TIT2", "Song"), id3TextFrame("TPE1", "Artist"), id3TextFrame("TSRC", "USRC17607839"), id3CommentFrame("USLT", "Words"), id3UserTextFrame("TXXX", "REPLAYGAIN_TRACK_GAIN", "-6.00 dB"), id3UserTextFrame("TXXX", "REPLAYGAIN_ALBUM_GAIN", "-4.00 dB")) - if err := os.WriteFile(path, data, 0600); err != nil { - t.Fatal(err) - } - case "flac": - writeSinglePassTestFlac(t, path, nil) - case "wav": - writeTestWAV(t, path) - case "m4a": - data, _ := buildTestM4A(t, buildM4ATextAtom("\xa9nam", "Song"), []byte("audio")) - if err := os.WriteFile(path, data, 0600); err != nil { - t.Fatal(err) - } - } - expected, err := ReadFileMetadata(path) - if err != nil { - t.Fatal(err) - } - extensionless := filepath.Join(dir, "descriptor") - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(extensionless, data, 0600); err != nil { - t.Fatal(err) - } - actual, err := ReadFileMetadataWithHint(extensionless, "track."+format) - if err != nil || actual != expected { - t.Fatalf("hinted metadata=%s expected=%s err=%v", actual, expected, err) - } - // Android uses /proc, which reopens with an independent offset. - // macOS /dev/fd duplicates the shared offset and is not that API. - if runtime.GOOS != "linux" { - return - } - file, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer file.Close() - prefix := "/proc/self/fd/" - actual, err = ReadFileMetadataWithHint(fmt.Sprintf("%s%d", prefix, file.Fd()), "track."+format) - if err != nil || actual != expected { - t.Fatalf("descriptor metadata=%s expected=%s err=%v", actual, expected, err) - } - }) - } -} diff --git a/go_backend/metadata_m4a.go b/go_backend/metadata_m4a.go deleted file mode 100644 index f912938f..00000000 --- a/go_backend/metadata_m4a.go +++ /dev/null @@ -1,1209 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "math" - "os" - "regexp" - "strconv" - "strings" -) - -func ReadM4ATags(filePath string) (*AudioMetadata, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return nil, err - } - - ilst, err := findM4AIlstAtom(f, fi.Size()) - if err != nil { - return nil, err - } - return readM4ATagsFromIlst(f, fi.Size(), ilst) -} - -func readM4ATagsFromIlst(f *os.File, fileSize int64, ilst atomHeader) (*AudioMetadata, error) { - metadata := &AudioMetadata{} - start := ilst.offset + ilst.headerSize - end := ilst.offset + ilst.size - for pos := start; pos+8 <= end; { - header, err := readAtomHeaderAt(f, pos, fileSize) - if err != nil { - return nil, err - } - if header.size == 0 { - header.size = end - pos - } - if header.size < header.headerSize { - return nil, fmt.Errorf("invalid atom size for %s", header.typ) - } - - switch header.typ { - case "\xa9nam": - metadata.Title, _ = readM4ATextValue(f, header, fileSize) - case "\xa9ART": - metadata.Artist, _ = readM4ATextValue(f, header, fileSize) - case "\xa9alb": - metadata.Album, _ = readM4ATextValue(f, header, fileSize) - case "aART": - metadata.AlbumArtist, _ = readM4ATextValue(f, header, fileSize) - case "\xa9day": - metadata.Date, _ = readM4ATextValue(f, header, fileSize) - metadata.Year = metadata.Date - case "\xa9gen": - metadata.Genre, _ = readM4ATextValue(f, header, fileSize) - case "\xa9wrt": - metadata.Composer, _ = readM4ATextValue(f, header, fileSize) - case "\xa9cmt": - metadata.Comment, _ = readM4ATextValue(f, header, fileSize) - case "cprt": - metadata.Copyright, _ = readM4ATextValue(f, header, fileSize) - case "\xa9lyr": - metadata.Lyrics, _ = readM4ATextValue(f, header, fileSize) - case "trkn": - metadata.TrackNumber, metadata.TotalTracks, _ = readM4AIndexPair(f, header, fileSize) - case "disk": - metadata.DiscNumber, metadata.TotalDiscs, _ = readM4AIndexPair(f, header, fileSize) - case "rtng": - if value, valueErr := readM4AByteValue(f, header, fileSize); valueErr == nil { - metadata.Explicit = value == 1 - } - case "cpil": - if value, valueErr := readM4AByteValue(f, header, fileSize); valueErr == nil && value != 0 && metadata.AlbumType == "" { - metadata.AlbumType = "compilation" - } - case "----": - name, value, freeformErr := readM4AFreeformValue(f, header, fileSize) - if freeformErr == nil { - switch strings.ToUpper(strings.TrimSpace(name)) { - case "ISRC": - metadata.ISRC = value - case "LABEL", "ORGANIZATION": - metadata.Label = value - case "COMMENT": - if metadata.Comment == "" { - metadata.Comment = value - } - case "COMPOSER": - if metadata.Composer == "" { - metadata.Composer = value - } - case "COPYRIGHT": - if metadata.Copyright == "" { - metadata.Copyright = value - } - case "LYRICS", "UNSYNCEDLYRICS", "SYNCEDLYRICS": - if metadata.Lyrics == "" { - metadata.Lyrics = value - } - case "REPLAYGAIN_TRACK_GAIN": - metadata.ReplayGainTrackGain = value - case "REPLAYGAIN_TRACK_PEAK": - metadata.ReplayGainTrackPeak = value - case "REPLAYGAIN_ALBUM_GAIN": - metadata.ReplayGainAlbumGain = value - case "REPLAYGAIN_ALBUM_PEAK": - metadata.ReplayGainAlbumPeak = value - case "ITUNESADVISORY": - metadata.Explicit = isTruthyTagValue(value) - case "RELEASETYPE": - metadata.AlbumType = value - case "BARCODE", "UPC": - metadata.UPC = value - } - } - } - - pos += header.size - } - - if metadata.Title == "" && - metadata.Artist == "" && - metadata.Album == "" && - metadata.AlbumArtist == "" && - metadata.Lyrics == "" && - metadata.TrackNumber == 0 && - metadata.DiscNumber == 0 { - return nil, fmt.Errorf("no M4A tags found") - } - - return metadata, nil -} - -func extractLyricsFromM4A(filePath string) (string, error) { - metadata, err := ReadM4ATags(filePath) - if err != nil { - return "", err - } - if metadata == nil || strings.TrimSpace(metadata.Lyrics) == "" { - return "", fmt.Errorf("no lyrics found in file") - } - return metadata.Lyrics, nil -} - -func extractCoverFromM4A(filePath string) ([]byte, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return nil, err - } - fileSize := fi.Size() - - ilst, err := findM4AIlstAtom(f, fileSize) - if err != nil { - return nil, err - } - return extractCoverFromM4AIlst(f, fileSize, ilst) -} - -func extractCoverFromM4AIlst(f *os.File, fileSize int64, ilst atomHeader) ([]byte, error) { - bodyStart := ilst.offset + ilst.headerSize - bodySize := ilst.size - ilst.headerSize - - covr, found, err := findAtomInRange(f, bodyStart, bodySize, "covr", fileSize) - if err != nil || !found { - return nil, fmt.Errorf("cover atom not found") - } - - dataStart := covr.offset + covr.headerSize - dataSize := covr.size - covr.headerSize - - dataAtom, found, err := findAtomInRange(f, dataStart, dataSize, "data", fileSize) - if err != nil || !found { - return nil, fmt.Errorf("data atom not found in cover") - } - - // data atom: header + 4 bytes type indicator + 4 bytes locale - imgStart := dataAtom.offset + dataAtom.headerSize + 8 - imgLen := dataAtom.size - dataAtom.headerSize - 8 - if imgLen <= 0 { - return nil, fmt.Errorf("empty cover data") - } - - buf := make([]byte, imgLen) - if _, err := f.ReadAt(buf, imgStart); err != nil { - return nil, err - } - - return buf, nil -} - -// findM4AIlstAtom locates the ilst atom that holds all iTunes-style tags. -// It tries two common layouts: -// 1. moov > udta > meta > ilst (iTunes, FFmpeg default) -// 2. moov > meta > ilst (some encoders omit the udta wrapper) -func findM4AIlstAtom(f *os.File, fileSize int64) (atomHeader, error) { - moov, found, err := findAtomInRange(f, 0, fileSize, "moov", fileSize) - if err != nil || !found { - return atomHeader{}, fmt.Errorf("moov not found") - } - - moovBodyStart := moov.offset + moov.headerSize - moovBodySize := moov.size - moov.headerSize - - // Path 1: moov > udta > meta > ilst - if udta, ok, _ := findAtomInRange(f, moovBodyStart, moovBodySize, "udta", fileSize); ok { - udtaBodyStart := udta.offset + udta.headerSize - udtaBodySize := udta.size - udta.headerSize - if meta, ok2, _ := findAtomInRange(f, udtaBodyStart, udtaBodySize, "meta", fileSize); ok2 { - if ilst, ok3 := findIlstInMeta(f, meta, fileSize); ok3 { - return ilst, nil - } - } - } - - // Path 2: moov > meta > ilst (no udta wrapper) - if meta, ok, _ := findAtomInRange(f, moovBodyStart, moovBodySize, "meta", fileSize); ok { - if ilst, ok2 := findIlstInMeta(f, meta, fileSize); ok2 { - return ilst, nil - } - } - - return atomHeader{}, fmt.Errorf("ilst not found (tried moov>udta>meta>ilst and moov>meta>ilst)") -} - -// findIlstInMeta locates the ilst atom inside a meta atom, handling both -// layouts: ISO-BMFF (4-byte version/flags before the child atoms, written by -// FFmpeg's mp4 muxer) and QuickTime (no version/flags, written by the mov muxer -// used for AC-4 passthrough). -func findIlstInMeta(f *os.File, meta atomHeader, fileSize int64) (atomHeader, bool) { - // ISO-BMFF: skip the 4-byte version/flags that precede the child atoms. - isoStart := meta.offset + meta.headerSize + 4 - isoSize := meta.size - meta.headerSize - 4 - if ilst, ok, _ := findAtomInRange(f, isoStart, isoSize, "ilst", fileSize); ok { - return ilst, true - } - // QuickTime: child atoms begin immediately after the meta header. - qtStart := meta.offset + meta.headerSize - qtSize := meta.size - meta.headerSize - if ilst, ok, _ := findAtomInRange(f, qtStart, qtSize, "ilst", fileSize); ok { - return ilst, true - } - return atomHeader{}, false -} - -func readM4ADataAtomPayload(f *os.File, dataAtom atomHeader) ([]byte, error) { - payloadStart := dataAtom.offset + dataAtom.headerSize + 8 - payloadLen := dataAtom.size - dataAtom.headerSize - 8 - if payloadLen <= 0 { - return nil, fmt.Errorf("empty data atom in %s", dataAtom.typ) - } - - buf := make([]byte, payloadLen) - if _, err := f.ReadAt(buf, payloadStart); err != nil { - return nil, err - } - return buf, nil -} - -func readM4ADataPayload(f *os.File, parent atomHeader, fileSize int64) ([]byte, error) { - dataStart := parent.offset + parent.headerSize - dataSize := parent.size - parent.headerSize - - dataAtom, found, err := findAtomInRange(f, dataStart, dataSize, "data", fileSize) - if err != nil || !found { - return nil, fmt.Errorf("data atom not found in %s", parent.typ) - } - return readM4ADataAtomPayload(f, dataAtom) -} - -func readM4ATextValue(f *os.File, parent atomHeader, fileSize int64) (string, error) { - payload, err := readM4ADataPayload(f, parent, fileSize) - if err != nil { - return "", err - } - return strings.TrimSpace(strings.TrimRight(string(payload), "\x00")), nil -} - -func readM4AIndexPair(f *os.File, parent atomHeader, fileSize int64) (int, int, error) { - payload, err := readM4ADataPayload(f, parent, fileSize) - if err != nil { - return 0, 0, err - } - if len(payload) < 6 { - return 0, 0, fmt.Errorf("index payload too short in %s", parent.typ) - } - return int(binary.BigEndian.Uint16(payload[2:4])), int(binary.BigEndian.Uint16(payload[4:6])), nil -} - -func readM4AByteValue(f *os.File, parent atomHeader, fileSize int64) (byte, error) { - payload, err := readM4ADataPayload(f, parent, fileSize) - if err != nil { - return 0, err - } - if len(payload) == 0 { - return 0, fmt.Errorf("integer payload is empty in %s", parent.typ) - } - return payload[len(payload)-1], nil -} - -func parsePositiveInt(value string) int { - value = strings.TrimSpace(value) - if value == "" { - return 0 - } - n, _ := strconv.Atoi(value) - return n -} - -func formatIndexValue(number, total int) string { - if number <= 0 { - return "" - } - if total > 0 { - return fmt.Sprintf("%d/%d", number, total) - } - return strconv.Itoa(number) -} - -func hasMapKey(fields map[string]string, key string) bool { - _, ok := fields[key] - return ok -} - -func readM4AFreeformValue(f *os.File, parent atomHeader, fileSize int64) (string, string, error) { - start := parent.offset + parent.headerSize - end := parent.offset + parent.size - - var nameValue string - var dataValue string - for pos := start; pos+8 <= end; { - header, err := readAtomHeaderAt(f, pos, fileSize) - if err != nil { - return "", "", err - } - if header.size == 0 { - header.size = end - pos - } - if header.size < header.headerSize { - return "", "", fmt.Errorf("invalid atom size for %s", header.typ) - } - - switch header.typ { - case "mean": - // Domain qualifier (e.g. "com.apple.iTunes") — not needed, skip. - case "name": - // The "name" atom payload is: 4-byte version/flags, then raw UTF-8 text. - // It does NOT contain a nested "data" atom, so read the payload directly. - payloadStart := header.offset + header.headerSize + 4 - payloadLen := header.size - header.headerSize - 4 - if payloadLen > 0 { - buf := make([]byte, payloadLen) - if _, readErr := f.ReadAt(buf, payloadStart); readErr == nil { - nameValue = strings.TrimSpace(strings.TrimRight(string(buf), "\x00")) - } - } - case "data": - payload, payloadErr := readM4ADataAtomPayload(f, header) - if payloadErr == nil { - dataValue = strings.TrimSpace(strings.TrimRight(string(payload), "\x00")) - } - } - - pos += header.size - } - - if nameValue == "" || dataValue == "" { - return "", "", fmt.Errorf("freeform M4A tag incomplete") - } - - return nameValue, dataValue, nil -} - -type m4aMetadataPath struct { - moov atomHeader - udta *atomHeader - meta atomHeader - ilst atomHeader -} - -func findM4AMetadataPath(f *os.File, fileSize int64) (m4aMetadataPath, error) { - moov, found, err := findAtomInRange(f, 0, fileSize, "moov", fileSize) - if err != nil || !found { - return m4aMetadataPath{}, fmt.Errorf("moov not found") - } - - moovBodyStart := moov.offset + moov.headerSize - moovBodySize := moov.size - moov.headerSize - - if udta, ok, _ := findAtomInRange(f, moovBodyStart, moovBodySize, "udta", fileSize); ok { - udtaBodyStart := udta.offset + udta.headerSize - udtaBodySize := udta.size - udta.headerSize - if meta, ok2, _ := findAtomInRange(f, udtaBodyStart, udtaBodySize, "meta", fileSize); ok2 { - if ilst, ok3 := findIlstInMeta(f, meta, fileSize); ok3 { - udtaCopy := udta - return m4aMetadataPath{ - moov: moov, - udta: &udtaCopy, - meta: meta, - ilst: ilst, - }, nil - } - } - } - - if meta, ok, _ := findAtomInRange(f, moovBodyStart, moovBodySize, "meta", fileSize); ok { - if ilst, ok2 := findIlstInMeta(f, meta, fileSize); ok2 { - return m4aMetadataPath{ - moov: moov, - meta: meta, - ilst: ilst, - }, nil - } - } - - return m4aMetadataPath{}, fmt.Errorf("ilst not found (tried moov>udta>meta>ilst and moov>meta>ilst)") -} - -func buildM4AAtom(typ string, payload []byte) []byte { - size := int64(8 + len(payload)) - buf := make([]byte, 8+len(payload)) - binary.BigEndian.PutUint32(buf[0:4], uint32(size)) - copy(buf[4:8], []byte(typ)) - copy(buf[8:], payload) - return buf -} - -func buildM4AFreeformAtom(name, value string) []byte { - meanPayload := append([]byte{0, 0, 0, 0}, []byte("com.apple.iTunes")...) - namePayload := append([]byte{0, 0, 0, 0}, []byte(name)...) - dataPayload := make([]byte, 8+len(value)) - binary.BigEndian.PutUint32(dataPayload[0:4], 1) // UTF-8 text - copy(dataPayload[8:], []byte(value)) - - payload := append([]byte{}, buildM4AAtom("mean", meanPayload)...) - payload = append(payload, buildM4AAtom("name", namePayload)...) - payload = append(payload, buildM4AAtom("data", dataPayload)...) - return buildM4AAtom("----", payload) -} - -func buildITunNORMTag(trackGain, trackPeak string) string { - gainDb, ok := parseReplayGainDb(trackGain) - if !ok { - return "" - } - peakLinear, ok := parseReplayGainPeak(trackPeak) - if !ok { - return "" - } - - clamp := func(v int64) int64 { - if v < 0 { - return 0 - } - if v > 65534 { - return 65534 - } - return v - } - - g1 := clamp(int64(math.Round(math.Pow(10, gainDb/-10.0) * 1000.0))) - g2 := clamp(int64(math.Round(math.Pow(10, gainDb/-10.0) * 2500.0))) - peak := clamp(int64(math.Round(peakLinear * 32768.0))) - values := []int64{g1, g1, g2, g2, 0, 0, peak, peak, 0, 0} - parts := make([]string, 0, len(values)) - for _, value := range values { - parts = append(parts, strings.ToUpper(fmt.Sprintf("%08x", value))) - } - return strings.Join(parts, " ") -} - -var replayGainNumberPattern = regexp.MustCompile(`([+-]?\d+(?:\.\d+)?)`) - -func parseReplayGainDb(value string) (float64, bool) { - match := replayGainNumberPattern.FindStringSubmatch(strings.TrimSpace(value)) - if len(match) < 2 { - return 0, false - } - parsed, err := strconv.ParseFloat(match[1], 64) - if err != nil { - return 0, false - } - return parsed, true -} - -func parseReplayGainPeak(value string) (float64, bool) { - parsed, err := strconv.ParseFloat(strings.TrimSpace(value), 64) - if err != nil || parsed <= 0 { - return 0, false - } - return parsed, true -} - -func collectM4AReplayGainFields(fields map[string]string) map[string]string { - result := map[string]string{} - if value := strings.TrimSpace(fields["replaygain_track_gain"]); value != "" { - result["replaygain_track_gain"] = value - } - if value := strings.TrimSpace(fields["replaygain_track_peak"]); value != "" { - result["replaygain_track_peak"] = value - } - if value := strings.TrimSpace(fields["replaygain_album_gain"]); value != "" { - result["replaygain_album_gain"] = value - } - if value := strings.TrimSpace(fields["replaygain_album_peak"]); value != "" { - result["replaygain_album_peak"] = value - } - - if norm := buildITunNORMTag(result["replaygain_track_gain"], result["replaygain_track_peak"]); norm != "" { - result["iTunNORM"] = norm - } - - return result -} - -func writeAtomSize(buf []byte, header atomHeader, newSize int64) error { - if newSize <= 0 { - return fmt.Errorf("invalid size for %s", header.typ) - } - if header.headerSize == 16 { - if int(header.offset)+16 > len(buf) { - return io.ErrUnexpectedEOF - } - binary.BigEndian.PutUint32(buf[header.offset:header.offset+4], 1) - binary.BigEndian.PutUint64(buf[header.offset+8:header.offset+16], uint64(newSize)) - return nil - } - if newSize > math.MaxUint32 { - return fmt.Errorf("atom %s too large for 32-bit header", header.typ) - } - if int(header.offset)+8 > len(buf) { - return io.ErrUnexpectedEOF - } - binary.BigEndian.PutUint32(buf[header.offset:header.offset+4], uint32(newSize)) - return nil -} - -func EditM4AReplayGain(filePath string, fields map[string]string) error { - replayGainFields := collectM4AReplayGainFields(fields) - if len(replayGainFields) == 0 { - return nil - } - - remove := map[string]struct{}{ - "REPLAYGAIN_TRACK_GAIN": {}, - "REPLAYGAIN_TRACK_PEAK": {}, - "REPLAYGAIN_ALBUM_GAIN": {}, - "REPLAYGAIN_ALBUM_PEAK": {}, - "ITUNNORM": {}, - } - - order := []string{ - "replaygain_track_gain", - "replaygain_track_peak", - "replaygain_album_gain", - "replaygain_album_peak", - "iTunNORM", - } - tags := make([]m4aFreeformTag, 0, len(order)) - for _, key := range order { - value := strings.TrimSpace(replayGainFields[key]) - if value == "" { - continue - } - name := key - if key != "iTunNORM" { - name = strings.ToLower(key) - } - tags = append(tags, m4aFreeformTag{name: name, value: value}) - } - - return writeM4AFreeformTags(filePath, remove, tags) -} - -type m4aFreeformTag struct { - name string - value string -} - -// writeM4AFreeformTags rewrites the ilst atom in place: it drops every existing -// freeform ("----") atom whose uppercased name is in `remove`, then appends the -// supplied tags (empty values are skipped, which effectively clears the field). -// Atom sizes are fixed up along the ilst -> meta -> udta -> moov chain. -// -// FFmpeg's MP4 muxer only writes a fixed set of recognized keys to the ilst, so -// fields like ISRC and LABEL are silently dropped when written via -metadata. -// Writing them as iTunes freeform atoms natively is the only way they persist. -func writeM4AFreeformTags(filePath string, remove map[string]struct{}, tags []m4aFreeformTag) error { - f, err := os.Open(filePath) - if err != nil { - return err - } - defer f.Close() - - info, err := f.Stat() - if err != nil { - return err - } - - path, err := findM4AMetadataPath(f, info.Size()) - if err != nil { - // MOV-style containers (e.g. AC-4 passthrough) store tags as QuickTime - // atoms under udta with no iTunes meta>ilst structure. There is nowhere - // to write freeform tags, so skip gracefully instead of failing. - if strings.Contains(err.Error(), "ilst not found") { - GoLog("[Metadata] No iTunes ilst container; skipping freeform tags") - return nil - } - return err - } - - // Only the moov box is buffered; the audio bulk is streamed on write. - base := path.moov.offset - moovBuf := make([]byte, path.moov.size) - if _, err := f.ReadAt(moovBuf, base); err != nil { - return err - } - - bodyStart := path.ilst.offset + path.ilst.headerSize - bodyEnd := path.ilst.offset + path.ilst.size - newBody := make([]byte, 0, int(path.ilst.size)) - - for pos := bodyStart; pos+8 <= bodyEnd; { - header, readErr := readAtomHeaderAt(f, pos, info.Size()) - if readErr != nil { - return readErr - } - if header.size == 0 { - header.size = bodyEnd - pos - } - if header.size < header.headerSize { - return fmt.Errorf("invalid atom size for %s", header.typ) - } - - keep := true - if header.typ == "----" { - name, _, freeformErr := readM4AFreeformValue(f, header, info.Size()) - if freeformErr == nil { - if _, ok := remove[strings.ToUpper(strings.TrimSpace(name))]; ok { - keep = false - } - } - } - if keep { - newBody = append(newBody, moovBuf[pos-base:pos-base+header.size]...) - } - - pos += header.size - } - - for _, tag := range tags { - if strings.TrimSpace(tag.value) == "" { - continue - } - newBody = append(newBody, buildM4AFreeformAtom(tag.name, tag.value)...) - } - - newIlst := buildM4AAtom("ilst", newBody) - ilstRel := path.ilst.offset - base - updated := append([]byte{}, moovBuf[:ilstRel]...) - updated = append(updated, newIlst...) - updated = append(updated, moovBuf[ilstRel+path.ilst.size:]...) - - // The path headers carry absolute file offsets; rebase them onto the - // moov-rooted buffer before patching sizes. - rel := func(h atomHeader) atomHeader { - h.offset -= base - return h - } - delta := int64(len(newIlst)) - path.ilst.size - if err := writeAtomSize(updated, rel(path.ilst), path.ilst.size+delta); err != nil { - return err - } - if err := writeAtomSize(updated, rel(path.meta), path.meta.size+delta); err != nil { - return err - } - if path.udta != nil { - if err := writeAtomSize(updated, rel(*path.udta), path.udta.size+delta); err != nil { - return err - } - } - if err := writeAtomSize(updated, rel(path.moov), path.moov.size+delta); err != nil { - return err - } - // Keep sample pointers valid when moov precedes mdat: every stco/co64 - // entry at or beyond the resized ilst must shift with it. Entries hold - // absolute file offsets, so compare against the absolute ilst position. - if delta != 0 { - if moov, ok := findChildMP4(updated, 0, int64(len(updated)), "moov"); ok { - shiftChunkOffsets(updated, moov, path.ilst.offset, delta) - } - } - - // Release the read handle before replacing the file (required on Windows). - f.Close() - return replaceFileSectionsStreaming(filePath, []fileSection{ - {start: base, end: base + path.moov.size, data: updated}, - }) -} - -// EditM4AFreeformText writes ISRC and label tags into an M4A/MP4 file as iTunes -// freeform atoms. These keys are not part of FFmpeg's MP4 metadata key set, so -// they must be written natively for the values to actually persist. An empty -// value clears the corresponding tag. Other (recognized) tags are left intact. -func EditM4AFreeformText(filePath string, fields map[string]string) error { - _, hasISRC := fields["isrc"] - _, hasLabel := fields["label"] - if !hasISRC && !hasLabel { - return nil - } - - remove := map[string]struct{}{} - tags := make([]m4aFreeformTag, 0, 2) - if hasISRC { - remove["ISRC"] = struct{}{} - tags = append(tags, m4aFreeformTag{name: "ISRC", value: strings.TrimSpace(fields["isrc"])}) - } - if hasLabel { - remove["LABEL"] = struct{}{} - remove["ORGANIZATION"] = struct{}{} - tags = append(tags, m4aFreeformTag{name: "LABEL", value: strings.TrimSpace(fields["label"])}) - } - - return writeM4AFreeformTags(filePath, remove, tags) -} - -func GetM4AQuality(filePath string) (AudioQuality, error) { - f, err := os.Open(filePath) - if err != nil { - return AudioQuality{}, fmt.Errorf("failed to open M4A file: %w", err) - } - defer f.Close() - - info, err := f.Stat() - if err != nil { - return AudioQuality{}, fmt.Errorf("failed to stat M4A file: %w", err) - } - fileSize := info.Size() - return m4aQualityFromFile(f, fileSize) -} - -func m4aQualityFromFile(f *os.File, fileSize int64) (AudioQuality, error) { - moovHeader, moovFound, err := findAtomInRange(f, 0, fileSize, "moov", fileSize) - if err != nil { - return AudioQuality{}, fmt.Errorf("failed to find moov atom: %w", err) - } - if !moovFound { - return AudioQuality{}, fmt.Errorf("moov atom not found") - } - - moovStart := moovHeader.offset - moovEnd := moovHeader.offset + moovHeader.size - duration := readM4ADurationSeconds(f, moovHeader, fileSize) - - sampleOffset, atomType, err := findAudioSampleEntry(f, moovStart, moovEnd, fileSize) - if err != nil { - return AudioQuality{}, err - } - - buf := make([]byte, 32) - if _, err := f.ReadAt(buf, sampleOffset); err != nil { - return AudioQuality{}, fmt.Errorf("failed to read audio sample entry: %w", err) - } - - // AudioSampleEntry layout from the box type field: - // [0:4] type ("mp4a"/"alac") - // [4:10] SampleEntry.reserved - // [10:12] data_reference_index - // [12:20] reserved[8] - // [20:22] channelcount - // [22:24] samplesize (bit depth) - // [24:26] pre_defined - // [26:28] reserved - // [28:32] samplerate (16.16 fixed-point) - sampleRate := int(buf[28])<<8 | int(buf[29]) - bitDepth := 0 - codec := normalizeM4AAudioCodec(atomType) - - switch atomType { - case "alac": - bitDepth = int(buf[22])<<8 | int(buf[23]) - if alacBitDepth, alacSampleRate, ok := readALACSpecificConfig(f, sampleOffset, fileSize); ok { - if alacBitDepth > 0 { - bitDepth = alacBitDepth - } - if alacSampleRate > 0 { - sampleRate = alacSampleRate - } - } - case "fLaC": - bitDepth = int(buf[22])<<8 | int(buf[23]) - if flacBitDepth, flacSampleRate, flacTotalSamples, ok := readMP4FLACSpecificConfig(f, sampleOffset, fileSize); ok { - if flacBitDepth > 0 { - bitDepth = flacBitDepth - } - if flacSampleRate > 0 { - sampleRate = flacSampleRate - } - if flacTotalSamples > 0 && sampleRate > 0 && duration <= 0 { - duration = int(flacTotalSamples / int64(sampleRate)) - } - } - } - - bitrate := estimateAudioBitrateKbps(fileSize, duration) - if bitrate > 0 && bitrate < 16 { - bitrate = 0 - } - return AudioQuality{ - BitDepth: bitDepth, - SampleRate: sampleRate, - Duration: duration, - Bitrate: bitrate, - Codec: codec, - }, nil -} - -func normalizeM4AAudioCodec(atomType string) string { - switch atomType { - case "mp4a": - return "aac" - case "alac": - return "alac" - case "fLaC": - return "flac" - case "ec-3": - return "eac3" - case "ac-3": - return "ac3" - case "ac-4": - return "ac4" - case "Opus": - return "opus" - default: - return strings.TrimSpace(atomType) - } -} - -func estimateAudioBitrateKbps(fileSize int64, durationSeconds int) int { - if fileSize <= 0 || durationSeconds <= 0 { - return 0 - } - return int(math.Round(float64(fileSize*8) / float64(durationSeconds) / 1000.0)) -} - -func readM4ADurationSeconds(f *os.File, moovHeader atomHeader, fileSize int64) int { - childStart := moovHeader.offset + moovHeader.headerSize - childSize := moovHeader.size - moovHeader.headerSize - mvhdHeader, found, err := findAtomInRange(f, childStart, childSize, "mvhd", fileSize) - if err == nil && found { - if duration := readMP4DurationAtomSeconds(f, mvhdHeader, fileSize); duration > 0 { - return duration - } - } - - return readM4ATrackDurationSeconds(f, moovHeader, fileSize) -} - -func readMP4DurationAtomSeconds(f *os.File, header atomHeader, _ int64) int { - payloadOffset := header.offset + header.headerSize - versionBuf := make([]byte, 1) - if _, err := f.ReadAt(versionBuf, payloadOffset); err != nil { - return 0 - } - - if versionBuf[0] == 1 { - buf := make([]byte, 32) - if _, err := f.ReadAt(buf, payloadOffset); err != nil { - return 0 - } - timescale := binary.BigEndian.Uint32(buf[20:24]) - duration := binary.BigEndian.Uint64(buf[24:32]) - if timescale == 0 || duration == 0 { - return 0 - } - return int(math.Round(float64(duration) / float64(timescale))) - } - - buf := make([]byte, 20) - if _, err := f.ReadAt(buf, payloadOffset); err != nil { - return 0 - } - timescale := binary.BigEndian.Uint32(buf[12:16]) - duration := binary.BigEndian.Uint32(buf[16:20]) - if timescale == 0 || duration == 0 { - return 0 - } - return int(math.Round(float64(duration) / float64(timescale))) -} - -func readM4ATrackDurationSeconds(f *os.File, moovHeader atomHeader, fileSize int64) int { - childStart := moovHeader.offset + moovHeader.headerSize - childSize := moovHeader.size - moovHeader.headerSize - bestDuration := 0 - _ = walkMP4AtomsInRange(f, childStart, childSize, fileSize, func(header atomHeader) bool { - if header.typ == "mdhd" { - if duration := readMP4DurationAtomSeconds(f, header, fileSize); duration > bestDuration { - bestDuration = duration - } - return false - } - return header.typ == "trak" || header.typ == "mdia" - }) - return bestDuration -} - -func walkMP4AtomsInRange(f *os.File, start, size, fileSize int64, visit func(atomHeader) bool) error { - if size <= 0 { - return nil - } - - end := start + size - for pos := start; pos+8 <= end; { - header, err := readAtomHeaderAt(f, pos, fileSize) - if err != nil { - return err - } - atomSize := header.size - if atomSize == 0 { - atomSize = end - pos - } - if atomSize < header.headerSize { - return fmt.Errorf("invalid atom size for %s", header.typ) - } - header.size = atomSize - if visit(header) { - childStart := header.offset + header.headerSize - childSize := header.size - header.headerSize - if err := walkMP4AtomsInRange(f, childStart, childSize, fileSize, visit); err != nil { - return err - } - } - pos += atomSize - } - return nil -} - -func readALACSpecificConfig(f *os.File, sampleOffset, fileSize int64) (int, int, bool) { - if sampleOffset < 4 { - return 0, 0, false - } - - sampleEntryHeader, err := readAtomHeaderAt(f, sampleOffset-4, fileSize) - if err != nil { - return 0, 0, false - } - - childStart := sampleOffset + 32 - childEnd := sampleEntryHeader.offset + sampleEntryHeader.size - if childStart >= childEnd { - return 0, 0, false - } - - configHeader, found, err := findAtomInRange(f, childStart, childEnd-childStart, "alac", fileSize) - if err != nil || !found { - return 0, 0, false - } - - payloadSize := configHeader.size - configHeader.headerSize - if payloadSize <= 0 { - return 0, 0, false - } - - payload := make([]byte, payloadSize) - if _, err := f.ReadAt(payload, configHeader.offset+configHeader.headerSize); err != nil { - return 0, 0, false - } - - return parseALACSpecificConfig(payload) -} - -func readMP4FLACSpecificConfig(f *os.File, sampleOffset, fileSize int64) (int, int, int64, bool) { - if sampleOffset < 4 { - return 0, 0, 0, false - } - - sampleEntryHeader, err := readAtomHeaderAt(f, sampleOffset-4, fileSize) - if err != nil { - return 0, 0, 0, false - } - - childStart := sampleOffset + 32 - childEnd := sampleEntryHeader.offset + sampleEntryHeader.size - if childStart >= childEnd { - return 0, 0, 0, false - } - - configHeader, found, err := findAtomInRange(f, childStart, childEnd-childStart, "dfLa", fileSize) - if err != nil || !found { - return 0, 0, 0, false - } - - payloadSize := configHeader.size - configHeader.headerSize - if payloadSize <= 0 { - return 0, 0, 0, false - } - - payload := make([]byte, payloadSize) - if _, err := f.ReadAt(payload, configHeader.offset+configHeader.headerSize); err != nil { - return 0, 0, 0, false - } - - return parseMP4FLACSpecificConfig(payload) -} - -func parseMP4FLACSpecificConfig(payload []byte) (int, int, int64, bool) { - if len(payload) >= 4 && string(payload[:4]) == "fLaC" { - payload = payload[4:] - } else if len(payload) >= 4 { - // FLACSpecificBox starts with a full-box version/flags field. - payload = payload[4:] - } - - for len(payload) >= 4 { - blockType := payload[0] & 0x7F - blockLen := int(payload[1])<<16 | int(payload[2])<<8 | int(payload[3]) - if blockLen < 0 || len(payload) < 4+blockLen { - return 0, 0, 0, false - } - block := payload[4 : 4+blockLen] - if blockType == 0 && len(block) >= 34 { - bitDepth, sampleRate, totalSamples := parseFLACStreamInfoQuality(block[:34]) - return bitDepth, sampleRate, totalSamples, bitDepth > 0 || sampleRate > 0 - } - payload = payload[4+blockLen:] - } - - return 0, 0, 0, false -} - -func parseFLACStreamInfoQuality(streamInfo []byte) (int, int, int64) { - if len(streamInfo) < 18 { - return 0, 0, 0 - } - sampleRate := (int(streamInfo[10]) << 12) | (int(streamInfo[11]) << 4) | (int(streamInfo[12]) >> 4) - bitsPerSample := (((int(streamInfo[12]) & 0x01) << 4) | (int(streamInfo[13]) >> 4)) + 1 - totalSamples := int64(streamInfo[13]&0x0F)<<32 | - int64(streamInfo[14])<<24 | - int64(streamInfo[15])<<16 | - int64(streamInfo[16])<<8 | - int64(streamInfo[17]) - return bitsPerSample, sampleRate, totalSamples -} - -func parseALACSpecificConfig(payload []byte) (int, int, bool) { - if len(payload) < 24 { - return 0, 0, false - } - - bitDepth := int(payload[5]) - sampleRate := int(binary.BigEndian.Uint32(payload[20:24])) - if bitDepth > 0 && sampleRate > 0 { - return bitDepth, sampleRate, true - } - - // Some encoders prepend 4 bytes before the ALACSpecificConfig payload. - if len(payload) >= 28 { - bitDepth = int(payload[9]) - sampleRate = int(binary.BigEndian.Uint32(payload[24:28])) - if bitDepth > 0 && sampleRate > 0 { - return bitDepth, sampleRate, true - } - } - - return 0, 0, false -} - -type atomHeader struct { - offset int64 - size int64 - headerSize int64 - typ string -} - -func readAtomHeaderAt(f *os.File, offset, fileSize int64) (atomHeader, error) { - if offset+8 > fileSize { - return atomHeader{}, io.ErrUnexpectedEOF - } - - headerBuf := make([]byte, 8) - if _, err := f.ReadAt(headerBuf, offset); err != nil { - return atomHeader{}, err - } - - size32 := binary.BigEndian.Uint32(headerBuf[0:4]) - typ := string(headerBuf[4:8]) - - if size32 == 1 { - if offset+16 > fileSize { - return atomHeader{}, io.ErrUnexpectedEOF - } - extBuf := make([]byte, 8) - if _, err := f.ReadAt(extBuf, offset+8); err != nil { - return atomHeader{}, err - } - size64 := binary.BigEndian.Uint64(extBuf) - return atomHeader{offset: offset, size: int64(size64), headerSize: 16, typ: typ}, nil - } - - return atomHeader{offset: offset, size: int64(size32), headerSize: 8, typ: typ}, nil -} - -func findAtomInRange(f *os.File, start, size int64, target string, fileSize int64) (atomHeader, bool, error) { - if size <= 0 { - return atomHeader{}, false, nil - } - - end := start + size - pos := start - - for pos+8 <= end { - header, err := readAtomHeaderAt(f, pos, fileSize) - if err != nil { - return atomHeader{}, false, err - } - - atomSize := header.size - if atomSize == 0 { - atomSize = end - pos - } - - if atomSize < header.headerSize { - return atomHeader{}, false, fmt.Errorf("invalid atom size for %s", header.typ) - } - - header.size = atomSize - if header.typ == target { - return header, true, nil - } - - pos += atomSize - } - - return atomHeader{}, false, nil -} - -func findAudioSampleEntry(f *os.File, start, end, fileSize int64) (int64, string, error) { - const chunkSize = 64 * 1024 - patterns := [][]byte{ - []byte("mp4a"), - []byte("alac"), - []byte("fLaC"), - []byte("ec-3"), - []byte("ac-3"), - []byte("ac-4"), - []byte("Opus"), - } - - var tail []byte - readPos := start - - for readPos < end { - toRead := end - readPos - if toRead > chunkSize { - toRead = chunkSize - } - - buf := make([]byte, toRead) - n, err := f.ReadAt(buf, readPos) - if err != nil && err != io.EOF { - return 0, "", fmt.Errorf("failed to read M4A atom data: %w", err) - } - if n == 0 { - break - } - - data := append(tail, buf[:n]...) - bestIdx := -1 - bestType := "" - for _, pattern := range patterns { - idx := bytes.Index(data, pattern) - if idx >= 0 && (bestIdx < 0 || idx < bestIdx) { - bestIdx = idx - bestType = string(pattern) - } - } - - if bestIdx >= 0 { - absolute := readPos - int64(len(tail)) + int64(bestIdx) - if absolute+32 > fileSize { - return 0, "", fmt.Errorf("audio info not found in M4A file") - } - return absolute, bestType, nil - } - - if len(data) >= 3 { - tail = append([]byte{}, data[len(data)-3:]...) - } else { - tail = append([]byte{}, data...) - } - - readPos += int64(n) - } - - return 0, "", fmt.Errorf("audio info not found in M4A file") -} diff --git a/go_backend/metadata_m4a_quality_test.go b/go_backend/metadata_m4a_quality_test.go deleted file mode 100644 index 11068b68..00000000 --- a/go_backend/metadata_m4a_quality_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package gobackend - -import ( - "encoding/binary" - "encoding/json" - "os" - "path/filepath" - "testing" -) - -func TestM4AAudioSampleEntryQuality(t *testing.T) { - for _, codec := range []string{"mp4a", "alac", "fLaC", "ec-3", "ac-3", "ac-4", "Opus"} { - t.Run(codec, func(t *testing.T) { - mvhd := make([]byte, 100) - binary.BigEndian.PutUint32(mvhd[12:16], 1000) - binary.BigEndian.PutUint32(mvhd[16:20], 10000) - sample := make([]byte, 28) - binary.BigEndian.PutUint16(sample[6:8], 1) - binary.BigEndian.PutUint16(sample[16:18], 2) - binary.BigEndian.PutUint16(sample[18:20], 16) - binary.BigEndian.PutUint32(sample[24:28], 48000<<16) - stsd := append([]byte{0, 0, 0, 0, 0, 0, 0, 1}, buildM4AAtom(codec, sample)...) - trak := buildM4AAtom("trak", buildM4AAtom("mdia", buildM4AAtom("minf", buildM4AAtom("stbl", buildM4AAtom("stsd", stsd))))) - moov := buildM4AAtom("moov", append(buildM4AAtom("mvhd", mvhd), trak...)) - data := append(moov, buildM4AAtom("mdat", make([]byte, 400000))...) - path := filepath.Join(t.TempDir(), "descriptor") - if err := os.WriteFile(path, data, 0600); err != nil { - t.Fatal(err) - } - quality, err := GetM4AQuality(path) - if err != nil { - t.Fatal(err) - } - if quality.Codec != normalizeM4AAudioCodec(codec) || quality.SampleRate != 48000 || quality.Duration != 10 || quality.Bitrate < 320 { - t.Fatalf("missing audio quality: %+v", quality) - } - if codec == "Opus" && quality.BitDepth != 0 { - t.Fatalf("lossy Opus must not expose the sample-entry bit depth: %+v", quality) - } - // The full reader and lightweight Library scanner must both expose - // bitrate, including the extensionless paths used by SAF. - for _, read := range []func(string, string) (string, error){ReadFileMetadataWithHint, ReadAudioMetadataWithDisplayName} { - payload, err := read(path, "Song.m4a") - var metadata map[string]any - if err != nil || json.Unmarshal([]byte(payload), &metadata) != nil { - t.Fatalf("metadata=%s, error=%v", payload, err) - } - if bitrate, ok := metadata["bitrate"].(float64); !ok || bitrate < 320 { - t.Fatalf("missing bitrate: %s", payload) - } - } - }) - } -} - -func TestParseALACSpecificConfigStandardPayload(t *testing.T) { - payload := make([]byte, 24) - payload[5] = 24 - payload[20] = 0x00 - payload[21] = 0x00 - payload[22] = 0xac - payload[23] = 0x44 - - bitDepth, sampleRate, ok := parseALACSpecificConfig(payload) - if !ok { - t.Fatal("expected standard ALAC payload to parse") - } - if bitDepth != 24 { - t.Fatalf("bitDepth = %d, want 24", bitDepth) - } - if sampleRate != 44100 { - t.Fatalf("sampleRate = %d, want 44100", sampleRate) - } -} - -func TestParseALACSpecificConfigPayloadWithLeadingFourBytes(t *testing.T) { - payload := make([]byte, 28) - payload[9] = 16 - payload[24] = 0x00 - payload[25] = 0x00 - payload[26] = 0xbb - payload[27] = 0x80 - - bitDepth, sampleRate, ok := parseALACSpecificConfig(payload) - if !ok { - t.Fatal("expected offset ALAC payload to parse") - } - if bitDepth != 16 { - t.Fatalf("bitDepth = %d, want 16", bitDepth) - } - if sampleRate != 48000 { - t.Fatalf("sampleRate = %d, want 48000", sampleRate) - } -} - -func TestParseALACSpecificConfigRejectsShortPayload(t *testing.T) { - if _, _, ok := parseALACSpecificConfig(make([]byte, 12)); ok { - t.Fatal("expected short ALAC payload to be rejected") - } -} - -func TestM4ACodecFormatMapping(t *testing.T) { - cases := map[string]string{ - "mp4a": "aac", - "alac": "alac", - "fLaC": "flac", - "ec-3": "eac3", - "ac-3": "ac3", - "ac-4": "ac4", - "Opus": "opus", - } - for atomType, want := range cases { - if got := normalizeM4AAudioCodec(atomType); got != want { - t.Fatalf("normalizeM4AAudioCodec(%q) = %q, want %q", atomType, got, want) - } - } - - if got := libraryFormatForM4ACodec("flac"); got != "flac" { - t.Fatalf("libraryFormatForM4ACodec(flac) = %q", got) - } - if got := libraryFormatForM4ACodec("eac3"); got != "eac3" { - t.Fatalf("libraryFormatForM4ACodec(eac3) = %q", got) - } - if got := libraryFormatForM4ACodec("Opus"); got != "opus" { - t.Fatalf("libraryFormatForM4ACodec(Opus) = %q", got) - } - if got := libraryFormatForM4ACodec("aac"); got != "m4a" { - t.Fatalf("libraryFormatForM4ACodec(aac) = %q", got) - } -} - -func TestParseMP4FLACSpecificConfig(t *testing.T) { - streamInfo := make([]byte, 34) - sampleRate := 48000 - bitsPerSample := 24 - totalSamples := int64(48000 * 180) - streamInfo[10] = byte(sampleRate >> 12) - streamInfo[11] = byte(sampleRate >> 4) - streamInfo[12] = byte((sampleRate&0x0F)<<4 | ((bitsPerSample-1)>>4)&0x01) - streamInfo[13] = byte(((bitsPerSample-1)&0x0F)<<4 | int((totalSamples>>32)&0x0F)) - streamInfo[14] = byte(totalSamples >> 24) - streamInfo[15] = byte(totalSamples >> 16) - streamInfo[16] = byte(totalSamples >> 8) - streamInfo[17] = byte(totalSamples) - - payload := append([]byte{0, 0, 0, 0, 0, 0, 0, 34}, streamInfo...) - bitDepth, parsedRate, parsedSamples, ok := parseMP4FLACSpecificConfig(payload) - if !ok { - t.Fatal("expected MP4 FLAC config to parse") - } - if bitDepth != bitsPerSample || parsedRate != sampleRate || parsedSamples != totalSamples { - t.Fatalf("FLAC config = %d/%d/%d", bitDepth, parsedRate, parsedSamples) - } -} diff --git a/go_backend/metadata_types.go b/go_backend/metadata_types.go deleted file mode 100644 index 91940b61..00000000 --- a/go_backend/metadata_types.go +++ /dev/null @@ -1,151 +0,0 @@ -package gobackend - -import "time" - -type cacheEntry struct { - data any - expiresAt time.Time -} - -func (e *cacheEntry) isExpired() bool { - return time.Now().After(e.expiresAt) -} - -type TrackMetadata struct { - SpotifyID string `json:"spotify_id,omitempty"` - Artists string `json:"artists"` - Name string `json:"name"` - AlbumName string `json:"album_name"` - AlbumArtist string `json:"album_artist,omitempty"` - DurationMS int `json:"duration_ms"` - Images string `json:"images"` - ReleaseDate string `json:"release_date"` - TrackNumber int `json:"track_number"` - TotalTracks int `json:"total_tracks,omitempty"` - DiscNumber int `json:"disc_number,omitempty"` - TotalDiscs int `json:"total_discs,omitempty"` - ExternalURL string `json:"external_urls"` - ISRC string `json:"isrc"` - AlbumID string `json:"album_id,omitempty"` - ArtistID string `json:"artist_id,omitempty"` - AlbumType string `json:"album_type,omitempty"` - Composer string `json:"composer,omitempty"` - Explicit bool `json:"explicit,omitempty"` -} - -type AlbumTrackMetadata struct { - SpotifyID string `json:"spotify_id,omitempty"` - Artists string `json:"artists"` - Name string `json:"name"` - AlbumName string `json:"album_name"` - AlbumArtist string `json:"album_artist,omitempty"` - DurationMS int `json:"duration_ms"` - Images string `json:"images"` - ReleaseDate string `json:"release_date"` - TrackNumber int `json:"track_number"` - TotalTracks int `json:"total_tracks,omitempty"` - DiscNumber int `json:"disc_number,omitempty"` - TotalDiscs int `json:"total_discs,omitempty"` - ExternalURL string `json:"external_urls"` - ISRC string `json:"isrc"` - AlbumID string `json:"album_id,omitempty"` - AlbumURL string `json:"album_url,omitempty"` - AlbumType string `json:"album_type,omitempty"` - Composer string `json:"composer,omitempty"` - Explicit bool `json:"explicit,omitempty"` -} - -type AlbumInfoMetadata struct { - TotalTracks int `json:"total_tracks"` - Name string `json:"name"` - ReleaseDate string `json:"release_date"` - Artists string `json:"artists"` - ArtistId string `json:"artist_id,omitempty"` - Images string `json:"images"` - Genre string `json:"genre,omitempty"` - Label string `json:"label,omitempty"` - Copyright string `json:"copyright,omitempty"` -} - -type AlbumResponsePayload struct { - AlbumInfo AlbumInfoMetadata `json:"album_info"` - TrackList []AlbumTrackMetadata `json:"track_list"` -} - -type PlaylistInfoMetadata struct { - Name string `json:"name,omitempty"` - Images string `json:"images,omitempty"` - Tracks struct { - Total int `json:"total"` - } `json:"tracks"` - Owner struct { - DisplayName string `json:"display_name"` - Name string `json:"name"` - Images string `json:"images"` - } `json:"owner"` -} - -type PlaylistResponsePayload struct { - PlaylistInfo PlaylistInfoMetadata `json:"playlist_info"` - TrackList []AlbumTrackMetadata `json:"track_list"` -} - -type ArtistInfoMetadata struct { - ID string `json:"id"` - Name string `json:"name"` - Images string `json:"images"` - Followers int `json:"followers"` - Popularity int `json:"popularity"` -} - -type ArtistAlbumMetadata struct { - ID string `json:"id"` - Name string `json:"name"` - ReleaseDate string `json:"release_date"` - TotalTracks int `json:"total_tracks"` - Images string `json:"images"` - AlbumType string `json:"album_type"` - Artists string `json:"artists"` -} - -type ArtistResponsePayload struct { - ArtistInfo ArtistInfoMetadata `json:"artist_info"` - Albums []ArtistAlbumMetadata `json:"albums"` -} - -type TrackResponse struct { - Track TrackMetadata `json:"track"` -} - -type SearchArtistResult struct { - ID string `json:"id"` - Name string `json:"name"` - Images string `json:"images"` - Followers int `json:"followers"` - Popularity int `json:"popularity"` -} - -type SearchAlbumResult struct { - ID string `json:"id"` - Name string `json:"name"` - Artists string `json:"artists"` - Images string `json:"images"` - ReleaseDate string `json:"release_date"` - TotalTracks int `json:"total_tracks"` - AlbumType string `json:"album_type"` -} - -type SearchPlaylistResult struct { - ID string `json:"id"` - Name string `json:"name"` - Owner string `json:"owner"` - Images string `json:"images"` - TotalTracks int `json:"total_tracks"` -} - -type SearchAllResult struct { - Tracks []TrackMetadata `json:"tracks"` - Artists []SearchArtistResult `json:"artists"` - Albums []SearchAlbumResult `json:"albums"` - Playlists []SearchPlaylistResult `json:"playlists"` -} diff --git a/go_backend/misc_coverage_supplement_test.go b/go_backend/misc_coverage_supplement_test.go deleted file mode 100644 index 0531e7c4..00000000 --- a/go_backend/misc_coverage_supplement_test.go +++ /dev/null @@ -1,308 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "net/url" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/dop251/goja" - "github.com/go-flac/flacvorbis/v2" -) - -func TestReadFileMetadataAndCueLibraryWrappers(t *testing.T) { - dir := t.TempDir() - mp3Path := filepath.Join(dir, "tagged.mp3") - tag := buildID3v23Tag( - id3TextFrame("TIT2", "Title"), - id3TextFrame("TPE1", "Artist"), - id3TextFrame("TALB", "Album"), - id3TextFrame("TRCK", "4/12"), - id3CommentFrame("USLT", "[00:00.00]Lyric"), - ) - if err := os.WriteFile(mp3Path, append(tag, []byte{0xFF, 0xFB, 0x90, 0x64, 0, 0, 0, 0}...), 0600); err != nil { - t.Fatal(err) - } - if jsonText, err := ReadFileMetadata(mp3Path); err != nil || !strings.Contains(jsonText, `"title":"Title"`) { - t.Fatalf("ReadFileMetadata mp3 = %q/%v", jsonText, err) - } - - m4aPath := filepath.Join(dir, "tagged.m4a") - ilst := buildM4ATextTag("\xa9nam", "M4A Title") - if err := os.WriteFile(m4aPath, buildM4AFileWithIlst(ilst, true), 0600); err != nil { - t.Fatal(err) - } - if jsonText, err := ReadFileMetadata(m4aPath); err != nil || !strings.Contains(jsonText, "M4A Title") { - t.Fatalf("ReadFileMetadata m4a = %q/%v", jsonText, err) - } - - cuePath, _ := writeExportCueFixture(t, dir) - results, err := ScanCueFileForLibrary(cuePath, time.Now().Format(time.RFC3339)) - if err != nil || len(results) != 1 || results[0].TrackName != "Song" { - t.Fatalf("ScanCueFileForLibrary = %#v/%v", results, err) - } - if _, err := ReadFileMetadata(filepath.Join(dir, "unsupported.txt")); err == nil { - t.Fatal("expected unsupported metadata format") - } -} - -func TestOutputFDFilePathBranches(t *testing.T) { - dir := t.TempDir() - outputPath := filepath.Join(dir, "out.bin") - file, err := openOutputForWrite(outputPath, 0) - if err != nil { - t.Fatalf("openOutputForWrite path: %v", err) - } - if _, err := file.Write([]byte("data")); err != nil { - t.Fatalf("write output: %v", err) - } - if err := file.Close(); err != nil { - t.Fatalf("close output: %v", err) - } - if !isFDOutput(1) || isFDOutput(0) { - t.Fatal("isFDOutput mismatch") - } - closeOwnedOutputFD(0) - fdSource, err := os.OpenFile(outputPath, os.O_RDWR, 0600) - if err != nil { - t.Fatalf("open fd source: %v", err) - } - dupFD, err := dupOutputFD(int(fdSource.Fd())) - if err != nil { - _ = fdSource.Close() - t.Fatalf("duplicate output fd: %v", err) - } - if err := prepareDupFDForWrite(dupFD, int(fdSource.Fd())); err != nil { - _ = fdSource.Close() - t.Fatalf("prepareDupFDForWrite: %v", err) - } - closeOwnedOutputFD(dupFD) - if err := fdSource.Close(); err != nil { - t.Fatalf("close fd source: %v", err) - } - cleanupOutputOnError(outputPath, 0) - if _, err := os.Stat(outputPath); !os.IsNotExist(err) { - t.Fatalf("cleanup should remove output path, stat err=%v", err) - } - cleanupOutputOnError("", 0) - cleanupOutputOnError("/proc/self/fd/1", 0) - cleanupOutputOnError(filepath.Join(dir, "kept.bin"), 10) -} - -func TestMoreSmallConstructorsRuntimeAndMetadataHelpers(t *testing.T) { - if cfg := DefaultRetryConfig(); cfg.MaxRetries == 0 || cfg.BackoffFactor <= 1 { - t.Fatalf("DefaultRetryConfig = %#v", cfg) - } - if NewAppleMusicClient().httpClient == nil || NewNeteaseClient().httpClient == nil || NewMusixmatchClient().httpClient == nil || NewQQMusicClient().httpClient == nil { - t.Fatal("expected lyric provider HTTP clients") - } - vm := goja.New() - runtime := &extensionRuntime{extensionID: "misc-runtime", vm: vm, settings: map[string]any{}} - if parseExtensionTimeoutSeconds(" 42 ") != 42 || parseExtensionTimeoutSeconds("bad") != 0 || parseExtensionTimeoutSeconds(float64(7)) != 7 { - t.Fatal("parseExtensionTimeoutSeconds mismatch") - } - if (&RedirectBlockedError{Domain: "blocked.example"}).Error() == "" || (&RedirectBlockedError{IsPrivate: true}).Error() == "" { - t.Fatal("RedirectBlockedError Error mismatch") - } - runtime.SetSettings(map[string]any{"quality": "lossless"}) - if runtime.settings["quality"] != "lossless" { - t.Fatal("SetSettings mismatch") - } - jar, _ := newSimpleCookieJar() - cookieURL, _ := url.Parse("https://example.test/") - jar.SetCookies(cookieURL, []*http.Cookie{{Name: "a", Value: "b"}}) - if cookies := jar.Cookies(cookieURL); len(cookies) != 1 || cookies[0].Value != "b" { - t.Fatalf("cookies = %#v", cookies) - } - - if result := runtime.ffmpegExecute(goja.FunctionCall{}).Export().(map[string]any); result["success"] != false { - t.Fatalf("ffmpegExecute missing args = %#v", result) - } - if result := runtime.ffmpegGetInfo(goja.FunctionCall{}).Export().(map[string]any); result["success"] != false { - t.Fatalf("ffmpegGetInfo missing args = %#v", result) - } - if result := runtime.ffmpegGetInfo(goja.FunctionCall{Arguments: []goja.Value{vm.ToValue("missing.flac")}}).Export().(map[string]any); result["success"] != false { - t.Fatalf("ffmpegGetInfo missing file = %#v", result) - } - if result := runtime.ffmpegConvert(goja.FunctionCall{}).Export().(map[string]any); result["success"] != false { - t.Fatalf("ffmpegConvert missing args = %#v", result) - } - - cmt := flacvorbis.New() - setComment(cmt, "TITLE", "Song") - setComment(cmt, "ARTIST", "Artist") - if getComment(cmt, "TITLE") != "Song" || getJoinedComment(cmt, "ARTIST") != "Artist" { - t.Fatalf("comments = %#v", cmt.Comments) - } - setOrClearComment(cmt, "TITLE", "") - if getComment(cmt, "TITLE") != "" { - t.Fatal("setOrClearComment should remove empty value") - } - setOrClearArtistComments(cmt, "ARTIST", "A; B", artistTagModeSplitVorbis) - if joined := getJoinedComment(cmt, "ARTIST"); !strings.Contains(joined, "A") || !strings.Contains(joined, "B") { - t.Fatalf("split artist comments = %q", joined) - } - removeCommentKey(cmt, "ARTIST") - if getComment(cmt, "ARTIST") != "" { - t.Fatal("removeCommentKey failed") - } - if fileExists(filepath.Join(t.TempDir(), "missing")) { - t.Fatal("missing file should not exist") - } - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("cover")) - })) - defer server.Close() - SetAllowPrivateNetwork(true) - defer SetAllowPrivateNetwork(false) - SetNetworkCompatibilityOptions(true, false) - defer SetNetworkCompatibilityOptions(false, false) - coverPath := filepath.Join(t.TempDir(), "cover.jpg") - if err := DownloadCoverToFile(server.URL+"/cover.jpg", coverPath, false); err != nil { - t.Fatalf("DownloadCoverToFile: %v", err) - } - if string(mustReadFile(t, coverPath)) != "cover" { - t.Fatal("downloaded cover mismatch") - } -} - -func TestExtensionHealthInitializeVMAndCustomSearchWrappers(t *testing.T) { - dir := t.TempDir() - extDir := filepath.Join(dir, "ext") - if err := os.MkdirAll(extDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(extDir, "index.js"), []byte(testExtensionJS), 0600); err != nil { - t.Fatal(err) - } - ext := &loadedExtension{ - ID: "health-ext", - Manifest: &ExtensionManifest{ - Name: "health-ext", - DisplayName: "Health", - Version: "1.0.0", - Description: "Health extension", - Types: []ExtensionType{ExtensionTypeMetadataProvider}, - SearchBehavior: &SearchBehaviorConfig{ - Enabled: true, - Primary: true, - }, - ServiceHealth: []ExtensionHealthCheck{{ - ID: "bad", - URL: "http://health.example.test/status", - Required: true, - }}, - }, - Enabled: true, - SourceDir: extDir, - DataDir: filepath.Join(dir, "data"), - } - manager := getExtensionManager() - manager.mu.Lock() - if manager.extensions == nil { - manager.extensions = map[string]*loadedExtension{} - } - manager.extensions[ext.ID] = ext - manager.mu.Unlock() - defer func() { - manager.mu.Lock() - delete(manager.extensions, ext.ID) - manager.mu.Unlock() - }() - - if err := manager.initializeVM(ext); err != nil { - t.Fatalf("initializeVM: %v", err) - } - if ext.VM == nil { - t.Fatal("expected initialized VM") - } - provider := &extensionProviderWrapper{extension: ext} - if tracks, err := provider.CustomSearch("needle", map[string]any{"type": "track"}); err != nil || len(tracks) == 0 { - t.Fatalf("CustomSearch = %#v/%v", tracks, err) - } - downloadCancels.mu.Lock() - delete(downloadCancels.entries, "custom-item-unique") - downloadCancels.mu.Unlock() - if tracks, err := provider.customSearch("needle", nil, "custom-item-unique", ""); err != nil || len(tracks) == 0 { - t.Fatalf("customSearch (item ID) = %#v/%v", tracks, err) - } - if healthJSON, err := CheckExtensionHealthJSON(ext.ID); err != nil || !strings.Contains(healthJSON, `"status":"offline"`) { - t.Fatalf("CheckExtensionHealthJSON = %q/%v", healthJSON, err) - } - teardownVMLocked(ext) -} - -func TestManifestPerfMatchingAndTitleHelpers(t *testing.T) { - manifest := &ExtensionManifest{ - Name: "misc-ext", - DisplayName: "Misc", - Version: "1.0.0", - Description: "Misc extension", - Types: []ExtensionType{ExtensionTypeMetadataProvider}, - URLHandler: &URLHandlerConfig{Enabled: true, Patterns: []string{"example.test"}}, - PostProcessing: &PostProcessingConfig{Hooks: []PostProcessingHook{{ - ID: "hook", Name: "Hook", - }}}, - } - data, err := manifest.ToJSON() - if err != nil || !strings.Contains(string(data), "misc-ext") { - t.Fatalf("ToJSON = %q/%v", string(data), err) - } - if !manifest.HasURLHandler() || !manifest.MatchesURL("https://example.test/track") || len(manifest.GetPostProcessingHooks()) != 1 { - t.Fatal("manifest helpers mismatch") - } - if (&ManifestValidationError{Field: "name", Message: "required"}).Error() == "" { - t.Fatal("manifest validation error string empty") - } - - if extensionDurationMs(1500*time.Microsecond) != 1.5 { - t.Fatal("extensionDurationMs mismatch") - } - vm := goja.New() - value := vm.ToValue(map[string]any{"tracks": []any{1, 2, 3}}) - if countExtensionTopLevelItems(vm, value) != 3 { - t.Fatal("countExtensionTopLevelItems mismatch") - } - if countExtensionTopLevelItems(vm, goja.Undefined()) != 0 { - t.Fatal("empty top-level item count mismatch") - } - - if calculateStringSimilarity("", "") != 1 || calculateStringSimilarity("", "x") != 0 || levenshteinDistance("kitten", "sitting") != 3 { - t.Fatal("string similarity helpers mismatch") - } - var b strings.Builder - writeNormalizedArtistRune(&b, 'ß') - writeNormalizedArtistRune(&b, 'æ') - if b.String() != "ssae" { - t.Fatalf("writeNormalizedArtistRune = %q", b.String()) - } - if !artistsMatch("Artist feat Guest", "Guest") || !sameWordsUnordered("B A", "A B") || !titlesMatch("Song (Remastered)", "Song") { - t.Fatal("artist/title matching mismatch") - } - if len(splitArtists("A & B, C x D")) != 4 { - t.Fatal("splitArtists mismatch") - } - if isLatinScript("東京") || !isLatinScript("Beyonce") { - t.Fatal("isLatinScript mismatch") - } - - req := DownloadRequest{TrackName: "Song", ArtistName: "Artist", DurationMS: 180000} - if !trackMatchesRequest(req, resolvedTrackInfo{Title: "Song", ArtistName: "Artist", Duration: 181}, "test") { - t.Fatal("expected matching track") - } - if trackMatchesRequest(req, resolvedTrackInfo{Title: "Other", ArtistName: "Other", Duration: 240}, "test") { - t.Fatal("expected mismatching track") - } - - var decoded map[string]any - if err := json.Unmarshal(data, &decoded); err != nil || decoded["name"] != "misc-ext" { - t.Fatalf("manifest JSON decode = %#v/%v", decoded, err) - } -} diff --git a/go_backend/mobile_deps.go b/go_backend/mobile_deps.go deleted file mode 100644 index 57aaaeec..00000000 --- a/go_backend/mobile_deps.go +++ /dev/null @@ -1,8 +0,0 @@ -// This file ensures gomobile dependencies are not removed by go mod tidy. -// These packages are required by gomobile bind but not directly imported in code. - -package gobackend - -import ( - _ "golang.org/x/mobile/bind" -) diff --git a/go_backend/mp3_id3_write.go b/go_backend/mp3_id3_write.go deleted file mode 100644 index 5b398abc..00000000 --- a/go_backend/mp3_id3_write.go +++ /dev/null @@ -1,438 +0,0 @@ -package gobackend - -import ( - "bytes" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" -) - -// Native ID3v2 editor for MP3 files. Unlike the ffmpeg remux path, this -// rewrites only the tag block and preserves every foreign frame it does not -// control (POPM ratings, SYLT synced lyrics, CHAP/CTOC chapters, custom TXXX -// from other taggers). The audio bytes are streamed through untouched, and the -// whole file is published via temp+fsync+rename so a crash never corrupts it. - -// id3RawFrame is a frame preserved verbatim (payload already de-unsynced and -// stripped of grouping/data-length prefixes, so it re-serializes with flags 0). -type id3RawFrame struct { - id string - payload []byte -} - -// id3v22FrameIDs maps ID3v2.2 3-char IDs to their v2.4 equivalents. -var id3v22FrameIDs = map[string]string{ - "TT2": "TIT2", "TP1": "TPE1", "TP2": "TPE2", "TAL": "TALB", - "TYE": "TDRC", "TCO": "TCON", "TRK": "TRCK", "TPA": "TPOS", - "TCM": "TCOM", "TPB": "TPUB", "TCR": "TCOP", "TXX": "TXXX", - "ULT": "USLT", "COM": "COMM", -} - -// readMP3ID3v2Frames parses the leading ID3v2 tag of an MP3 into raw frames -// and returns the offset where the audio data begins. A file without an ID3v2 -// tag yields no frames and audioStart 0. Compressed/encrypted frames cannot be -// re-serialized safely and are dropped. -func readMP3ID3v2Frames(f *os.File) (frames []id3RawFrame, audioStart int64, err error) { - header := make([]byte, 10) - n, err := f.ReadAt(header, 0) - if err != nil && err != io.EOF { - return nil, 0, err - } - if n < 10 || string(header[0:3]) != "ID3" { - return nil, 0, nil - } - - majorVersion := header[3] - flags := header[5] - tagUnsync := (flags & 0x80) != 0 - extendedHeader := (flags & 0x40) != 0 - footerPresent := (flags & 0x10) != 0 - - size := synchsafeDecode(header[6:10]) - if size <= 0 { - return nil, 0, fmt.Errorf("invalid ID3v2 tag size") - } - audioStart = int64(10 + size) - if footerPresent { - audioStart += 10 - } - - tagData := make([]byte, size) - if _, err := f.ReadAt(tagData, 10); err != nil { - return nil, 0, fmt.Errorf("truncated ID3v2 tag: %w", err) - } - if footerPresent && len(tagData) >= 10 { - footerStart := len(tagData) - 10 - if string(tagData[footerStart:footerStart+3]) == "3DI" { - tagData = tagData[:footerStart] - } - } - if extendedHeader { - if skip := extendedHeaderSize(tagData, majorVersion); skip > 0 && skip < len(tagData) { - tagData = tagData[skip:] - } - } - - if majorVersion == 2 { - return parseID3v22RawFrames(tagData, tagUnsync), audioStart, nil - } - return parseID3v2xRawFrames(tagData, majorVersion, tagUnsync), audioStart, nil -} - -func parseID3v22RawFrames(data []byte, tagUnsync bool) []id3RawFrame { - var frames []id3RawFrame - pos := 0 - for pos+6 < len(data) { - id := string(data[pos : pos+3]) - if id[0] == 0 { - break - } - size := int(data[pos+3])<<16 | int(data[pos+4])<<8 | int(data[pos+5]) - if size <= 0 || pos+6+size > len(data) { - break - } - payload := data[pos+6 : pos+6+size] - if tagUnsync { - payload = removeUnsync(payload) - } - pos += 6 + size - - if id == "PIC" { - if converted := convertPICToAPIC(payload); converted != nil { - frames = append(frames, id3RawFrame{id: "APIC", payload: converted}) - } - continue - } - mapped, ok := id3v22FrameIDs[id] - if !ok { - continue // no safe v2.4 equivalent - } - frames = append(frames, id3RawFrame{id: mapped, payload: append([]byte(nil), payload...)}) - } - return frames -} - -// convertPICToAPIC rewrites a v2.2 PIC payload (3-char image format) into the -// v2.3+ APIC layout (null-terminated MIME string). Returns nil when malformed. -func convertPICToAPIC(payload []byte) []byte { - if len(payload) < 5 { - return nil - } - mime := "image/jpeg" - if strings.EqualFold(string(payload[1:4]), "PNG") { - mime = "image/png" - } - out := []byte{payload[0]} - out = append(out, []byte(mime)...) - out = append(out, 0x00) - out = append(out, payload[4:]...) // picture type + description + data - return out -} - -func parseID3v2xRawFrames(data []byte, version byte, tagUnsync bool) []id3RawFrame { - var frames []id3RawFrame - pos := 0 - for pos+10 < len(data) { - id := string(data[pos : pos+4]) - if id[0] == 0 { - break - } - var size int - if version == 4 { - size = synchsafeDecode(data[pos+4 : pos+8]) - } else { - size = int(data[pos+4])<<24 | int(data[pos+5])<<16 | int(data[pos+6])<<8 | int(data[pos+7]) - } - if size <= 0 || pos+10+size > len(data) { - break - } - payload := data[pos+10 : pos+10+size] - formatFlags := data[pos+9] - pos += 10 + size - - if version == 3 { - if formatFlags&(0x80|0x40) != 0 { // compression | encryption - continue - } - if formatFlags&0x20 != 0 { // grouping - if len(payload) < 1 { - continue - } - payload = payload[1:] - } - if tagUnsync { - payload = removeUnsync(payload) - } - } else { - if formatFlags&(0x08|0x04) != 0 { // compression | encryption - continue - } - if formatFlags&0x40 != 0 { // grouping - if len(payload) < 1 { - continue - } - payload = payload[1:] - } - if formatFlags&0x01 != 0 { // data length indicator - if len(payload) < 4 { - continue - } - payload = payload[4:] - } - if formatFlags&0x02 != 0 || tagUnsync { // frame unsync - payload = removeUnsync(payload) - } - } - frames = append(frames, id3RawFrame{id: id, payload: append([]byte(nil), payload...)}) - } - return frames -} - -func id3TextPayload(value string) []byte { - return append([]byte{0x03}, []byte(value)...) // UTF-8 -} - -func id3LangTextPayload(text string) []byte { - payload := []byte{0x03} - payload = append(payload, []byte("eng")...) - payload = append(payload, 0x00) // empty description - payload = append(payload, []byte(text)...) - return payload -} - -func id3TXXXPayload(desc, value string) []byte { - payload := []byte{0x03} - payload = append(payload, []byte(desc)...) - payload = append(payload, 0x00) - payload = append(payload, []byte(value)...) - return payload -} - -// firstFrameText returns the decoded text of the first frame with the given ID. -func firstFrameText(frames []id3RawFrame, id string) string { - for _, fr := range frames { - if fr.id == id { - return firstTextValue(extractTextFrame(fr.payload)) - } - } - return "" -} - -// EditMP3Fields updates only the ID3v2 frames whose keys are explicitly present -// in the fields map (set-or-clear semantics, mirroring EditFlacFields) while -// preserving every other frame byte-for-byte. The tag is rewritten as ID3v2.4. -func EditMP3Fields(filePath string, fields map[string]string) error { - f, err := os.Open(filePath) - if err != nil { - return err - } - frames, audioStart, err := readMP3ID3v2Frames(f) - if err != nil { - f.Close() - return err - } - - drop := map[string]bool{} - var added []id3RawFrame - - setOrClear := func(frameID, value string, aliases ...string) { - drop[frameID] = true - for _, alias := range aliases { - drop[alias] = true - } - if strings.TrimSpace(value) != "" { - added = append(added, id3RawFrame{id: frameID, payload: id3TextPayload(value)}) - } - } - - simpleKeys := []struct { - fieldKey string - frameID string - aliases []string - }{ - {"title", "TIT2", nil}, - {"artist", "TPE1", nil}, - {"album", "TALB", nil}, - {"album_artist", "TPE2", nil}, - {"date", "TDRC", []string{"TYER", "TDAT", "TIME"}}, - {"genre", "TCON", nil}, - {"label", "TPUB", nil}, - {"copyright", "TCOP", nil}, - {"composer", "TCOM", nil}, - {"isrc", "TSRC", nil}, - } - for _, key := range simpleKeys { - if v, ok := fields[key.fieldKey]; ok { - setOrClear(key.frameID, v, key.aliases...) - } - } - - if v, ok := fields["comment"]; ok { - drop["COMM"] = true - if strings.TrimSpace(v) != "" { - added = append(added, id3RawFrame{id: "COMM", payload: id3LangTextPayload(v)}) - } - } - if v, ok := fields["lyrics"]; ok { - drop["USLT"] = true // synced SYLT frames are intentionally preserved - if strings.TrimSpace(v) != "" { - added = append(added, id3RawFrame{id: "USLT", payload: id3LangTextPayload(v)}) - } - } - - // Track/disc numbers: merge with the current value when only one half is - // edited, mirroring the FLAC editor's semantics. - if hasMapKey(fields, "track_number") || hasMapKey(fields, "track_total") { - num, total := parseIndexPair(firstFrameText(frames, "TRCK")) - if v, ok := fields["track_number"]; ok { - num = parsePositiveInt(v) - } - if v, ok := fields["track_total"]; ok { - total = parsePositiveInt(v) - } - setOrClear("TRCK", formatIndexValue(num, total)) - } - if hasMapKey(fields, "disc_number") || hasMapKey(fields, "disc_total") { - num, total := parseIndexPair(firstFrameText(frames, "TPOS")) - if v, ok := fields["disc_number"]; ok { - num = parsePositiveInt(v) - } - if v, ok := fields["disc_total"]; ok { - total = parsePositiveInt(v) - } - setOrClear("TPOS", formatIndexValue(num, total)) - } - - // ReplayGain lives in TXXX frames matched by description; only the edited - // descriptions are dropped so foreign TXXX frames survive. - dropTXXXDesc := map[string]bool{} - for _, key := range []string{ - "replaygain_track_gain", "replaygain_track_peak", - "replaygain_album_gain", "replaygain_album_peak", - } { - if v, ok := fields[key]; ok { - desc := strings.ToUpper(key) - dropTXXXDesc[desc] = true - if strings.TrimSpace(v) != "" { - added = append(added, id3RawFrame{id: "TXXX", payload: id3TXXXPayload(desc, v)}) - } - } - } - - // Release identity (advisory/type/barcode) also lives in TXXX frames so - // any tagger can read it back; compilation uses the iTunes TCMP frame. - txxxDescriptions := map[string]string{ - "explicit": "ITUNESADVISORY", - "album_type": "RELEASETYPE", - "upc": "BARCODE", - } - for fieldKey, desc := range txxxDescriptions { - if v, ok := fields[fieldKey]; ok { - dropTXXXDesc[desc] = true - if strings.TrimSpace(v) != "" { - added = append(added, id3RawFrame{id: "TXXX", payload: id3TXXXPayload(desc, v)}) - } - } - } - if v, ok := fields["compilation"]; ok { - setOrClear("TCMP", v) - } - - coverPath := strings.TrimSpace(fields["cover_path"]) - if coverPath != "" { - if coverData, err := os.ReadFile(coverPath); err == nil && len(coverData) > 0 { - drop["APIC"] = true - mime := http.DetectContentType(coverData) - payload := []byte{0x03} - payload = append(payload, []byte(mime)...) - payload = append(payload, 0x00, 0x03, 0x00) // front cover, no description - payload = append(payload, coverData...) - added = append(added, id3RawFrame{id: "APIC", payload: payload}) - } - } - - var kept []id3RawFrame - for _, fr := range frames { - if drop[fr.id] { - continue - } - if fr.id == "TXXX" && len(dropTXXXDesc) > 0 { - desc, _ := extractUserTextFrame(fr.payload) - if dropTXXXDesc[strings.ToUpper(strings.TrimSpace(desc))] { - continue - } - } - kept = append(kept, fr) - } - kept = append(kept, added...) - - tag := serializeID3v24Tag(kept) - err = spliceFileAtomic(filePath, f, tag, audioStart) - f.Close() // harmless double close when the splice already closed it - return err -} - -func serializeID3v24Tag(frames []id3RawFrame) []byte { - var body bytes.Buffer - for _, fr := range frames { - body.WriteString(fr.id) - body.Write(synchsafeEncode(len(fr.payload))) - body.Write([]byte{0, 0}) - body.Write(fr.payload) - } - const padding = 512 - var out bytes.Buffer - out.WriteString("ID3") - out.Write([]byte{0x04, 0x00, 0x00}) - out.Write(synchsafeEncode(body.Len() + padding)) - out.Write(body.Bytes()) - out.Write(make([]byte, padding)) - return out.Bytes() -} - -// spliceFileAtomic writes head followed by src's bytes from tailStart onward to -// a sibling temp, fsyncs, and renames over filePath. On success src (the open -// original) is closed before the rename, which Windows requires. -func spliceFileAtomic(filePath string, src *os.File, head []byte, tailStart int64) error { - tmpPath := filePath + ".tag.partial" - os.Remove(tmpPath) - tmp, err := os.Create(tmpPath) - if err != nil { - return err - } - cleanup := func() { - tmp.Close() - os.Remove(tmpPath) - } - if _, err := tmp.Write(head); err != nil { - cleanup() - return err - } - if _, err := src.Seek(tailStart, io.SeekStart); err != nil { - cleanup() - return err - } - if _, err := io.Copy(tmp, src); err != nil { - cleanup() - return err - } - if err := tmp.Sync(); err != nil { - cleanup() - return err - } - if err := tmp.Close(); err != nil { - os.Remove(tmpPath) - return err - } - // Release the source handle before replacing the file (required on Windows). - src.Close() - if err := os.Rename(tmpPath, filePath); err != nil { - os.Remove(tmpPath) - return err - } - syncDir(filepath.Dir(filePath)) - return nil -} diff --git a/go_backend/native_tag_editors_test.go b/go_backend/native_tag_editors_test.go deleted file mode 100644 index f9a1c34f..00000000 --- a/go_backend/native_tag_editors_test.go +++ /dev/null @@ -1,492 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/binary" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/go-flac/flacvorbis/v2" -) - -// --- MP3 ------------------------------------------------------------------- - -func writeTestMP3(t *testing.T, dir string, frames ...[]byte) (string, []byte) { - t.Helper() - audio := []byte("FAKEMP3AUDIODATA-0123456789") - tag := buildID3v23Tag(frames...) - path := filepath.Join(dir, "test.mp3") - if err := os.WriteFile(path, append(append([]byte{}, tag...), audio...), 0o644); err != nil { - t.Fatalf("write mp3: %v", err) - } - return path, audio -} - -func TestEditMP3FieldsPreservesForeignFrames(t *testing.T) { - dir := t.TempDir() - popm := id3v23Frame("POPM", append([]byte("rater@example.com\x00"), 0xff)) - foreignTXXX := id3UserTextFrame("TXXX", "MusicBrainz Id", "abc-123") - rgTXXX := id3UserTextFrame("TXXX", "REPLAYGAIN_TRACK_GAIN", "-1.00 dB") - path, audio := writeTestMP3(t, dir, - id3TextFrame("TIT2", "Old Title"), - id3TextFrame("TRCK", "3"), - popm, foreignTXXX, rgTXXX, - ) - - err := EditMP3Fields(path, map[string]string{ - "title": "New Title", - "track_total": "12", - "replaygain_track_gain": "-2.50 dB", - }) - if err != nil { - t.Fatalf("EditMP3Fields: %v", err) - } - - meta, err := ReadID3Tags(path) - if err != nil { - t.Fatalf("ReadID3Tags: %v", err) - } - if meta.Title != "New Title" { - t.Errorf("title = %q, want New Title", meta.Title) - } - if meta.TrackNumber != 3 || meta.TotalTracks != 12 { - t.Errorf("track = %d/%d, want 3/12 (merge with existing number)", meta.TrackNumber, meta.TotalTracks) - } - if meta.ReplayGainTrackGain != "-2.50 dB" { - t.Errorf("replaygain = %q, want -2.50 dB", meta.ReplayGainTrackGain) - } - - raw := mustReadFile(t, path) - if !bytes.Contains(raw, []byte("POPM")) || !bytes.Contains(raw, []byte("rater@example.com")) { - t.Error("foreign POPM frame was dropped") - } - if !bytes.Contains(raw, []byte("MusicBrainz Id")) { - t.Error("foreign TXXX frame was dropped") - } - if bytes.Count(raw, []byte("REPLAYGAIN_TRACK_GAIN")) != 1 { - t.Error("edited TXXX description duplicated or missing") - } - if !bytes.HasSuffix(raw, audio) { - t.Error("audio bytes were modified") - } -} - -func TestEditMP3FieldsClearsAndWithoutTag(t *testing.T) { - dir := t.TempDir() - path, audio := writeTestMP3(t, dir, - id3TextFrame("TIT2", "Old"), - id3CommentFrame("COMM", "remove me"), - ) - if err := EditMP3Fields(path, map[string]string{"comment": ""}); err != nil { - t.Fatalf("clear comment: %v", err) - } - raw := mustReadFile(t, path) - if bytes.Contains(raw, []byte("remove me")) { - t.Error("cleared comment still present") - } - if meta, _ := ReadID3Tags(path); meta == nil || meta.Title != "Old" { - t.Error("untouched title lost") - } - - // A bare MP3 with no ID3v2 header gets a fresh tag prepended. - bare := filepath.Join(dir, "bare.mp3") - if err := os.WriteFile(bare, audio, 0o644); err != nil { - t.Fatal(err) - } - if err := EditMP3Fields(bare, map[string]string{"title": "Fresh"}); err != nil { - t.Fatalf("EditMP3Fields on bare file: %v", err) - } - if meta, err := ReadID3Tags(bare); err != nil || meta.Title != "Fresh" { - t.Errorf("bare file title = %v (err %v), want Fresh", meta, err) - } - if !bytes.HasSuffix(mustReadFile(t, bare), audio) { - t.Error("bare file audio modified") - } -} - -func TestEditMP3FieldsWritesReleaseIdentityTags(t *testing.T) { - dir := t.TempDir() - path, audio := writeTestMP3(t, dir, id3TextFrame("TIT2", "Song")) - - if err := EditMP3Fields(path, map[string]string{ - "explicit": "1", - "album_type": "compilation", - "upc": "0012345678901", - "compilation": "1", - }); err != nil { - t.Fatalf("EditMP3Fields: %v", err) - } - - raw := mustReadFile(t, path) - for desc := range map[string]string{ - "ITUNESADVISORY": "1", - "RELEASETYPE": "compilation", - "BARCODE": "0012345678901", - } { - if !bytes.Contains(raw, []byte(desc)) { - t.Errorf("missing TXXX description %s", desc) - } - } - if !bytes.Contains(raw, []byte("TCMP")) { - t.Error("missing TCMP compilation frame") - } - if !bytes.HasSuffix(raw, audio) { - t.Error("audio bytes were modified") - } - - // Clearing removes the tags without touching the rest. - if err := EditMP3Fields(path, map[string]string{ - "explicit": "", - "album_type": "", - "upc": "", - "compilation": "", - }); err != nil { - t.Fatalf("clear release tags: %v", err) - } - raw = mustReadFile(t, path) - for _, desc := range []string{"ITUNESADVISORY", "RELEASETYPE", "BARCODE", "TCMP"} { - if bytes.Contains(raw, []byte(desc)) { - t.Errorf("cleared %s still present", desc) - } - } - if !bytes.Contains(raw, []byte("TIT2")) { - t.Error("untouched title lost") - } -} - -// --- M4A ------------------------------------------------------------------- - -// buildTestM4A assembles ftyp + moov(trak stub with stco + udta>meta>ilst) + mdat -// with moov BEFORE mdat so chunk-offset shifting is exercised. -func buildTestM4A(t *testing.T, ilstChildren []byte, mdatPayload []byte) ([]byte, int64) { - t.Helper() - ftyp := buildM4AAtom("ftyp", append([]byte("M4A "), make([]byte, 8)...)) - - hdlrPayload := make([]byte, 25) - copy(hdlrPayload[8:12], "mdir") - copy(hdlrPayload[12:16], "appl") - meta := buildM4AAtom("meta", append(append([]byte{0, 0, 0, 0}, buildM4AAtom("hdlr", hdlrPayload)...), buildM4AAtom("ilst", ilstChildren)...)) - udta := buildM4AAtom("udta", meta) - - // stco with one entry; the real value is patched below once mdat's - // position is known. - stcoPayload := make([]byte, 4+4+4) - binary.BigEndian.PutUint32(stcoPayload[4:8], 1) - stco := buildM4AAtom("stco", stcoPayload) - stbl := buildM4AAtom("stbl", stco) - minf := buildM4AAtom("minf", stbl) - mdia := buildM4AAtom("mdia", minf) - trak := buildM4AAtom("trak", mdia) - - moov := buildM4AAtom("moov", append(append([]byte{}, trak...), udta...)) - mdat := buildM4AAtom("mdat", mdatPayload) - - file := append(append(append([]byte{}, ftyp...), moov...), mdat...) - chunkOffset := int64(len(ftyp) + len(moov) + 8) // first byte of mdat payload - idx := bytes.Index(file, []byte("stco")) - if idx < 0 { - t.Fatal("stco not found in synthetic file") - } - binary.BigEndian.PutUint32(file[idx+4+8:idx+4+12], uint32(chunkOffset)) - return file, chunkOffset -} - -func readTestM4ATitle(t *testing.T, data []byte) string { - t.Helper() - loc, ok := locateM4AIlstInBuf(data) - if !ok { - t.Fatal("ilst not found") - } - for pos := loc.ilst.body(); pos+8 <= loc.ilst.end(); { - child, ok := readMP4Box(data, pos) - if !ok { - t.Fatal("malformed ilst") - } - if child.typ == "\xa9nam" { - d, ok := readMP4Box(data, child.body()) - if !ok || d.typ != "data" { - t.Fatal("no data atom in \xa9nam") - } - return string(data[d.body()+8 : d.end()]) - } - pos = child.end() - } - return "" -} - -func TestEditM4AFieldsPreservesAtomsAndShiftsChunkOffsets(t *testing.T) { - dir := t.TempDir() - existing := append([]byte{}, buildM4ATextAtom("\xa9nam", "Old")...) - existing = append(existing, buildM4ATextAtom("\xa9too", "SomeEncoder")...) // foreign, untouched - existing = append(existing, buildM4AFreeformAtom("MusicBrainz Track Id", "xyz")...) - existing = append(existing, buildM4AFreeformAtom("SYNCEDLYRICS", "Old synced lyrics")...) - mdatPayload := []byte("M4ADATA") - file, oldOffset := buildTestM4A(t, existing, mdatPayload) - - path := filepath.Join(dir, "test.m4a") - if err := os.WriteFile(path, file, 0o644); err != nil { - t.Fatal(err) - } - - if err := EditM4AFields(path, map[string]string{ - "title": "A Much Longer Replacement Title", - "isrc": "USABC1234567", - "lyrics": "Updated lyrics", - }); err != nil { - t.Fatalf("EditM4AFields: %v", err) - } - - updated := mustReadFile(t, path) - if got := readTestM4ATitle(t, updated); got != "A Much Longer Replacement Title" { - t.Errorf("title = %q", got) - } - if !bytes.Contains(updated, []byte("SomeEncoder")) { - t.Error("foreign \xa9too atom dropped") - } - if !bytes.Contains(updated, []byte("MusicBrainz Track Id")) { - t.Error("foreign freeform atom dropped") - } - if !bytes.Contains(updated, []byte("USABC1234567")) { - t.Error("ISRC freeform missing") - } - if bytes.Contains(updated, []byte("SYNCEDLYRICS")) || - bytes.Contains(updated, []byte("Old synced lyrics")) { - t.Error("stale SYNCEDLYRICS freeform was not removed") - } - if meta, err := ReadM4ATags(path); err != nil || meta.Lyrics != "Updated lyrics" { - t.Fatalf("updated M4A lyrics = %#v/%v", meta, err) - } - - // stco entry must still point at the mdat payload. - idx := bytes.Index(updated, []byte("stco")) - if idx < 0 { - t.Fatal("stco lost") - } - newOffset := int64(binary.BigEndian.Uint32(updated[idx+4+8 : idx+4+12])) - payloadAt := bytes.Index(updated, mdatPayload) - if newOffset != int64(payloadAt) { - t.Errorf("stco offset %d does not track mdat payload at %d (was %d)", newOffset, payloadAt, oldOffset) - } -} - -func TestEditM4AFieldsWritesReleaseIdentityAtoms(t *testing.T) { - dir := t.TempDir() - file, oldOffset := buildTestM4A(t, nil, []byte("DATA")) - _ = oldOffset - path := filepath.Join(dir, "release.m4a") - if err := os.WriteFile(path, file, 0o644); err != nil { - t.Fatal(err) - } - - if err := EditM4AFields(path, map[string]string{ - "title": "Song", - "explicit": "1", - "compilation": "1", - "album_type": "compilation", - "upc": "0012345678901", - }); err != nil { - t.Fatalf("EditM4AFields: %v", err) - } - - updated := mustReadFile(t, path) - if !bytes.Contains(updated, []byte("rtng")) { - t.Error("rtng advisory atom missing") - } - if !bytes.Contains(updated, []byte("cpil")) { - t.Error("cpil compilation atom missing") - } - if !bytes.Contains(updated, []byte("RELEASETYPE")) { - t.Error("RELEASETYPE freeform atom missing") - } - if !bytes.Contains(updated, []byte("0012345678901")) { - t.Error("BARCODE freeform atom missing") - } - - // Clearing drops the atoms again while the title survives. - if err := EditM4AFields(path, map[string]string{ - "explicit": "", - "compilation": "", - "album_type": "", - "upc": "", - }); err != nil { - t.Fatalf("clear release atoms: %v", err) - } - updated = mustReadFile(t, path) - for _, atom := range []string{"rtng", "cpil", "RELEASETYPE", "BARCODE"} { - if bytes.Contains(updated, []byte(atom)) { - t.Errorf("cleared %s still present", atom) - } - } - if got := readTestM4ATitle(t, updated); got != "Song" { - t.Errorf("title = %q, want Song", got) - } -} - -func TestEditM4AFieldsCreatesMissingChain(t *testing.T) { - dir := t.TempDir() - // moov with only a trak stub — no udta/meta/ilst. - stcoPayload := make([]byte, 12) - binary.BigEndian.PutUint32(stcoPayload[4:8], 1) - trak := buildM4AAtom("trak", buildM4AAtom("mdia", buildM4AAtom("minf", buildM4AAtom("stbl", buildM4AAtom("stco", stcoPayload))))) - moov := buildM4AAtom("moov", trak) - ftyp := buildM4AAtom("ftyp", append([]byte("M4A "), make([]byte, 8)...)) - mdat := buildM4AAtom("mdat", []byte("DATA")) - file := append(append(append([]byte{}, ftyp...), moov...), mdat...) - - path := filepath.Join(dir, "bare.m4a") - if err := os.WriteFile(path, file, 0o644); err != nil { - t.Fatal(err) - } - if err := EditM4AFields(path, map[string]string{"title": "Created"}); err != nil { - t.Fatalf("EditM4AFields: %v", err) - } - if got := readTestM4ATitle(t, mustReadFile(t, path)); got != "Created" { - t.Errorf("title = %q, want Created", got) - } -} - -// --- Ogg/Opus --------------------------------------------------------------- - -func TestApplyVorbisFieldEditsReleaseTags(t *testing.T) { - cmt := flacvorbis.New() - applyVorbisFieldEdits(cmt, map[string]string{ - "explicit": "1", - "album_type": "compilation", - "upc": "0012345678901", - "compilation": "1", - }) - - for key, want := range map[string]string{ - "ITUNESADVISORY": "1", - "RELEASETYPE": "compilation", - "BARCODE": "0012345678901", - "COMPILATION": "1", - } { - if got := getComment(cmt, key); got != want { - t.Errorf("%s = %q, want %q", key, got, want) - } - } - - applyVorbisFieldEdits(cmt, map[string]string{ - "explicit": "", - "album_type": "", - "upc": "", - "compilation": "", - }) - for _, key := range []string{"ITUNESADVISORY", "RELEASETYPE", "BARCODE", "COMPILATION"} { - if got := getComment(cmt, key); got != "" { - t.Errorf("cleared %s = %q, want empty", key, got) - } - } -} - -func buildTestOpus(t *testing.T, path string, comments []string, audioPages int) { - t.Helper() - head := append([]byte("OpusHead"), make([]byte, 11)...) - tags := append([]byte("OpusTags"), serializeVorbisCommentBlock("test-vendor", comments)...) - - f, err := os.Create(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - - bos := oggEditPage{headerType: 0x02, serial: 0xBEEF, seq: 0, segments: []byte{byte(len(head))}, data: head} - if err := bos.serialize(f); err != nil { - t.Fatal(err) - } - for _, p := range paginateOggPackets([][]byte{tags}, 0xBEEF, 1) { - if err := p.serialize(f); err != nil { - t.Fatal(err) - } - } - seq := uint32(2) - for i := 0; i < audioPages; i++ { - audio := bytes.Repeat([]byte{byte(0x40 + i)}, 100) - page := oggEditPage{granule: uint64((i + 1) * 960), serial: 0xBEEF, seq: seq, segments: []byte{100}, data: audio} - if i == audioPages-1 { - page.headerType = 0x04 // EOS - } - if err := page.serialize(f); err != nil { - t.Fatal(err) - } - seq++ - } -} - -func TestEditOggFieldsPreservesForeignComments(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.opus") - buildTestOpus(t, path, []string{"TITLE=Old", "FOO=BAR", "ENCODER=opusenc"}, 3) - - if err := EditOggFields(path, map[string]string{"title": "New Title"}); err != nil { - t.Fatalf("EditOggFields: %v", err) - } - - meta, err := ReadOggVorbisComments(path) - if err != nil { - t.Fatalf("ReadOggVorbisComments: %v", err) - } - if meta.Title != "New Title" { - t.Errorf("title = %q, want New Title", meta.Title) - } - raw := mustReadFile(t, path) - if !bytes.Contains(raw, []byte("FOO=BAR")) || !bytes.Contains(raw, []byte("ENCODER=opusenc")) { - t.Error("foreign comments dropped") - } - - // The stream must reparse with sequential page numbers and valid layout. - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - pages, err := readAllOggEditPages(f) - if err != nil { - t.Fatalf("reparse: %v", err) - } - for i, p := range pages { - if p.seq != uint32(i) { - t.Errorf("page %d has seq %d", i, p.seq) - } - } -} - -func TestEditOggFieldsGrowingCommentRenumbersAudioPages(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "grow.opus") - buildTestOpus(t, path, []string{"TITLE=Old"}, 2) - - // >64KB of lyrics forces the OpusTags packet across multiple pages. - if err := EditOggFields(path, map[string]string{"lyrics": strings.Repeat("la ", 30000)}); err != nil { - t.Fatalf("EditOggFields: %v", err) - } - - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - pages, err := readAllOggEditPages(f) - if err != nil { - t.Fatalf("reparse: %v", err) - } - if len(pages) < 4 { - t.Fatalf("expected multi-page comment, got %d pages", len(pages)) - } - for i, p := range pages { - if p.seq != uint32(i) { - t.Errorf("page %d has seq %d", i, p.seq) - } - } - // Audio payloads intact after renumbering. - last := pages[len(pages)-1] - if last.headerType&0x04 == 0 || !bytes.Equal(last.data, bytes.Repeat([]byte{0x41}, 100)) { - t.Error("audio pages corrupted by renumbering") - } - if meta, err := ReadOggVorbisComments(path); err != nil || meta.Title != "Old" { - t.Errorf("untouched title lost: %v (err %v)", meta, err) - } -} diff --git a/go_backend/ogg_edit.go b/go_backend/ogg_edit.go deleted file mode 100644 index b8f90c43..00000000 --- a/go_backend/ogg_edit.go +++ /dev/null @@ -1,483 +0,0 @@ -package gobackend - -import ( - "bufio" - "encoding/base64" - "encoding/binary" - "errors" - "fmt" - "io" - "math" - "os" - "path/filepath" - "strconv" - "strings" - - flacvorbis "github.com/go-flac/flacvorbis/v2" -) - -// Native Vorbis-comment editor for Ogg Opus/Vorbis files. Rebuilds only the -// comment header packet and repaginates the header pages; audio pages are -// copied verbatim (renumbered + re-CRC'd only when the header page count -// changes). No ffmpeg remux, so foreign comments survive and the container -// layout is untouched. Published via temp+fsync+rename. - -// oggCRCTable implements the Ogg page checksum: CRC-32 with polynomial -// 0x04c11db7, no bit reflection, zero init, zero final xor. -var oggCRCTable = func() [256]uint32 { - var table [256]uint32 - for i := range table { - r := uint32(i) << 24 - for range 8 { - if r&0x80000000 != 0 { - r = (r << 1) ^ 0x04c11db7 - } else { - r <<= 1 - } - } - table[i] = r - } - return table -}() - -func oggCRC(data []byte) uint32 { - var crc uint32 - for _, b := range data { - crc = (crc << 8) ^ oggCRCTable[byte(crc>>24)^b] - } - return crc -} - -type oggEditPage struct { - headerType byte - granule uint64 - serial uint32 - seq uint32 - segments []byte - data []byte -} - -func (p *oggEditPage) serialize(w io.Writer) error { - header := make([]byte, 27) - copy(header[0:4], "OggS") - header[5] = p.headerType - binary.LittleEndian.PutUint64(header[6:14], p.granule) - binary.LittleEndian.PutUint32(header[14:18], p.serial) - binary.LittleEndian.PutUint32(header[18:22], p.seq) - // CRC (bytes 22-26) is computed over the whole page with the field zeroed. - header[26] = byte(len(p.segments)) - - page := make([]byte, 0, len(header)+len(p.segments)+len(p.data)) - page = append(page, header...) - page = append(page, p.segments...) - page = append(page, p.data...) - binary.LittleEndian.PutUint32(page[22:26], oggCRC(page)) - _, err := w.Write(page) - return err -} - -// readNextOggEditPage parses a single page from r. Returns io.EOF at a clean -// page boundary. -func readNextOggEditPage(r *bufio.Reader) (*oggEditPage, error) { - header := make([]byte, 27) - if _, err := io.ReadFull(r, header); err != nil { - if err == io.EOF { - return nil, io.EOF - } - return nil, fmt.Errorf("truncated ogg page header: %w", err) - } - if string(header[0:4]) != "OggS" || header[4] != 0 { - return nil, fmt.Errorf("invalid ogg page") - } - page := &oggEditPage{ - headerType: header[5], - granule: binary.LittleEndian.Uint64(header[6:14]), - serial: binary.LittleEndian.Uint32(header[14:18]), - seq: binary.LittleEndian.Uint32(header[18:22]), - segments: make([]byte, int(header[26])), - } - if _, err := io.ReadFull(r, page.segments); err != nil { - return nil, err - } - size := 0 - for _, s := range page.segments { - size += int(s) - } - page.data = make([]byte, size) - if _, err := io.ReadFull(r, page.data); err != nil { - return nil, err - } - return page, nil -} - -func (p *oggEditPage) byteSize() int64 { - return int64(27 + len(p.segments) + len(p.data)) -} - -// readAllOggEditPages parses the entire file into pages, rejecting multiplexed -// (multi-serial) streams the editor cannot safely rewrite. Test helper; the -// editor itself streams and never holds the whole file. -func readAllOggEditPages(f *os.File) ([]oggEditPage, error) { - if _, err := f.Seek(0, io.SeekStart); err != nil { - return nil, err - } - br := bufio.NewReader(f) - var pages []oggEditPage - for { - page, err := readNextOggEditPage(br) - if err == io.EOF { - if len(pages) > 0 { - return pages, nil - } - return nil, fmt.Errorf("truncated ogg page header: %w", io.EOF) - } - if err != nil { - return nil, err - } - if len(pages) > 0 && page.serial != pages[0].serial { - return nil, fmt.Errorf("multiplexed ogg streams are not supported") - } - pages = append(pages, *page) - } -} - -// assembleOggHeaderPackets extracts the first count packets and verifies the -// last one ends exactly at the end of its page (spec-required for Opus/Vorbis -// headers), returning the packets and the index of that last header page. -func assembleOggHeaderPackets(pages []oggEditPage, count int) ([][]byte, int, error) { - var packets [][]byte - var cur []byte - for pageIdx, page := range pages { - offset := 0 - for segIdx, seg := range page.segments { - cur = append(cur, page.data[offset:offset+int(seg)]...) - offset += int(seg) - if seg < 255 { // packet complete - packets = append(packets, cur) - cur = nil - if len(packets) == count { - if segIdx != len(page.segments)-1 { - return nil, 0, fmt.Errorf("header packet shares a page with audio") - } - return packets, pageIdx, nil - } - } - } - } - return nil, 0, errOggHeaderIncomplete -} - -var errOggHeaderIncomplete = errors.New("incomplete ogg header packets") - -// paginateOggPackets lays consecutive packets into pages (max 255 segments -// per page), setting the continued-packet flag on pages that begin mid-packet. -func paginateOggPackets(packets [][]byte, serial uint32, firstSeq uint32) []oggEditPage { - // Build the full lacing sequence with packet-spanning continuation info. - type lacing struct { - value byte - data []byte - } - var segs []lacing - for _, pkt := range packets { - rest := pkt - for { - n := len(rest) - if n >= 255 { - segs = append(segs, lacing{value: 255, data: rest[:255]}) - rest = rest[255:] - continue - } - segs = append(segs, lacing{value: byte(n), data: rest}) - break - } - } - - var pages []oggEditPage - seq := firstSeq - for start := 0; start < len(segs); { - end := min(start+255, len(segs)) - page := oggEditPage{granule: 0, serial: serial, seq: seq} - if start > 0 && segs[start-1].value == 255 { - page.headerType = 0x01 // continues a packet from the previous page - } - for _, s := range segs[start:end] { - page.segments = append(page.segments, s.value) - page.data = append(page.data, s.data...) - } - pages = append(pages, page) - seq++ - start = end - } - return pages -} - -// parseVorbisCommentBlock decodes vendor + raw comment strings. -func parseVorbisCommentBlock(data []byte) (string, []string, error) { - if len(data) < 4 { - return "", nil, fmt.Errorf("comment block too short") - } - vendorLen := int(binary.LittleEndian.Uint32(data[0:4])) - pos := 4 + vendorLen - if vendorLen < 0 || pos+4 > len(data) { - return "", nil, fmt.Errorf("invalid vendor length") - } - vendor := string(data[4 : 4+vendorLen]) - count := int(binary.LittleEndian.Uint32(data[pos : pos+4])) - pos += 4 - comments := make([]string, 0, count) - for range count { - if pos+4 > len(data) { - return "", nil, fmt.Errorf("truncated comment list") - } - l := int(binary.LittleEndian.Uint32(data[pos : pos+4])) - pos += 4 - if l < 0 || pos+l > len(data) { - return "", nil, fmt.Errorf("truncated comment entry") - } - comments = append(comments, string(data[pos:pos+l])) - pos += l - } - return vendor, comments, nil -} - -func serializeVorbisCommentBlock(vendor string, comments []string) []byte { - size := 4 + len(vendor) + 4 - for _, c := range comments { - size += 4 + len(c) - } - out := make([]byte, 0, size) - var num [4]byte - binary.LittleEndian.PutUint32(num[:], uint32(len(vendor))) - out = append(out, num[:]...) - out = append(out, vendor...) - binary.LittleEndian.PutUint32(num[:], uint32(len(comments))) - out = append(out, num[:]...) - for _, c := range comments { - binary.LittleEndian.PutUint32(num[:], uint32(len(c))) - out = append(out, num[:]...) - out = append(out, c...) - } - return out -} - -// EditOggFields updates only the Vorbis comments whose keys are explicitly -// present in the fields map (same semantics as EditFlacFields) in an Ogg -// Opus or Vorbis file, preserving all other comments and the audio verbatim. -func EditOggFields(filePath string, fields map[string]string) error { - f, err := os.Open(filePath) - if err != nil { - return err - } - // Read only the header pages into memory; audio pages are streamed to the - // temp file later — a full-album Opus file would otherwise sit whole on - // the heap. - br := bufio.NewReaderSize(f, 64<<10) - firstPage, err := readNextOggEditPage(br) - if err != nil { - f.Close() - if err == io.EOF { - return fmt.Errorf("ogg stream too short") - } - return err - } - - // Identify the codec from the first packet (BOS page). - first := firstPage.data - var headerPacketCount int - var isOpus bool - switch { - case len(first) >= 8 && string(first[0:8]) == "OpusHead": - isOpus = true - headerPacketCount = 2 // OpusHead, OpusTags - case len(first) >= 7 && first[0] == 0x01 && string(first[1:7]) == "vorbis": - headerPacketCount = 3 // id, comment, setup - default: - f.Close() - return fmt.Errorf("unsupported ogg codec") - } - - headerPages := []oggEditPage{*firstPage} - headerBytes := firstPage.byteSize() - var packets [][]byte - for { - var assembleErr error - packets, _, assembleErr = assembleOggHeaderPackets(headerPages, headerPacketCount) - if assembleErr == nil { - break - } - if !errors.Is(assembleErr, errOggHeaderIncomplete) { - f.Close() - return assembleErr - } - if len(headerPages) >= 1024 { - f.Close() - return fmt.Errorf("ogg header spans too many pages") - } - page, readErr := readNextOggEditPage(br) - if readErr != nil { - f.Close() - if readErr == io.EOF { - return assembleErr - } - return readErr - } - if page.serial != firstPage.serial { - f.Close() - return fmt.Errorf("multiplexed ogg streams are not supported") - } - headerPages = append(headerPages, *page) - headerBytes += page.byteSize() - } - - // Decode the comment packet. - commentPacket := packets[1] - var commentBody []byte - switch { - case isOpus && len(commentPacket) >= 8 && string(commentPacket[0:8]) == "OpusTags": - commentBody = commentPacket[8:] - case !isOpus && len(commentPacket) >= 7 && commentPacket[0] == 0x03 && string(commentPacket[1:7]) == "vorbis": - commentBody = commentPacket[7:] // trailing framing bit is ignored by the length-driven parser - default: - f.Close() - return fmt.Errorf("comment header not found") - } - - vendor, comments, err := parseVorbisCommentBlock(commentBody) - if err != nil { - f.Close() - return err - } - - cmt := flacvorbis.New() - cmt.Vendor = vendor - cmt.Comments = comments - applyVorbisFieldEdits(cmt, fields) - if isOpus { - if err := applyOpusReplayGainEdits(cmt, fields); err != nil { - f.Close() - return err - } - } - - coverPath := strings.TrimSpace(fields["cover_path"]) - if coverPath != "" && fileExists(coverPath) { - if coverData, err := os.ReadFile(coverPath); err == nil && len(coverData) > 0 { - if picBlock, err := buildPictureBlock("", coverData); err == nil { - removeCommentKey(cmt, "METADATA_BLOCK_PICTURE") - encoded := base64.StdEncoding.EncodeToString(picBlock.Data) - cmt.Comments = append(cmt.Comments, "METADATA_BLOCK_PICTURE="+encoded) - } - } - } - - newBody := serializeVorbisCommentBlock(cmt.Vendor, cmt.Comments) - var newComment []byte - if isOpus { - newComment = append([]byte("OpusTags"), newBody...) - } else { - newComment = append([]byte{0x03}, []byte("vorbis")...) - newComment = append(newComment, newBody...) - newComment = append(newComment, 0x01) // framing bit - } - - headerPackets := [][]byte{newComment} - if !isOpus { - headerPackets = append(headerPackets, packets[2]) // setup header, verbatim - } - newHeaderPages := paginateOggPackets(headerPackets, firstPage.serial, 1) - - tmpPath := filePath + ".tag.partial" - os.Remove(tmpPath) - tmp, err := os.Create(tmpPath) - if err != nil { - f.Close() - return err - } - cleanup := func(err error) error { - tmp.Close() - os.Remove(tmpPath) - f.Close() - return err - } - - if err := firstPage.serialize(tmp); err != nil { - return cleanup(err) - } - for i := range newHeaderPages { - if err := newHeaderPages[i].serialize(tmp); err != nil { - return cleanup(err) - } - } - if len(newHeaderPages)+1 == len(headerPages) { - // Header page count unchanged: audio pages' sequence numbers and CRCs - // are still valid, copy them through verbatim. - if _, err := f.Seek(headerBytes, io.SeekStart); err != nil { - return cleanup(err) - } - if _, err := io.Copy(tmp, f); err != nil { - return cleanup(err) - } - } else { - nextSeq := uint32(1 + len(newHeaderPages)) - for { - page, readErr := readNextOggEditPage(br) - if readErr == io.EOF { - break - } - if readErr != nil { - return cleanup(readErr) - } - if page.serial != firstPage.serial { - return cleanup(fmt.Errorf("multiplexed ogg streams are not supported")) - } - page.seq = nextSeq - nextSeq++ - if err := page.serialize(tmp); err != nil { - return cleanup(err) - } - } - } - - if err := tmp.Sync(); err != nil { - return cleanup(err) - } - if err := tmp.Close(); err != nil { - os.Remove(tmpPath) - f.Close() - return err - } - f.Close() // release before rename (required on Windows) - if err := os.Rename(tmpPath, filePath); err != nil { - os.Remove(tmpPath) - return err - } - syncDir(filepath.Dir(filePath)) - return nil -} - -// Opus uses R128 gain comments relative to -23 LUFS, in signed Q7.8 units. -// The scan already includes OpusHead's output gain; preserve that header and -// store only the additional adjustment. Remove legacy tags for edited scopes -// so players cannot select conflicting gains or peaks (RFC 7845 section 5.2). -func applyOpusReplayGainEdits(cmt *flacvorbis.MetaDataBlockVorbisComment, fields map[string]string) error { - for _, scope := range []string{"track", "album"} { - raw, present := fields["replaygain_"+scope+"_gain"] - if !present { - continue - } - value := "" - if raw = strings.TrimSpace(raw); raw != "" { - db, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimSuffix(raw, "dB")), 64) - q := math.Round((db - 5) * 256) - if err != nil || math.IsNaN(q) || math.IsInf(q, 0) || q < -32768 || q > 32767 { - return fmt.Errorf("invalid Opus %s ReplayGain: %q", scope, raw) - } - value = strconv.Itoa(int(q)) - } - upper := strings.ToUpper(scope) - setOrClearComment(cmt, "R128_"+upper+"_GAIN", value) - removeCommentKey(cmt, "REPLAYGAIN_"+upper+"_GAIN") - removeCommentKey(cmt, "REPLAYGAIN_"+upper+"_PEAK") - } - return nil -} diff --git a/go_backend/output_fd.go b/go_backend/output_fd.go deleted file mode 100644 index 9ec3b135..00000000 --- a/go_backend/output_fd.go +++ /dev/null @@ -1,88 +0,0 @@ -package gobackend - -import ( - "fmt" - "os" - "strings" -) - -func isFDOutput(outputFD int) bool { - return outputFD > 0 -} - -func openOutputForWrite(outputPath string, outputFD int) (*os.File, error) { - if isFDOutput(outputFD) { - // Never hand the original detached FD directly to a provider attempt. - // Fallback chains may retry with another provider after a failure. - // If the first attempt closes the original FD, its numeric ID can be - // reused by unrelated resources and a later close may trigger fdsan abort. - dupFD, err := dupOutputFD(outputFD) - if err != nil { - return nil, fmt.Errorf("failed to duplicate output fd %d: %w", outputFD, err) - } - if err := prepareDupFDForWrite(dupFD, outputFD); err != nil { - _ = closeFD(dupFD) - return nil, err - } - return os.NewFile(uintptr(dupFD), fmt.Sprintf("saf_fd_%d_dup_%d", outputFD, dupFD)), nil - } - - path := strings.TrimSpace(outputPath) - if strings.HasPrefix(path, "/proc/self/fd/") { - // Re-open procfs fd path instead of taking ownership of raw detached fd. - // Some SAF providers reject O_TRUNC on these descriptors with EACCES/EPERM. - file, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0) - if err == nil { - return file, nil - } - if os.IsPermission(err) { - return os.OpenFile(path, os.O_WRONLY, 0) - } - return nil, err - } - - return os.Create(outputPath) -} - -func prepareDupFDForWrite(dupFD, originalFD int) error { - // Best-effort reset so retries start writing from byte 0. - if err := truncateFD(dupFD); err != nil { - if isBestEffortTruncateError(err) { - GoLog("[OutputFD] truncate not supported on fd %d (dup of %d): %v\n", dupFD, originalFD, err) - } else { - return fmt.Errorf("failed to truncate output fd %d (dup of %d): %w", dupFD, originalFD, err) - } - } - if err := seekFDStart(dupFD); err != nil { - GoLog("[OutputFD] seek reset failed on fd %d (dup of %d): %v\n", dupFD, originalFD, err) - } - return nil -} - -func closeOwnedOutputFD(outputFD int) { - if !isFDOutput(outputFD) { - return - } - - if err := closeFD(outputFD); err != nil { - if !isBadFD(err) { - GoLog("[OutputFD] failed to close detached fd %d: %v\n", outputFD, err) - } - return - } - - GoLog("[OutputFD] closed detached fd %d\n", outputFD) -} - -func cleanupOutputOnError(outputPath string, outputFD int) { - if isFDOutput(outputFD) { - return - } - - path := strings.TrimSpace(outputPath) - if path == "" || strings.HasPrefix(path, "/proc/self/fd/") { - return - } - - _ = os.Remove(path) -} diff --git a/go_backend/output_fd_unix.go b/go_backend/output_fd_unix.go deleted file mode 100644 index a9eb76aa..00000000 --- a/go_backend/output_fd_unix.go +++ /dev/null @@ -1,35 +0,0 @@ -//go:build !windows - -package gobackend - -import "syscall" - -func dupOutputFD(fd int) (int, error) { - return syscall.Dup(fd) -} - -func truncateFD(fd int) error { - return syscall.Ftruncate(fd, 0) -} - -func seekFDStart(fd int) error { - _, err := syscall.Seek(fd, 0, 0) - return err -} - -func closeFD(fd int) error { - return syscall.Close(fd) -} - -func isBestEffortTruncateError(err error) bool { - switch err { - case syscall.EPERM, syscall.EACCES, syscall.EINVAL, syscall.ESPIPE, syscall.ENOSYS: - return true - default: - return false - } -} - -func isBadFD(err error) bool { - return err == syscall.EBADF -} diff --git a/go_backend/output_fd_windows.go b/go_backend/output_fd_windows.go deleted file mode 100644 index 2d9ba30b..00000000 --- a/go_backend/output_fd_windows.go +++ /dev/null @@ -1,29 +0,0 @@ -//go:build windows - -package gobackend - -func dupOutputFD(fd int) (int, error) { - // Windows build is primarily for local tooling/tests. - // Android runtime uses the !windows implementation. - return fd, nil -} - -func truncateFD(_ int) error { - return nil -} - -func seekFDStart(_ int) error { - return nil -} - -func closeFD(_ int) error { - return nil -} - -func isBestEffortTruncateError(_ error) bool { - return true -} - -func isBadFD(_ error) bool { - return false -} diff --git a/go_backend/performance_benchmark_test.go b/go_backend/performance_benchmark_test.go deleted file mode 100644 index b45d273f..00000000 --- a/go_backend/performance_benchmark_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "fmt" - "path/filepath" - "testing" - "time" - - "github.com/dop251/goja" -) - -var ( - benchmarkStringSink string - benchmarkIntSink int64 - benchmarkScanSink *LibraryScanResult - benchmarkValueSink goja.Value -) - -func BenchmarkGojaByteArrayConversion(b *testing.B) { - const payloadSize = 64 << 10 - payload := make([]byte, payloadSize) - vm := goja.New() - - b.Run("go_backed_bytes", func(b *testing.B) { - b.ReportAllocs() - b.SetBytes(payloadSize) - for b.Loop() { - benchmarkValueSink = vm.ToValue(payload) - } - }) - - b.Run("boxed_interfaces", func(b *testing.B) { - b.ReportAllocs() - b.SetBytes(payloadSize) - for b.Loop() { - boxed := make([]any, len(payload)) - for index, value := range payload { - boxed[index] = int(value) - } - benchmarkValueSink = vm.ToValue(boxed) - } - }) -} - -func BenchmarkGojaProviderInvocation(b *testing.B) { - vm := goja.New() - if _, err := vm.RunString(`var extension = { searchTracks: function(query, limit) { return query.length + limit; } };`); err != nil { - b.Fatal(err) - } - - b.Run("direct_callable", func(b *testing.B) { - b.ReportAllocs() - for b.Loop() { - value, err := invokeExtensionMethod(vm, "searchTracks", "lossless", 25) - if err != nil { - b.Fatal(err) - } - benchmarkIntSink = value.ToInteger() - } - }) - - b.Run("compile_source", func(b *testing.B) { - b.ReportAllocs() - for b.Loop() { - script := fmt.Sprintf(`extension.searchTracks(%q, %d)`, "lossless", 25) - value, err := vm.RunString(script) - if err != nil { - b.Fatal(err) - } - benchmarkIntSink = value.ToInteger() - } - }) -} - -func BenchmarkCheckFilesExistBatch(b *testing.B) { - const trackCount = 1000 - dir := b.TempDir() - idx := &ISRCIndex{ - index: make(map[string]string, trackCount), - files: make(map[string]isrcFileEntry), - outputDir: dir, - } - tracks := make([]map[string]string, trackCount) - for i := 0; i < trackCount; i++ { - isrc := fmt.Sprintf("USAA%08d", i) - idx.index[isrc] = filepath.Join(dir, fmt.Sprintf("%d.flac", i)) - tracks[i] = map[string]string{ - "isrc": isrc, - "track_name": fmt.Sprintf("Track %d", i), - "artist_name": "Artist", - } - } - idx.buildTime.Store(time.Now().UnixNano()) - isrcIndexCacheMu.Lock() - isrcIndexCache[dir] = idx - isrcIndexCacheMu.Unlock() - b.Cleanup(func() { InvalidateISRCCache(dir) }) - payload, err := json.Marshal(tracks) - if err != nil { - b.Fatal(err) - } - - b.ReportAllocs() - b.SetBytes(int64(len(payload))) - b.ResetTimer() - for b.Loop() { - result, err := CheckFilesExistParallel(dir, string(payload)) - if err != nil { - b.Fatal(err) - } - benchmarkStringSink = result - } -} - -func BenchmarkLibraryScanFLACSinglePass(b *testing.B) { - dir := b.TempDir() - path := filepath.Join(dir, "track.flac") - cover := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3} - writeSinglePassTestFlac(b, path, cover) - cacheDir := filepath.Join(dir, "covers") - - b.ReportAllocs() - b.ResetTimer() - for b.Loop() { - result := &LibraryScanResult{FilePath: path, Format: "flac"} - var err error - benchmarkScanSink, err = scanFLACFileWithCoverCache(path, result, "", cacheDir, "stable-key") - if err != nil { - b.Fatal(err) - } - } -} diff --git a/go_backend/platform_resolver_fallbacks.go b/go_backend/platform_resolver_fallbacks.go deleted file mode 100644 index 12f5920a..00000000 --- a/go_backend/platform_resolver_fallbacks.go +++ /dev/null @@ -1,663 +0,0 @@ -package gobackend - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "golang.org/x/net/html" -) - -const ( - resolverFallbackTimeout = 12 * time.Second - resolverResponseLimit = 2 << 20 - squiglyPageLimit = 8 << 20 -) - -type resolverMetadata struct { - Title string - Artist string -} - -type resolverResult struct { - Links map[string]songLinkPlatformLink - Metadata resolverMetadata -} - -type platformFallbackResolver interface { - Resolve(context.Context, string, resolverMetadata) (resolverResult, error) -} - -type platformResolverChain struct { - songLinkWeb platformFallbackResolver - unitune platformFallbackResolver - musicBrainz platformFallbackResolver - squigly platformFallbackResolver -} - -var defaultPlatformResolverFallbacks platformFallbackResolver = &platformResolverChain{ - songLinkWeb: &songLinkWebResolver{ - client: NewMetadataHTTPClient(6 * time.Second), - rateLimiter: NewRateLimiter(20, time.Minute), - }, - unitune: &unituneResolver{ - client: NewMetadataHTTPClient(6 * time.Second), - rateLimiter: NewRateLimiter(30, time.Minute), - }, - musicBrainz: &musicBrainzPlatformResolver{ - client: NewMetadataHTTPClient(6 * time.Second), - rateLimiter: NewRateLimiter(1, time.Second), - }, - squigly: &squiglyResolver{ - client: NewMetadataHTTPClient(6 * time.Second), - rateLimiter: NewRateLimiter(18, time.Minute), - }, -} - -func (c *platformResolverChain) Resolve( - ctx context.Context, - inputURL string, - hint resolverMetadata, -) (resolverResult, error) { - result := resolverResult{Links: make(map[string]songLinkPlatformLink), Metadata: hint} - var resolverErrors []error - - resolvers := []struct { - name string - resolver platformFallbackResolver - }{ - {name: "Song.link Web", resolver: c.songLinkWeb}, - {name: "UniTune", resolver: c.unitune}, - {name: "MusicBrainz", resolver: c.musicBrainz}, - {name: "Squigly", resolver: c.squigly}, - } - - for _, candidate := range resolvers { - if candidate.resolver == nil { - continue - } - resolved, err := candidate.resolver.Resolve(ctx, inputURL, result.Metadata) - if err != nil { - resolverErrors = append(resolverErrors, fmt.Errorf("%s: %w", candidate.name, err)) - LogDebug("PlatformResolver", "%s resolver failed: %v", candidate.name, err) - continue - } - - mergeResolverLinks(result.Links, resolved.Links) - if result.Metadata.Title == "" { - result.Metadata.Title = strings.TrimSpace(resolved.Metadata.Title) - } - if result.Metadata.Artist == "" { - result.Metadata.Artist = strings.TrimSpace(resolved.Metadata.Artist) - } - LogInfo("PlatformResolver", "%s contributed %d direct platform links", candidate.name, len(resolved.Links)) - - if hasUsefulResolverCoverage(result.Links) { - break - } - } - - addResolverSourceLink(result.Links, inputURL) - if len(result.Links) > 0 { - return result, nil - } - if len(resolverErrors) == 0 { - return resolverResult{}, fmt.Errorf("no additional resolver was available") - } - return resolverResult{}, errors.Join(resolverErrors...) -} - -type songLinkWebResolver struct { - client *http.Client - rateLimiter *RateLimiter -} - -func (r *songLinkWebResolver) Resolve( - ctx context.Context, - inputURL string, - _ resolverMetadata, -) (resolverResult, error) { - platform := resolverPlatformFromURL(inputURL) - if directResolverURL(platform, inputURL) == "" { - return resolverResult{}, fmt.Errorf("unsupported source URL") - } - if err := r.rateLimiter.WaitForSlotContext(ctx); err != nil { - return resolverResult{}, err - } - - endpoint := "https://song.link/" + url.PathEscape(inputURL) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return resolverResult{}, err - } - req.Header.Set("Accept", "text/html,application/xhtml+xml") - req.Header.Set("User-Agent", getRandomUserAgent()) - resp, err := r.client.Do(req) - if err != nil { - return resolverResult{}, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return resolverResult{}, fmt.Errorf("web page returned status %d", resp.StatusCode) - } - if resp.Request == nil || resp.Request.URL == nil || !isSongLinkLandingHost(resp.Request.URL.Hostname()) { - return resolverResult{}, fmt.Errorf("web page redirected to an unexpected host") - } - - body, err := readResolverResponse(resp, squiglyPageLimit) - if err != nil { - return resolverResult{}, err - } - document, err := html.Parse(bytes.NewReader(body)) - if err != nil { - return resolverResult{}, fmt.Errorf("failed to parse web page: %w", err) - } - - result := resolverResult{Links: make(map[string]songLinkPlatformLink)} - var visit func(*html.Node) - visit = func(node *html.Node) { - if node.Type == html.ElementNode && node.Data == "a" { - for _, attr := range node.Attr { - if attr.Key != "href" { - continue - } - platform := resolverPlatformFromURL(attr.Val) - if _, exists := result.Links[platform]; exists { - break - } - if directURL := directResolverURL(platform, attr.Val); directURL != "" { - result.Links[platform] = songLinkPlatformLink{URL: directURL} - } - break - } - } - for child := node.FirstChild; child != nil; child = child.NextSibling { - visit(child) - } - } - visit(document) - addResolverSourceLink(result.Links, inputURL) - if len(result.Links) < 2 { - return resolverResult{}, fmt.Errorf("web page returned no cross-platform links") - } - return result, nil -} - -func isSongLinkLandingHost(host string) bool { - host = strings.ToLower(strings.TrimSpace(host)) - switch host { - case "song.link", "album.link", "artist.link", "odesli.co", "www.odesli.co": - return true - default: - return false - } -} - -func mergeResolverLinks(dst, src map[string]songLinkPlatformLink) { - for platform, link := range src { - if _, exists := dst[platform]; exists { - continue - } - if directURL := directResolverURL(platform, link.URL); directURL != "" { - dst[platform] = songLinkPlatformLink{URL: directURL} - } - } -} - -func hasUsefulResolverCoverage(links map[string]songLinkPlatformLink) bool { - if len(links) < 4 { - return false - } - downloadProviders := 0 - for _, platform := range []string{"deezer", "tidal", "amazonMusic", "qobuz"} { - if link, ok := links[platform]; ok && link.URL != "" { - downloadProviders++ - } - } - return downloadProviders >= 2 -} - -func canonicalResolverPlatform(platform string) string { - normalized := strings.ToLower(strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.TrimSpace(platform))) - switch normalized { - case "spotify", "deezer", "tidal", "qobuz", "soundcloud", "bandcamp": - return normalized - case "apple", "applemusic": - return "appleMusic" - case "amazon", "amazonmusic": - return "amazonMusic" - case "youtube": - return "youtube" - case "youtubemusic": - return "youtubeMusic" - default: - return "" - } -} - -func resolverURLFromPlatformID(platform, entityType, entityID string) (string, error) { - platform = canonicalResolverPlatform(platform) - entityID = strings.TrimSpace(entityID) - if platform == "" || entityID == "" { - return "", fmt.Errorf("invalid platform or entity ID") - } - - entityType = strings.ToLower(strings.TrimSpace(entityType)) - if entityType == "song" { - entityType = "track" - } - if entityType != "track" && entityType != "album" && entityType != "artist" { - return "", fmt.Errorf("unsupported entity type %q", entityType) - } - - id := url.PathEscape(entityID) - switch platform { - case "spotify": - return fmt.Sprintf("https://open.spotify.com/%s/%s", entityType, id), nil - case "deezer": - return fmt.Sprintf("https://www.deezer.com/%s/%s", entityType, id), nil - case "tidal": - return fmt.Sprintf("https://tidal.com/browse/%s/%s", entityType, id), nil - case "qobuz": - return fmt.Sprintf("https://open.qobuz.com/%s/%s", entityType, id), nil - case "amazonMusic": - return fmt.Sprintf("https://music.amazon.com/%ss/%s", entityType, id), nil - case "youtube", "youtubeMusic": - if entityType != "track" { - return "", fmt.Errorf("unsupported %s entity type %q", platform, entityType) - } - host := "www.youtube.com" - if platform == "youtubeMusic" { - host = "music.youtube.com" - } - return fmt.Sprintf("https://%s/watch?v=%s", host, url.QueryEscape(entityID)), nil - default: - return "", fmt.Errorf("cannot build a direct %s URL from an ID", platform) - } -} - -func directResolverURL(platform, value string) string { - platform = canonicalResolverPlatform(platform) - if platform == "" { - return "" - } - parsed, err := url.Parse(strings.TrimSpace(value)) - if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" { - return "" - } - - host := strings.ToLower(parsed.Hostname()) - if strings.Contains(strings.ToLower(parsed.EscapedPath()), "/search") { - return "" - } - - hostAllowed := false - switch platform { - case "spotify": - hostAllowed = host == "open.spotify.com" - case "deezer": - hostAllowed = host == "deezer.com" || host == "www.deezer.com" - case "tidal": - hostAllowed = host == "tidal.com" || host == "www.tidal.com" || host == "listen.tidal.com" - case "qobuz": - hostAllowed = host == "open.qobuz.com" || host == "play.qobuz.com" || host == "www.qobuz.com" - case "appleMusic": - hostAllowed = host == "music.apple.com" || host == "geo.music.apple.com" - case "amazonMusic": - hostAllowed = host == "music.amazon.com" - case "youtubeMusic": - hostAllowed = host == "music.youtube.com" - case "youtube": - hostAllowed = host == "youtube.com" || host == "www.youtube.com" || host == "youtu.be" - case "soundcloud": - hostAllowed = host == "soundcloud.com" || host == "www.soundcloud.com" || host == "m.soundcloud.com" - case "bandcamp": - hostAllowed = host == "bandcamp.com" || strings.HasSuffix(host, ".bandcamp.com") - } - if !hostAllowed { - return "" - } - return parsed.String() -} - -func addResolverSourceLink(links map[string]songLinkPlatformLink, inputURL string) { - platform := resolverPlatformFromURL(inputURL) - if platform == "" { - return - } - if directURL := directResolverURL(platform, inputURL); directURL != "" { - if _, exists := links[platform]; !exists { - links[platform] = songLinkPlatformLink{URL: directURL} - } - } -} - -func readResolverResponse(resp *http.Response, limit int64) ([]byte, error) { - if resp == nil || resp.Body == nil { - return nil, fmt.Errorf("response is empty") - } - body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) - if err != nil { - return nil, err - } - if int64(len(body)) > limit { - return nil, fmt.Errorf("response exceeds %d bytes", limit) - } - if len(body) == 0 { - return nil, fmt.Errorf("response body is empty") - } - return body, nil -} - -type unituneResolver struct { - client *http.Client - rateLimiter *RateLimiter -} - -func (r *unituneResolver) Resolve(ctx context.Context, inputURL string, _ resolverMetadata) (resolverResult, error) { - if err := r.rateLimiter.WaitForSlotContext(ctx); err != nil { - return resolverResult{}, err - } - endpoint := "https://api.unitune.art/v1-alpha.1/links?url=" + url.QueryEscape(inputURL) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return resolverResult{}, err - } - req.Header.Set("User-Agent", getRandomUserAgent()) - resp, err := r.client.Do(req) - if err != nil { - return resolverResult{}, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return resolverResult{}, fmt.Errorf("API returned status %d", resp.StatusCode) - } - body, err := readResolverResponse(resp, resolverResponseLimit) - if err != nil { - return resolverResult{}, err - } - - var payload struct { - EntityUniqueID string `json:"entityUniqueId"` - Links map[string]songLinkPlatformLink `json:"linksByPlatform"` - Entities map[string]struct { - Title string `json:"title"` - ArtistName string `json:"artistName"` - } `json:"entitiesByUniqueId"` - } - if err := json.Unmarshal(body, &payload); err != nil { - return resolverResult{}, err - } - - result := resolverResult{Links: make(map[string]songLinkPlatformLink)} - for platform, link := range payload.Links { - canonical := canonicalResolverPlatform(platform) - if directURL := directResolverURL(canonical, link.URL); directURL != "" { - result.Links[canonical] = songLinkPlatformLink{URL: directURL} - } - } - if entity, ok := payload.Entities[payload.EntityUniqueID]; ok { - result.Metadata = resolverMetadata{Title: entity.Title, Artist: entity.ArtistName} - } else { - for _, entity := range payload.Entities { - result.Metadata = resolverMetadata{Title: entity.Title, Artist: entity.ArtistName} - break - } - } - if len(result.Links) == 0 && result.Metadata.Title == "" { - return resolverResult{}, fmt.Errorf("API returned no direct platform links or metadata") - } - return result, nil -} - -type musicBrainzPlatformResolver struct { - client *http.Client - rateLimiter *RateLimiter -} - -func (r *musicBrainzPlatformResolver) Resolve( - ctx context.Context, - _ string, - hint resolverMetadata, -) (resolverResult, error) { - title := strings.TrimSpace(hint.Title) - artist := strings.TrimSpace(hint.Artist) - if title == "" || artist == "" { - return resolverResult{}, fmt.Errorf("title and artist metadata are required") - } - - query := fmt.Sprintf("recording:\"%s\" AND artist:\"%s\"", escapeMusicBrainzQuery(title), escapeMusicBrainzQuery(artist)) - searchURL := musicBrainzAPIBase + "/recording?fmt=json&limit=5&query=" + url.QueryEscape(query) - var search struct { - Recordings []struct { - ID string `json:"id"` - Score int `json:"score"` - Title string `json:"title"` - ArtistCredit []struct { - Name string `json:"name"` - } `json:"artist-credit"` - } `json:"recordings"` - } - if err := r.getJSON(ctx, searchURL, &search); err != nil { - return resolverResult{}, err - } - - var recordingID string - for _, candidate := range search.Recordings { - candidateArtist := "" - if len(candidate.ArtistCredit) > 0 { - candidateArtist = candidate.ArtistCredit[0].Name - } - if candidate.Score < 90 || normalizeLooseTitle(candidate.Title) != normalizeLooseTitle(title) || !artistsMatch(artist, candidateArtist) { - continue - } - recordingID = candidate.ID - break - } - if recordingID == "" { - return resolverResult{}, fmt.Errorf("no verified recording match") - } - - lookupURL := fmt.Sprintf("%s/recording/%s?fmt=json&inc=url-rels+isrcs", musicBrainzAPIBase, url.PathEscape(recordingID)) - var recording struct { - Relations []struct { - URL struct { - Resource string `json:"resource"` - } `json:"url"` - } `json:"relations"` - } - if err := r.getJSON(ctx, lookupURL, &recording); err != nil { - return resolverResult{}, err - } - - result := resolverResult{Links: make(map[string]songLinkPlatformLink), Metadata: hint} - for _, relation := range recording.Relations { - platform := resolverPlatformFromURL(relation.URL.Resource) - if directURL := directResolverURL(platform, relation.URL.Resource); directURL != "" { - if _, exists := result.Links[platform]; !exists { - result.Links[platform] = songLinkPlatformLink{URL: directURL} - } - } - } - if len(result.Links) == 0 { - return resolverResult{}, fmt.Errorf("recording has no supported platform relations") - } - return result, nil -} - -func (r *musicBrainzPlatformResolver) getJSON(ctx context.Context, endpoint string, payload any) error { - if err := r.rateLimiter.WaitForSlotContext(ctx); err != nil { - return err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return err - } - req.Header.Set("User-Agent", getRandomUserAgent()) - resp, err := r.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("API returned status %d", resp.StatusCode) - } - body, err := readResolverResponse(resp, resolverResponseLimit) - if err != nil { - return err - } - return json.Unmarshal(body, payload) -} - -func escapeMusicBrainzQuery(value string) string { - value = strings.ReplaceAll(value, `\`, `\\`) - return strings.ReplaceAll(value, `"`, `\"`) -} - -func resolverPlatformFromURL(value string) string { - parsed, err := url.Parse(strings.TrimSpace(value)) - if err != nil { - return "" - } - host := strings.ToLower(parsed.Hostname()) - switch { - case host == "open.spotify.com": - return "spotify" - case host == "deezer.com" || host == "www.deezer.com": - return "deezer" - case host == "tidal.com" || host == "www.tidal.com" || host == "listen.tidal.com": - return "tidal" - case host == "music.apple.com" || host == "geo.music.apple.com": - return "appleMusic" - case host == "music.amazon.com": - return "amazonMusic" - case host == "music.youtube.com": - return "youtubeMusic" - case host == "youtube.com" || host == "www.youtube.com" || host == "youtu.be": - return "youtube" - case host == "soundcloud.com" || host == "www.soundcloud.com" || host == "m.soundcloud.com": - return "soundcloud" - case host == "open.qobuz.com" || host == "play.qobuz.com" || host == "www.qobuz.com": - return "qobuz" - case host == "bandcamp.com" || strings.HasSuffix(host, ".bandcamp.com"): - return "bandcamp" - default: - return "" - } -} - -type squiglyResolver struct { - client *http.Client - rateLimiter *RateLimiter -} - -func (r *squiglyResolver) Resolve(ctx context.Context, inputURL string, _ resolverMetadata) (resolverResult, error) { - if err := r.rateLimiter.WaitForSlotContext(ctx); err != nil { - return resolverResult{}, err - } - requestBody, err := json.Marshal(map[string]string{"url": inputURL}) - if err != nil { - return resolverResult{}, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://squigly.link/api/create", bytes.NewReader(requestBody)) - if err != nil { - return resolverResult{}, err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", getRandomUserAgent()) - resp, err := r.client.Do(req) - if err != nil { - return resolverResult{}, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { - return resolverResult{}, fmt.Errorf("create endpoint returned status %d", resp.StatusCode) - } - body, err := readResolverResponse(resp, resolverResponseLimit) - if err != nil { - return resolverResult{}, err - } - var created struct { - FullURL string `json:"full_url"` - Title string `json:"title"` - Artist string `json:"artist"` - } - if err := json.Unmarshal(body, &created); err != nil { - return resolverResult{}, err - } - parsedPageURL, err := url.Parse(strings.TrimSpace(created.FullURL)) - if err != nil || parsedPageURL.Scheme != "https" || parsedPageURL.Hostname() != "squigly.link" { - return resolverResult{}, fmt.Errorf("create endpoint returned an invalid page URL") - } - - pageReq, err := http.NewRequestWithContext(ctx, http.MethodGet, parsedPageURL.String(), nil) - if err != nil { - return resolverResult{}, err - } - pageReq.Header.Set("User-Agent", getRandomUserAgent()) - pageResp, err := r.client.Do(pageReq) - if err != nil { - return resolverResult{}, err - } - defer pageResp.Body.Close() - if pageResp.StatusCode != http.StatusOK { - return resolverResult{}, fmt.Errorf("result page returned status %d", pageResp.StatusCode) - } - pageBody, err := readResolverResponse(pageResp, squiglyPageLimit) - if err != nil { - return resolverResult{}, err - } - - const marker = "window.__SQUIGLY_LINK__ =" - markerIndex := bytes.Index(pageBody, []byte(marker)) - if markerIndex < 0 { - return resolverResult{}, fmt.Errorf("result page contains no resolver payload") - } - decoder := json.NewDecoder(bytes.NewReader(pageBody[markerIndex+len(marker):])) - var embedded struct { - Data struct { - Title string `json:"title"` - Artist string `json:"artist"` - Services map[string]*struct { - URL string `json:"url"` - } `json:"services"` - } `json:"data"` - } - if err := decoder.Decode(&embedded); err != nil { - return resolverResult{}, fmt.Errorf("failed to decode result page: %w", err) - } - - result := resolverResult{ - Links: make(map[string]songLinkPlatformLink), - Metadata: resolverMetadata{ - Title: embedded.Data.Title, - Artist: embedded.Data.Artist, - }, - } - if result.Metadata.Title == "" { - result.Metadata.Title = created.Title - } - if result.Metadata.Artist == "" { - result.Metadata.Artist = created.Artist - } - for platform, service := range embedded.Data.Services { - if service == nil { - continue - } - canonical := canonicalResolverPlatform(platform) - if directURL := directResolverURL(canonical, service.URL); directURL != "" { - result.Links[canonical] = songLinkPlatformLink{URL: directURL} - } - } - if len(result.Links) == 0 { - return resolverResult{}, fmt.Errorf("result page returned no direct platform links") - } - return result, nil -} diff --git a/go_backend/platform_resolver_fallbacks_test.go b/go_backend/platform_resolver_fallbacks_test.go deleted file mode 100644 index 73f98ac2..00000000 --- a/go_backend/platform_resolver_fallbacks_test.go +++ /dev/null @@ -1,245 +0,0 @@ -package gobackend - -import ( - "context" - "io" - "net/http" - "strings" - "testing" - "time" -) - -func resolverTestResponse(req *http.Request, status int, body string) *http.Response { - return &http.Response{ - StatusCode: status, - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader(body)), - Request: req, - } -} - -func TestSongLinkWebResolverParsesServerRenderedLinks(t *testing.T) { - resolver := &songLinkWebResolver{ - client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - if req.URL.Host != "song.link" || !strings.Contains(req.URL.EscapedPath(), "https:%2F%2Fopen.spotify.com") { - t.Fatalf("unexpected Song.link web request: %s", req.URL.String()) - } - return resolverTestResponse(req, http.StatusOK, `<html><body> - <a href="https://open.spotify.com/track/source">Spotify</a> - <a href="https://www.deezer.com/track/101">Deezer</a> - <a href="https://listen.tidal.com/track/202">Tidal</a> - <a href="https://music.amazon.com/tracks/TESTASIN?ref=x&tag=y">Amazon</a> - <a href="https://evil.example/track/ignored">Untrusted</a> - </body></html>`), nil - })}, - rateLimiter: NewRateLimiter(100, time.Minute), - } - - result, err := resolver.Resolve(context.Background(), "https://open.spotify.com/track/source", resolverMetadata{}) - if err != nil { - t.Fatalf("Resolve() error = %v", err) - } - if len(result.Links) != 4 || result.Links["deezer"].URL == "" || result.Links["amazonMusic"].URL == "" { - t.Fatalf("Song.link web links = %#v", result.Links) - } - if strings.Contains(result.Links["amazonMusic"].URL, "&") { - t.Fatalf("HTML entity was not decoded: %s", result.Links["amazonMusic"].URL) - } -} - -func TestUnituneResolverKeepsOnlyDirectTrustedLinks(t *testing.T) { - resolver := &unituneResolver{ - client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - if req.URL.Host != "api.unitune.art" || req.URL.Query().Get("url") == "" { - t.Fatalf("unexpected UniTune request: %s", req.URL.String()) - } - return resolverTestResponse(req, http.StatusOK, `{ - "entityUniqueId":"SPOTIFY::TRACK::source", - "entitiesByUniqueId":{"SPOTIFY::TRACK::source":{"title":"Track","artistName":"Artist"}}, - "linksByPlatform":{ - "spotify":{"url":"https://open.spotify.com/track/source"}, - "deezer":{"url":"https://www.deezer.com/track/123"}, - "tidal":{"url":"https://listen.tidal.com/search?q=Track"}, - "appleMusic":{"url":"https://music.apple.com/search?term=Track"}, - "amazonMusic":{"url":"https://evil.example/track/123"} - } - }`), nil - })}, - rateLimiter: NewRateLimiter(100, time.Minute), - } - - result, err := resolver.Resolve(context.Background(), "https://open.spotify.com/track/source", resolverMetadata{}) - if err != nil { - t.Fatalf("Resolve() error = %v", err) - } - if len(result.Links) != 2 || result.Links["deezer"].URL == "" || result.Links["spotify"].URL == "" { - t.Fatalf("direct links = %#v, want only Spotify and Deezer", result.Links) - } - if result.Metadata.Title != "Track" || result.Metadata.Artist != "Artist" { - t.Fatalf("metadata = %+v", result.Metadata) - } -} - -func TestMusicBrainzResolverAcceptsVerifiedProviderRelations(t *testing.T) { - resolver := &musicBrainzPlatformResolver{ - client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch req.URL.Path { - case "/ws/2/recording": - return resolverTestResponse(req, http.StatusOK, `{ - "recordings":[ - {"id":"wrong","score":100,"title":"Live Version","artist-credit":[{"name":"Artist"}]}, - {"id":"match","score":100,"title":"Track","artist-credit":[{"name":"Artist"}]} - ] - }`), nil - case "/ws/2/recording/match": - return resolverTestResponse(req, http.StatusOK, `{ - "relations":[ - {"url":{"resource":"https://open.spotify.com/track/spotify-id"}}, - {"url":{"resource":"https://tidal.com/browse/track/123"}}, - {"url":{"resource":"https://evil.example/track/not-allowed"}} - ] - }`), nil - default: - t.Fatalf("unexpected MusicBrainz request: %s", req.URL.String()) - return nil, nil - } - })}, - rateLimiter: NewRateLimiter(100, time.Minute), - } - - result, err := resolver.Resolve( - context.Background(), - "https://example.invalid/source", - resolverMetadata{Title: "Track", Artist: "Artist"}, - ) - if err != nil { - t.Fatalf("Resolve() error = %v", err) - } - if len(result.Links) != 2 || result.Links["spotify"].URL == "" || result.Links["tidal"].URL == "" { - t.Fatalf("MusicBrainz links = %#v", result.Links) - } -} - -func TestSquiglyResolverParsesEmbeddedPayload(t *testing.T) { - resolver := &squiglyResolver{ - client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - switch { - case req.Method == http.MethodPost && req.URL.Path == "/api/create": - return resolverTestResponse(req, http.StatusCreated, `{ - "full_url":"https://squigly.link/song/artist/track", - "title":"Track", - "artist":"Artist" - }`), nil - case req.Method == http.MethodGet && req.URL.Path == "/song/artist/track": - return resolverTestResponse(req, http.StatusOK, `<html><script> - window.__SQUIGLY_LINK__ = {"data":{"title":"Track","artist":"Artist","services":{ - "spotify":{"url":"https://open.spotify.com/track/spotify-id"}, - "apple":{"url":"https://music.apple.com/us/album/track/1?i=2"}, - "tidal":{"url":"https://tidal.com/browse/track/3"}, - "bandcamp":null - }}}; - </script></html>`), nil - default: - t.Fatalf("unexpected Squigly request: %s %s", req.Method, req.URL.String()) - return nil, nil - } - })}, - rateLimiter: NewRateLimiter(100, time.Minute), - } - - result, err := resolver.Resolve(context.Background(), "https://open.spotify.com/track/source", resolverMetadata{}) - if err != nil { - t.Fatalf("Resolve() error = %v", err) - } - if len(result.Links) != 3 || result.Links["appleMusic"].URL == "" || result.Links["tidal"].URL == "" { - t.Fatalf("Squigly links = %#v", result.Links) - } -} - -type stubPlatformResolver struct { - calls int - result resolverResult - err error -} - -func (r *stubPlatformResolver) Resolve(context.Context, string, resolverMetadata) (resolverResult, error) { - r.calls++ - return r.result, r.err -} - -func TestPlatformResolverChainMergesFallbacksWithoutReplacingEarlierLinks(t *testing.T) { - unitune := &stubPlatformResolver{result: resolverResult{ - Metadata: resolverMetadata{Title: "Track", Artist: "Artist"}, - Links: map[string]songLinkPlatformLink{ - "spotify": {URL: "https://open.spotify.com/track/source"}, - "deezer": {URL: "https://www.deezer.com/track/1"}, - "youtubeMusic": {URL: "https://music.youtube.com/watch?v=one"}, - }, - }} - musicBrainz := &stubPlatformResolver{result: resolverResult{Links: map[string]songLinkPlatformLink{ - "spotify": {URL: "https://open.spotify.com/track/different"}, - "appleMusic": {URL: "https://music.apple.com/us/album/track/1?i=2"}, - }}} - squigly := &stubPlatformResolver{result: resolverResult{Links: map[string]songLinkPlatformLink{ - "tidal": {URL: "https://tidal.com/browse/track/3"}, - }}} - chain := &platformResolverChain{unitune: unitune, musicBrainz: musicBrainz, squigly: squigly} - - result, err := chain.Resolve(context.Background(), "https://open.spotify.com/track/source", resolverMetadata{}) - if err != nil { - t.Fatalf("Resolve() error = %v", err) - } - if unitune.calls != 1 || musicBrainz.calls != 1 || squigly.calls != 1 { - t.Fatalf("resolver calls = %d/%d/%d, want 1/1/1", unitune.calls, musicBrainz.calls, squigly.calls) - } - if result.Links["spotify"].URL != "https://open.spotify.com/track/source" { - t.Fatalf("earlier resolver link was replaced: %#v", result.Links["spotify"]) - } - if len(result.Links) != 5 || result.Links["tidal"].URL == "" { - t.Fatalf("merged links = %#v", result.Links) - } -} - -func TestPlatformResolverChainPrefersSongLinkWeb(t *testing.T) { - songLinkWeb := &stubPlatformResolver{result: resolverResult{Links: map[string]songLinkPlatformLink{ - "spotify": {URL: "https://open.spotify.com/track/source"}, - "deezer": {URL: "https://www.deezer.com/track/101"}, - "tidal": {URL: "https://listen.tidal.com/track/202"}, - "amazonMusic": {URL: "https://music.amazon.com/tracks/TESTASIN"}, - }}} - unitune := &stubPlatformResolver{} - musicBrainz := &stubPlatformResolver{} - squigly := &stubPlatformResolver{} - chain := &platformResolverChain{ - songLinkWeb: songLinkWeb, - unitune: unitune, - musicBrainz: musicBrainz, - squigly: squigly, - } - - result, err := chain.Resolve(context.Background(), "https://open.spotify.com/track/source", resolverMetadata{}) - if err != nil { - t.Fatalf("Resolve() error = %v", err) - } - if songLinkWeb.calls != 1 || unitune.calls != 0 || musicBrainz.calls != 0 || squigly.calls != 0 { - t.Fatalf("resolver calls = %d/%d/%d/%d, want 1/0/0/0", songLinkWeb.calls, unitune.calls, musicBrainz.calls, squigly.calls) - } - if len(result.Links) != 4 { - t.Fatalf("Song.link web links = %#v", result.Links) - } -} - -func TestActiveResolversRunWithoutRetiredNetworkHop(t *testing.T) { - active := &stubPlatformResolver{result: resolverResult{Links: map[string]songLinkPlatformLink{ - "deezer": {URL: "https://www.deezer.com/track/123"}, - }}} - client := &SongLinkClient{fallbackResolver: active} - - links, err := client.resolveTrackPlatformsUncoalesced("https://open.spotify.com/track/source") - if err != nil { - t.Fatalf("resolveTrackPlatformsUncoalesced() error = %v", err) - } - if active.calls != 1 || links["deezer"].URL == "" { - t.Fatalf("active resolver calls/links = %d/%#v", active.calls, links) - } -} diff --git a/go_backend/progress.go b/go_backend/progress.go deleted file mode 100644 index 6539526c..00000000 --- a/go_backend/progress.go +++ /dev/null @@ -1,531 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "math" - "sort" - "sync" - "time" -) - -type DownloadProgress struct { - CurrentFile string `json:"current_file"` - Progress float64 `json:"progress"` - Speed float64 `json:"speed_mbps"` - BytesTotal int64 `json:"bytes_total"` - BytesReceived int64 `json:"bytes_received"` - IsDownloading bool `json:"is_downloading"` - Status string `json:"status"` - Stage string `json:"stage,omitempty"` -} - -type ItemProgress struct { - ItemID string `json:"item_id"` - BytesTotal int64 `json:"bytes_total"` - BytesReceived int64 `json:"bytes_received"` - Progress float64 `json:"progress"` - SpeedMBps float64 `json:"speed_mbps"` - IsDownloading bool `json:"is_downloading"` - Status string `json:"status"` - Stage string `json:"stage,omitempty"` - revision int64 -} - -const ( - itemProgressStatusPreparing = "preparing" - itemProgressStatusDownloading = "downloading" - itemProgressStatusCompleted = "completed" - itemProgressStatusFinalizing = "finalizing" -) - -type MultiProgress struct { - Items map[string]*ItemProgress `json:"items"` -} - -type MultiProgressDelta struct { - Seq int64 `json:"seq"` - Reset bool `json:"reset,omitempty"` - Items map[string]*ItemProgress `json:"items,omitempty"` - Removed []string `json:"removed,omitempty"` -} - -type progressBridgeState struct { - bytesBucket int64 - bytesTotal int64 - progressPct int64 - speedDeciMBps int64 - downloading bool - status string - stage string -} - -var ( - multiProgress = MultiProgress{Items: make(map[string]*ItemProgress)} - multiMu sync.RWMutex - multiProgressDirty = true - cachedMultiProgress = "{\"items\":{}}" - multiProgressSeq int64 - multiProgressReset int64 - removedProgressSeq = make(map[string]int64) - multiProgressNotify = make(chan struct{}) -) - -func markMultiProgressDirtyLocked() { - multiProgressDirty = true -} - -func nextMultiProgressSeqLocked() int64 { - multiProgressSeq++ - close(multiProgressNotify) - multiProgressNotify = make(chan struct{}) - return multiProgressSeq -} - -func itemProgressBridgeState(item *ItemProgress) progressBridgeState { - progress := item.Progress - if math.IsNaN(progress) || progress <= 0 { - progress = 0 - } else if progress >= 1 { - progress = 1 - } - - speed := item.SpeedMBps - if math.IsNaN(speed) || speed <= 0 { - speed = 0 - } - - return progressBridgeState{ - bytesBucket: item.BytesReceived / progressUpdateThreshold, - bytesTotal: item.BytesTotal, - progressPct: int64(math.Round(progress * 100)), - speedDeciMBps: int64(math.Round(speed * 10)), - downloading: item.IsDownloading, - status: item.Status, - stage: item.Stage, - } -} - -func markMultiProgressDirtyIfChangedLocked(item *ItemProgress, before progressBridgeState) { - if itemProgressBridgeState(item) != before { - item.revision = nextMultiProgressSeqLocked() - markMultiProgressDirtyLocked() - } -} - -func GetMultiProgress() string { - multiMu.RLock() - if !multiProgressDirty { - cached := cachedMultiProgress - multiMu.RUnlock() - return cached - } - multiMu.RUnlock() - - multiMu.Lock() - defer multiMu.Unlock() - if !multiProgressDirty { - return cachedMultiProgress - } - jsonBytes, err := json.Marshal(multiProgress) - if err != nil { - return "{\"items\":{}}" - } - cachedMultiProgress = string(jsonBytes) - multiProgressDirty = false - return cachedMultiProgress -} - -func GetMultiProgressDelta(sinceSeq int64) string { - multiMu.RLock() - currentSeq := multiProgressSeq - if sinceSeq >= currentSeq { - multiMu.RUnlock() - return "" - } - - reset := sinceSeq <= 0 || sinceSeq < multiProgressReset - delta := MultiProgressDelta{ - Seq: currentSeq, - Reset: reset, - } - if reset { - if len(multiProgress.Items) > 0 { - delta.Items = make(map[string]*ItemProgress, len(multiProgress.Items)) - for id, item := range multiProgress.Items { - copy := *item - copy.revision = 0 - delta.Items[id] = © - } - } - } else { - for id, item := range multiProgress.Items { - if item.revision > sinceSeq { - if delta.Items == nil { - delta.Items = make(map[string]*ItemProgress) - } - copy := *item - copy.revision = 0 - delta.Items[id] = © - } - } - for id, revision := range removedProgressSeq { - if revision > sinceSeq { - delta.Removed = append(delta.Removed, id) - } - } - } - multiMu.RUnlock() - - jsonBytes, err := json.Marshal(delta) - if err != nil { - return "" - } - return string(jsonBytes) -} - -// WaitForMultiProgressDelta blocks without polling until the bridge revision -// advances or the bounded heartbeat expires. Each waiter observes the same -// close-only notification channel, so UI and native-worker consumers do not -// compete for events. -func WaitForMultiProgressDelta(sinceSeq, timeoutMs int64) string { - if timeoutMs <= 0 { - timeoutMs = 15_000 - } - if timeoutMs > 60_000 { - timeoutMs = 60_000 - } - timer := time.NewTimer(time.Duration(timeoutMs) * time.Millisecond) - defer timer.Stop() - for { - multiMu.RLock() - if sinceSeq < multiProgressSeq { - multiMu.RUnlock() - return GetMultiProgressDelta(sinceSeq) - } - notify := multiProgressNotify - multiMu.RUnlock() - - select { - case <-notify: - continue - case <-timer.C: - return "" - } - } -} - -func StartItemProgress(itemID string) { - multiMu.Lock() - defer multiMu.Unlock() - - multiProgress.Items[itemID] = &ItemProgress{ - ItemID: itemID, - BytesTotal: 0, - BytesReceived: 0, - Progress: 0, - IsDownloading: false, - Status: itemProgressStatusPreparing, - revision: nextMultiProgressSeqLocked(), - } - delete(removedProgressSeq, itemID) - markMultiProgressDirtyLocked() -} - -func SetItemPreparing(itemID string) { - SetItemPreparingStage(itemID, "") -} - -func SetItemPreparingStage(itemID, stage string) { - multiMu.Lock() - defer multiMu.Unlock() - - if item, ok := multiProgress.Items[itemID]; ok { - before := itemProgressBridgeState(item) - item.Progress = 0 - item.BytesReceived = 0 - item.BytesTotal = 0 - item.SpeedMBps = 0 - item.IsDownloading = true - item.Status = itemProgressStatusPreparing - item.Stage = stage - markMultiProgressDirtyIfChangedLocked(item, before) - } -} - -func SetItemDownloading(itemID string) { - multiMu.Lock() - defer multiMu.Unlock() - - if item, ok := multiProgress.Items[itemID]; ok { - before := itemProgressBridgeState(item) - item.IsDownloading = true - item.Status = itemProgressStatusDownloading - item.Stage = "" - markMultiProgressDirtyIfChangedLocked(item, before) - } -} - -func SetItemBytesTotal(itemID string, total int64) { - multiMu.Lock() - defer multiMu.Unlock() - - if item, ok := multiProgress.Items[itemID]; ok { - before := itemProgressBridgeState(item) - item.BytesTotal = total - markMultiProgressDirtyIfChangedLocked(item, before) - } -} - -func SetItemBytesReceived(itemID string, received int64) { - multiMu.Lock() - defer multiMu.Unlock() - - if item, ok := multiProgress.Items[itemID]; ok { - before := itemProgressBridgeState(item) - item.BytesReceived = received - if item.BytesTotal > 0 { - item.Progress = float64(received) / float64(item.BytesTotal) - } - if received > 0 { - item.IsDownloading = true - item.Status = itemProgressStatusDownloading - item.Stage = "" - } - markMultiProgressDirtyIfChangedLocked(item, before) - } -} - -func SetItemBytesReceivedWithSpeed(itemID string, received int64, speedMBps float64) { - multiMu.Lock() - defer multiMu.Unlock() - - if item, ok := multiProgress.Items[itemID]; ok { - before := itemProgressBridgeState(item) - item.BytesReceived = received - item.SpeedMBps = speedMBps - if item.BytesTotal > 0 { - item.Progress = float64(received) / float64(item.BytesTotal) - } - if received > 0 { - item.IsDownloading = true - item.Status = itemProgressStatusDownloading - item.Stage = "" - } - markMultiProgressDirtyIfChangedLocked(item, before) - } -} - -func CompleteItemProgress(itemID string) { - multiMu.Lock() - defer multiMu.Unlock() - - if item, ok := multiProgress.Items[itemID]; ok { - before := itemProgressBridgeState(item) - item.Progress = 1.0 - item.IsDownloading = false - item.Status = itemProgressStatusCompleted - item.Stage = "" - markMultiProgressDirtyIfChangedLocked(item, before) - } -} - -func SetItemProgress(itemID string, progress float64, bytesReceived, bytesTotal int64) { - multiMu.Lock() - defer multiMu.Unlock() - - if item, ok := multiProgress.Items[itemID]; ok { - before := itemProgressBridgeState(item) - hasByteProgress := bytesReceived > 0 || bytesTotal > 0 - if item.Status != itemProgressStatusPreparing || hasByteProgress || progress >= 1 { - item.Progress = progress - } else { - item.Progress = 0 - } - if bytesReceived > 0 { - item.BytesReceived = bytesReceived - } - if bytesTotal > 0 { - item.BytesTotal = bytesTotal - } - if hasByteProgress || progress >= 1 || item.Status == itemProgressStatusDownloading { - item.IsDownloading = true - item.Status = itemProgressStatusDownloading - item.Stage = "" - } - markMultiProgressDirtyIfChangedLocked(item, before) - } -} - -func SetItemFinalizing(itemID string) { - multiMu.Lock() - defer multiMu.Unlock() - - if item, ok := multiProgress.Items[itemID]; ok { - before := itemProgressBridgeState(item) - item.Progress = 1.0 - item.Status = itemProgressStatusFinalizing - item.Stage = "" - markMultiProgressDirtyIfChangedLocked(item, before) - } -} - -// maxRemovedProgressEntries bounds the removal tombstones a long batch session -// can accumulate; pruning bumps multiProgressReset so lagging delta clients -// fall back to a full resync instead of missing removals. -const maxRemovedProgressEntries = 512 - -func RemoveItemProgress(itemID string) { - multiMu.Lock() - defer multiMu.Unlock() - - if _, ok := multiProgress.Items[itemID]; ok { - delete(multiProgress.Items, itemID) - removedProgressSeq[itemID] = nextMultiProgressSeqLocked() - if len(removedProgressSeq) > maxRemovedProgressEntries { - pruneRemovedProgressLocked() - } - } - markMultiProgressDirtyLocked() -} - -func pruneRemovedProgressLocked() { - revisions := make([]int64, 0, len(removedProgressSeq)) - for _, revision := range removedProgressSeq { - revisions = append(revisions, revision) - } - sort.Slice(revisions, func(i, j int) bool { return revisions[i] < revisions[j] }) - cutoff := revisions[len(revisions)/2] - for id, revision := range removedProgressSeq { - if revision <= cutoff { - delete(removedProgressSeq, id) - } - } - if cutoff > multiProgressReset { - multiProgressReset = cutoff - } -} - -func GetItemProgress(itemID string) string { - multiMu.RLock() - defer multiMu.RUnlock() - - if item, ok := multiProgress.Items[itemID]; ok { - jsonBytes, _ := json.Marshal(item) - return string(jsonBytes) - } - return "{}" -} - -func ClearAllItemProgress() { - multiMu.Lock() - defer multiMu.Unlock() - - multiProgress.Items = make(map[string]*ItemProgress) - removedProgressSeq = make(map[string]int64) - multiProgressReset = nextMultiProgressSeqLocked() - markMultiProgressDirtyLocked() -} - -func setDownloadDir(_ string) error { - return nil -} - -type ItemProgressWriter struct { - writer interface{ Write([]byte) (int, error) } - itemID string - current int64 - lastReported int64 - startTime time.Time - lastTime time.Time - lastBytes int64 -} - -const progressUpdateThreshold = 128 * 1024 -const progressUpdateMaxInterval = 250 * time.Millisecond - -// ItemTransferProgressReporter coalesces hot-path byte updates before they -// acquire multiMu. Transfer loops commonly read in 64 KiB chunks; reporting -// every read needlessly serializes parallel workers even though the bridge -// exposes progress at a much lower cadence. -type ItemTransferProgressReporter struct { - itemID string - mu sync.Mutex - lastReported int64 - lastTotal int64 - lastReportAt time.Time -} - -func NewItemTransferProgressReporter(itemID string, received, total int64) *ItemTransferProgressReporter { - return &ItemTransferProgressReporter{ - itemID: itemID, - lastReported: received, - lastTotal: total, - lastReportAt: time.Now(), - } -} - -func (reporter *ItemTransferProgressReporter) Report(received, total int64) { - if reporter == nil || reporter.itemID == "" { - return - } - - reporter.mu.Lock() - defer reporter.mu.Unlock() - now := time.Now() - bytesDelta := received - reporter.lastReported - if bytesDelta >= 0 && - bytesDelta < progressUpdateThreshold && - total == reporter.lastTotal && - now.Sub(reporter.lastReportAt) < progressUpdateMaxInterval { - return - } - - reporter.lastReported = received - reporter.lastTotal = total - reporter.lastReportAt = now - if total > 0 { - SetItemProgress(reporter.itemID, float64(received)/float64(total), received, total) - } else { - SetItemBytesReceived(reporter.itemID, received) - } -} - -func NewItemProgressWriter(w interface{ Write([]byte) (int, error) }, itemID string) *ItemProgressWriter { - now := time.Now() - return &ItemProgressWriter{ - writer: w, - itemID: itemID, - current: 0, - lastReported: 0, - startTime: now, - lastTime: now, - lastBytes: 0, - } -} - -func (pw *ItemProgressWriter) Write(p []byte) (int, error) { - if pw.itemID != "" && isDownloadCancelled(pw.itemID) { - return 0, ErrDownloadCancelled - } - n, err := pw.writer.Write(p) - if err != nil { - return n, err - } - pw.current += int64(n) - - if pw.lastReported == 0 || pw.current-pw.lastReported >= progressUpdateThreshold { - now := time.Now() - elapsed := now.Sub(pw.lastTime).Seconds() - var speedMBps float64 - if elapsed > 0 { - bytesInInterval := pw.current - pw.lastBytes - speedMBps = float64(bytesInInterval) / (1024 * 1024) / elapsed - } - - SetItemBytesReceivedWithSpeed(pw.itemID, pw.current, speedMBps) - pw.lastReported = pw.current - pw.lastTime = now - pw.lastBytes = pw.current - } - return n, nil -} diff --git a/go_backend/progress_test.go b/go_backend/progress_test.go deleted file mode 100644 index 6c19a9dc..00000000 --- a/go_backend/progress_test.go +++ /dev/null @@ -1,220 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "testing" - "time" -) - -func TestWaitForMultiProgressDeltaWakesOnRevision(t *testing.T) { - ClearAllItemProgress() - defer ClearAllItemProgress() - - multiMu.RLock() - since := multiProgressSeq - multiMu.RUnlock() - result := make(chan string, 1) - go func() { - result <- WaitForMultiProgressDelta(since, 1_000) - }() - time.Sleep(10 * time.Millisecond) - StartItemProgress("wait-progress") - select { - case payload := <-result: - if payload == "" { - t.Fatal("waiter woke without a progress delta") - } - var delta MultiProgressDelta - if err := json.Unmarshal([]byte(payload), &delta); err != nil { - t.Fatalf("decode delta: %v", err) - } - if delta.Items["wait-progress"] == nil { - t.Fatalf("delta missing item: %#v", delta) - } - case <-time.After(time.Second): - t.Fatal("progress waiter did not wake") - } -} - -func TestWaitForMultiProgressDeltaHeartbeatTimeout(t *testing.T) { - ClearAllItemProgress() - defer ClearAllItemProgress() - multiMu.RLock() - since := multiProgressSeq - multiMu.RUnlock() - startedAt := time.Now() - if payload := WaitForMultiProgressDelta(since, 20); payload != "" { - t.Fatalf("timeout payload = %q, want empty", payload) - } - if elapsed := time.Since(startedAt); elapsed < 15*time.Millisecond { - t.Fatalf("wait returned too early after %v", elapsed) - } -} - -func TestItemTransferProgressReporterCoalescesHotPathUpdates(t *testing.T) { - ClearAllItemProgress() - defer ClearAllItemProgress() - - const itemID = "coalesced-transfer-progress" - const total = int64(1024 * 1024) - StartItemProgress(itemID) - SetItemDownloading(itemID) - SetItemBytesTotal(itemID, total) - reporter := NewItemTransferProgressReporter(itemID, 0, total) - - reporter.Report(64*1024, total) - if received := multiProgress.Items[itemID].BytesReceived; received != 0 { - t.Fatalf("sub-threshold bytes = %d, want 0", received) - } - - reporter.Report(progressUpdateThreshold, total) - if received := multiProgress.Items[itemID].BytesReceived; received != progressUpdateThreshold { - t.Fatalf("threshold bytes = %d, want %d", received, progressUpdateThreshold) - } - - reporter.lastReportAt = time.Now().Add(-progressUpdateMaxInterval) - reporter.Report(progressUpdateThreshold+1, total) - if received := multiProgress.Items[itemID].BytesReceived; received != progressUpdateThreshold+1 { - t.Fatalf("interval flush bytes = %d, want %d", received, progressUpdateThreshold+1) - } -} - -func TestItemProgressPreparingAndDownloadingStatuses(t *testing.T) { - const itemID = "progress-phase-item" - RemoveItemProgress(itemID) - defer RemoveItemProgress(itemID) - - StartItemProgress(itemID) - SetItemPreparing(itemID) - - if item := multiProgress.Items[itemID]; item == nil { - t.Fatal("expected item progress entry to exist") - } else { - if item.Status != itemProgressStatusPreparing { - t.Fatalf("status = %q, want %q", item.Status, itemProgressStatusPreparing) - } - if item.Progress != 0 { - t.Fatalf("progress = %v, want 0", item.Progress) - } - } - - SetItemProgress(itemID, 0.05, 0, 0) - if item := multiProgress.Items[itemID]; item == nil { - t.Fatal("expected item progress entry to exist after update") - } else if item.Status != itemProgressStatusPreparing { - t.Fatalf("status after synthetic pre-download progress = %q, want %q", item.Status, itemProgressStatusPreparing) - } else if item.Progress != 0 { - t.Fatalf("progress after synthetic pre-download progress = %v, want 0", item.Progress) - } - - SetItemDownloading(itemID) - if item := multiProgress.Items[itemID]; item == nil { - t.Fatal("expected item progress entry to exist after downloading status") - } else if item.Status != itemProgressStatusDownloading { - t.Fatalf("status after download start = %q, want %q", item.Status, itemProgressStatusDownloading) - } - - SetItemProgress(itemID, 0.37, 0, 0) - if item := multiProgress.Items[itemID]; item == nil { - t.Fatal("expected item progress entry to exist after real update") - } else if item.Status != itemProgressStatusDownloading { - t.Fatalf("status after real progress update = %q, want %q", item.Status, itemProgressStatusDownloading) - } else if item.Progress != 0.37 { - t.Fatalf("progress after real update = %v, want 0.37", item.Progress) - } -} - -func TestItemProgressPreparationStageIsObservable(t *testing.T) { - ClearAllItemProgress() - defer ClearAllItemProgress() - - itemID := "stage-item" - StartItemProgress(itemID) - SetItemPreparingStage(itemID, "resolving_metadata") - - var progress ItemProgress - if err := json.Unmarshal([]byte(GetItemProgress(itemID)), &progress); err != nil { - t.Fatalf("decode progress: %v", err) - } - if progress.Status != itemProgressStatusPreparing || progress.Stage != "resolving_metadata" { - t.Fatalf("unexpected preparation progress: %#v", progress) - } - - SetItemDownloading(itemID) - progress = ItemProgress{} - if err := json.Unmarshal([]byte(GetItemProgress(itemID)), &progress); err != nil { - t.Fatalf("decode downloading progress: %v", err) - } - if progress.Stage != "" { - t.Fatalf("download stage was not cleared: %#v", progress) - } -} - -func TestItemProgressFinalizingAndCompletedStatuses(t *testing.T) { - const itemID = "progress-finalizing-item" - RemoveItemProgress(itemID) - defer RemoveItemProgress(itemID) - - StartItemProgress(itemID) - SetItemFinalizing(itemID) - - if item := multiProgress.Items[itemID]; item == nil { - t.Fatal("expected item progress entry to exist") - } else if item.Status != itemProgressStatusFinalizing { - t.Fatalf("status = %q, want %q", item.Status, itemProgressStatusFinalizing) - } - - CompleteItemProgress(itemID) - if item := multiProgress.Items[itemID]; item == nil { - t.Fatal("expected item progress entry to exist after completion") - } else if item.Status != itemProgressStatusCompleted { - t.Fatalf("status = %q, want %q", item.Status, itemProgressStatusCompleted) - } -} - -func TestMultiProgressDeltaResetChangedAndRemoved(t *testing.T) { - ClearAllItemProgress() - defer ClearAllItemProgress() - - StartItemProgress("item-a") - SetItemBytesTotal("item-a", 1000) - - var initial MultiProgressDelta - if err := json.Unmarshal([]byte(GetMultiProgressDelta(0)), &initial); err != nil { - t.Fatalf("initial delta parse failed: %v", err) - } - if !initial.Reset { - t.Fatal("initial delta should reset") - } - if initial.Seq <= 0 { - t.Fatalf("initial seq = %d, want > 0", initial.Seq) - } - if _, ok := initial.Items["item-a"]; !ok { - t.Fatal("initial delta missing item-a") - } - - if delta := GetMultiProgressDelta(initial.Seq); delta != "" { - t.Fatalf("delta after same seq = %q, want empty", delta) - } - - SetItemBytesReceivedWithSpeed("item-a", 256*1024, 2.5) - var changed MultiProgressDelta - if err := json.Unmarshal([]byte(GetMultiProgressDelta(initial.Seq)), &changed); err != nil { - t.Fatalf("changed delta parse failed: %v", err) - } - if changed.Reset { - t.Fatal("changed delta should not reset") - } - if _, ok := changed.Items["item-a"]; !ok { - t.Fatal("changed delta missing item-a") - } - - RemoveItemProgress("item-a") - var removed MultiProgressDelta - if err := json.Unmarshal([]byte(GetMultiProgressDelta(changed.Seq)), &removed); err != nil { - t.Fatalf("removed delta parse failed: %v", err) - } - if len(removed.Removed) != 1 || removed.Removed[0] != "item-a" { - t.Fatalf("removed = %#v, want item-a", removed.Removed) - } -} diff --git a/go_backend/ratelimit.go b/go_backend/ratelimit.go deleted file mode 100644 index 56ca1a8e..00000000 --- a/go_backend/ratelimit.go +++ /dev/null @@ -1,102 +0,0 @@ -package gobackend - -import ( - "context" - "sync" - "time" -) - -type RateLimiter struct { - mu sync.Mutex - maxRequests int - window time.Duration - timestamps []time.Time -} - -func NewRateLimiter(maxRequests int, window time.Duration) *RateLimiter { - return &RateLimiter{ - maxRequests: maxRequests, - window: window, - timestamps: make([]time.Time, 0, maxRequests), - } -} - -func (r *RateLimiter) WaitForSlot() { - _ = r.WaitForSlotContext(context.Background()) - -} - -// WaitForSlotContext reserves exactly one slot, rechecking the window after -// every wake-up. Multiple waiters may wake together, but only the first one -// that reacquires the mutex can consume the newly available slot. -func (r *RateLimiter) WaitForSlotContext(ctx context.Context) error { - if ctx == nil { - ctx = context.Background() - } - for { - r.mu.Lock() - now := time.Now() - r.cleanOldTimestamps(now) - if len(r.timestamps) < r.maxRequests { - r.timestamps = append(r.timestamps, now) - r.mu.Unlock() - return nil - } - - waitDuration := r.timestamps[0].Add(r.window).Sub(now) - r.mu.Unlock() - if waitDuration <= 0 { - continue - } - - timer := time.NewTimer(waitDuration) - select { - case <-ctx.Done(): - if !timer.Stop() { - <-timer.C - } - return ctx.Err() - case <-timer.C: - } - } -} - -func (r *RateLimiter) cleanOldTimestamps(now time.Time) { - cutoff := now.Add(-r.window) - validStart := 0 - - for i, ts := range r.timestamps { - if ts.After(cutoff) { - validStart = i - break - } - validStart = i + 1 - } - - if validStart > 0 { - r.timestamps = r.timestamps[validStart:] - } -} - -func (r *RateLimiter) TryAcquire() bool { - r.mu.Lock() - defer r.mu.Unlock() - - now := time.Now() - r.cleanOldTimestamps(now) - - if len(r.timestamps) < r.maxRequests { - r.timestamps = append(r.timestamps, now) - return true - } - - return false -} - -func (r *RateLimiter) Available() int { - r.mu.Lock() - defer r.mu.Unlock() - - r.cleanOldTimestamps(time.Now()) - return r.maxRequests - len(r.timestamps) -} diff --git a/go_backend/ratelimit_test.go b/go_backend/ratelimit_test.go deleted file mode 100644 index ef32b54a..00000000 --- a/go_backend/ratelimit_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package gobackend - -import ( - "context" - "errors" - "sync" - "testing" - "time" -) - -func TestRateLimiterRechecksCapacityAfterConcurrentWait(t *testing.T) { - const ( - waiters = 4 - window = 25 * time.Millisecond - ) - limiter := NewRateLimiter(1, window) - if !limiter.TryAcquire() { - t.Fatal("failed to consume initial slot") - } - - started := make(chan struct{}) - admitted := make(chan time.Time, waiters) - var ready sync.WaitGroup - ready.Add(waiters) - for i := 0; i < waiters; i++ { - go func() { - ready.Done() - <-started - limiter.WaitForSlot() - admitted <- time.Now() - }() - } - ready.Wait() - start := time.Now() - close(started) - - previous := start - for i := 0; i < waiters; i++ { - select { - case admittedAt := <-admitted: - if gap := admittedAt.Sub(previous); gap < window/2 { - t.Fatalf("waiters %d and %d admitted only %v apart", i, i+1, gap) - } - previous = admittedAt - case <-time.After(time.Second): - t.Fatal("timed out waiting for rate-limiter admission") - } - } -} - -func TestRateLimiterWaitCanBeCancelled(t *testing.T) { - limiter := NewRateLimiter(1, time.Hour) - if !limiter.TryAcquire() { - t.Fatal("failed to consume initial slot") - } - ctx, cancel := context.WithCancel(context.Background()) - cancel() - startedAt := time.Now() - err := limiter.WaitForSlotContext(ctx) - if !errors.Is(err, context.Canceled) { - t.Fatalf("WaitForSlotContext error = %v, want context.Canceled", err) - } - if elapsed := time.Since(startedAt); elapsed > 100*time.Millisecond { - t.Fatalf("cancelled wait took %v", elapsed) - } -} diff --git a/go_backend/replaygain_opus_test.go b/go_backend/replaygain_opus_test.go deleted file mode 100644 index ba4d8bc6..00000000 --- a/go_backend/replaygain_opus_test.go +++ /dev/null @@ -1,216 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/base64" - "encoding/binary" - "encoding/json" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" -) - -func readReplayGainOggPages(t *testing.T, path string) []oggEditPage { - t.Helper() - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - pages, err := readAllOggEditPages(f) - if err != nil { - t.Fatal(err) - } - return pages -} - -func TestOpusReplayGainReplacesLegacyTagsAndPreservesAudio(t *testing.T) { - for _, ext := range []string{".opus", ".ogg"} { - t.Run(ext, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "song"+ext) - buildTestOpus(t, path, []string{ - "TITLE=Example Song", "CUSTOM=Keep me", "METADATA_BLOCK_PICTURE=existing-picture", - "REPLAYGAIN_TRACK_GAIN=-4.00 dB", "REPLAYGAIN_TRACK_PEAK=0.800000", - "R128_TRACK_GAIN=-2304", "r128_track_gain=100", - "REPLAYGAIN_ALBUM_GAIN=-3.00 dB", "REPLAYGAIN_ALBUM_PEAK=0.900000", - "R128_ALBUM_GAIN=-2048", - }, 3) - // Preserve a nonzero OpusHead output gain as well as every audio page. - pages := readReplayGainOggPages(t, path) - binary.LittleEndian.PutUint16(pages[0].data[16:18], 256) - var original bytes.Buffer - for _, page := range pages { - if err := page.serialize(&original); err != nil { - t.Fatal(err) - } - } - if err := os.WriteFile(path, original.Bytes(), 0o600); err != nil { - t.Fatal(err) - } - - for _, scope := range []string{"track", "album"} { - fields, _ := json.Marshal(map[string]string{ - "replaygain_" + scope + "_gain": "-12.20 dB", - "replaygain_" + scope + "_peak": "1.258925", - }) - result, err := EditFileMetadata(path, string(fields)) - if err != nil || !strings.Contains(result, "native_ogg") { - t.Fatalf("write: %s, %v", result, err) - } - metadata, err := ReadFileMetadata(path) - if err != nil { - t.Fatal(err) - } - var readback map[string]any - if err := json.Unmarshal([]byte(metadata), &readback); err != nil { - t.Fatal(err) - } - if got := readback["replaygain_"+scope+"_gain"]; got != "-12.20 dB" { - t.Fatalf("%s gain = %v", scope, got) - } - raw := mustReadFile(t, path) - upper := strings.ToUpper(scope) - if bytes.Count(bytes.ToUpper(raw), []byte("R128_"+upper+"_GAIN=")) != 1 || - !bytes.Contains(raw, []byte("R128_"+upper+"_GAIN=-4403")) || - bytes.Contains(bytes.ToUpper(raw), []byte("REPLAYGAIN_"+upper+"_")) { - t.Fatalf("conflicting or incorrect %s comments", scope) - } - if scope == "track" && !bytes.Contains(raw, []byte("R128_ALBUM_GAIN=-2048")) { - t.Fatal("track update changed album gain") - } - } - after := readReplayGainOggPages(t, path) - if len(after) != len(pages) { - t.Fatalf("page count changed: %d -> %d", len(pages), len(after)) - } - for i := range pages { - if i == 1 { // Only OpusTags may change. - continue - } - var beforePage, afterPage bytes.Buffer - _ = pages[i].serialize(&beforePage) - _ = after[i].serialize(&afterPage) - if !bytes.Equal(beforePage.Bytes(), afterPage.Bytes()) { - t.Fatalf("header/audio page %d changed", i) - } - } - raw := mustReadFile(t, path) - for _, comment := range []string{"CUSTOM=Keep me", "METADATA_BLOCK_PICTURE=existing-picture"} { - if !bytes.Contains(raw, []byte(comment)) { - t.Fatalf("lost %s", comment) - } - } - if err := EditOggFields(path, map[string]string{"replaygain_track_gain": ""}); err != nil { - t.Fatal(err) - } - raw = mustReadFile(t, path) - if bytes.Contains(raw, []byte("R128_TRACK_GAIN=")) || !bytes.Contains(raw, []byte("R128_ALBUM_GAIN=-4403")) { - t.Fatal("clearing track gain affected the wrong scope") - } - }) - } -} - -func TestOpusReplayGainInvalidValuesLeaveFileUntouched(t *testing.T) { - for _, gain := range []string{"invalid", "NaN", "+Inf", "-Inf", "-124 dB", "133 dB"} { - t.Run(gain, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "song.opus") - buildTestOpus(t, path, []string{"R128_TRACK_GAIN=-1280"}, 1) - before := mustReadFile(t, path) - if err := EditOggFields(path, map[string]string{"replaygain_track_gain": gain}); err == nil { - t.Fatal("invalid gain was accepted") - } - if !bytes.Equal(before, mustReadFile(t, path)) { - t.Fatal("failed write changed the original") - } - }) - } - for _, raw := range []string{"32768", "-32769", "1.5"} { - if _, ok := r128ToReplayGainDb(raw); ok { - t.Fatalf("invalid R128 value accepted: %s", raw) - } - } -} - -func TestOpusReplayGainWithoutTitleOrArtist(t *testing.T) { - path := filepath.Join(t.TempDir(), "song.opus") - buildTestOpus(t, path, nil, 1) - if err := EditOggFields(path, map[string]string{"replaygain_track_gain": "0.00 dB"}); err != nil { - t.Fatal(err) - } - metadata, err := ReadFileMetadata(path) - if err != nil || !strings.Contains(metadata, `"replaygain_track_gain":"0.00 dB"`) { - t.Fatalf("ReplayGain-only comments were not read: %s, %v", metadata, err) - } -} - -func TestOpusReplayGainMediaRoundTrip(t *testing.T) { - ffmpeg, err := exec.LookPath("ffmpeg") - if err != nil { - t.Skip("requires ffmpeg on PATH") - } - run := func(t *testing.T, args ...string) []byte { - t.Helper() - cmd := exec.Command(ffmpeg, append([]string{"-hide_banner", "-v", "error"}, args...)...) - var stderr bytes.Buffer - cmd.Stderr = &stderr - out, err := cmd.Output() - if err != nil { - t.Fatalf("ffmpeg: %v: %s", err, &stderr) - } - return out - } - for _, cover := range []bool{false, true} { - name := "without-cover" - if cover { - name = "with-cover" - } - t.Run(name, func(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "song.opus") - run(t, "-f", "lavfi", "-i", "sine=frequency=440:duration=0.2", "-c:a", "libopus", - "-metadata", "title=Example Song", "-metadata", "REPLAYGAIN_TRACK_GAIN=-4.00 dB", - "-metadata", "R128_TRACK_GAIN=-2304", path) - var picture []byte - if cover { - picture, err = base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aB1sAAAAASUVORK5CYII=") - if err != nil { - t.Fatal(err) - } - coverPath := filepath.Join(dir, "cover.png") - if err := os.WriteFile(coverPath, picture, 0o600); err != nil { - t.Fatal(err) - } - if err := EditOggFields(path, map[string]string{"cover_path": coverPath}); err != nil { - t.Fatal(err) - } - } - audioHash := func() []byte { - return run(t, "-i", path, "-map", "0:a:0", "-c:a", "copy", "-f", "hash", "-hash", "sha256", "-") - } - beforeHash := audioHash() - if err := EditOggFields(path, map[string]string{ - "replaygain_track_gain": "-12.20 dB", "replaygain_track_peak": "1.258925", - "replaygain_album_gain": "-10.00 dB", "replaygain_album_peak": "1.300000", - }); err != nil { - t.Fatal(err) - } - if !bytes.Equal(beforeHash, audioHash()) { - t.Fatal("encoded audio changed") - } - run(t, "-i", path, "-map", "0:a:0", "-f", "null", "-") - metadata, err := ReadOggVorbisComments(path) - if err != nil || metadata.ReplayGainTrackGain != "-12.20 dB" || metadata.ReplayGainAlbumGain != "-10.00 dB" { - t.Fatalf("gain reread: %+v, %v", metadata, err) - } - if cover { - got, _, err := extractOggCoverArt(path) - if err != nil || !bytes.Equal(got, picture) { - t.Fatalf("cover changed: %v", err) - } - } - }) - } -} diff --git a/go_backend/runtime_metrics.go b/go_backend/runtime_metrics.go deleted file mode 100644 index d81248f4..00000000 --- a/go_backend/runtime_metrics.go +++ /dev/null @@ -1,54 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "runtime" - "time" -) - -type goRuntimeMetrics struct { - CapturedAtUnixMS int64 `json:"captured_at_unix_ms"` - HeapAllocBytes uint64 `json:"heap_alloc_bytes"` - HeapInuseBytes uint64 `json:"heap_inuse_bytes"` - HeapIdleBytes uint64 `json:"heap_idle_bytes"` - HeapReleasedBytes uint64 `json:"heap_released_bytes"` - HeapObjects uint64 `json:"heap_objects"` - StackInuseBytes uint64 `json:"stack_inuse_bytes"` - SysBytes uint64 `json:"sys_bytes"` - NextGCBytes uint64 `json:"next_gc_bytes"` - LastGCUnixMS int64 `json:"last_gc_unix_ms"` - NumGC uint32 `json:"num_gc"` - PauseTotalNS uint64 `json:"pause_total_ns"` - Goroutines int `json:"goroutines"` - GOMAXPROCS int `json:"gomaxprocs"` - CgoCalls int64 `json:"cgo_calls"` -} - -// GetRuntimeMetricsJSON reports Go-owned runtime memory. Native allocations -// from Flutter, FFmpeg, SQLite, and the OS are intentionally outside it. -func GetRuntimeMetricsJSON() string { - var stats runtime.MemStats - runtime.ReadMemStats(&stats) - metrics := goRuntimeMetrics{ - CapturedAtUnixMS: time.Now().UnixMilli(), - HeapAllocBytes: stats.HeapAlloc, - HeapInuseBytes: stats.HeapInuse, - HeapIdleBytes: stats.HeapIdle, - HeapReleasedBytes: stats.HeapReleased, - HeapObjects: stats.HeapObjects, - StackInuseBytes: stats.StackInuse, - SysBytes: stats.Sys, - NextGCBytes: stats.NextGC, - LastGCUnixMS: int64(stats.LastGC / uint64(time.Millisecond)), - NumGC: stats.NumGC, - PauseTotalNS: stats.PauseTotalNs, - Goroutines: runtime.NumGoroutine(), - GOMAXPROCS: runtime.GOMAXPROCS(0), - CgoCalls: runtime.NumCgoCall(), - } - payload, err := json.Marshal(metrics) - if err != nil { - return "{}" - } - return string(payload) -} diff --git a/go_backend/runtime_metrics_test.go b/go_backend/runtime_metrics_test.go deleted file mode 100644 index 10a9c436..00000000 --- a/go_backend/runtime_metrics_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package gobackend - -import ( - "encoding/json" - "testing" -) - -func TestGetRuntimeMetricsJSON(t *testing.T) { - var metrics goRuntimeMetrics - if err := json.Unmarshal([]byte(GetRuntimeMetricsJSON()), &metrics); err != nil { - t.Fatalf("runtime metrics JSON: %v", err) - } - if metrics.CapturedAtUnixMS <= 0 || metrics.HeapInuseBytes == 0 || metrics.SysBytes == 0 || metrics.Goroutines <= 0 || metrics.GOMAXPROCS <= 0 { - t.Fatalf("runtime metrics = %#v", metrics) - } - if metrics.HeapAllocBytes > metrics.HeapInuseBytes { - t.Fatalf("heap alloc %d exceeds heap in-use %d", metrics.HeapAllocBytes, metrics.HeapInuseBytes) - } -} diff --git a/go_backend/security_test_setup_test.go b/go_backend/security_test_setup_test.go deleted file mode 100644 index 29065574..00000000 --- a/go_backend/security_test_setup_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package gobackend - -import "encoding/base64" - -func init() { - // Production receives this key from platform secure storage before the - // extension system starts. Tests install a deterministic process-local key. - key := make([]byte, extensionStorageMasterKeyBytes) - for index := range key { - key[index] = byte(index + 1) - } - if err := SetExtensionStorageMasterKey(base64.StdEncoding.EncodeToString(key)); err != nil { - panic(err) - } -} diff --git a/go_backend/songlink.go b/go_backend/songlink.go deleted file mode 100644 index 59a633c7..00000000 --- a/go_backend/songlink.go +++ /dev/null @@ -1,830 +0,0 @@ -package gobackend - -import ( - "context" - "fmt" - "net/url" - "strings" - "sync" - "time" - - "golang.org/x/sync/singleflight" -) - -type SongLinkClient struct { - fallbackResolver platformFallbackResolver - resolutionFlight singleflight.Group - availabilityFlight singleflight.Group - platformLinksFlight singleflight.Group -} - -type songLinkPlatformLink struct { - URL string `json:"url"` -} - -type TrackAvailability struct { - SpotifyID string `json:"spotify_id"` - Tidal bool `json:"tidal"` - Amazon bool `json:"amazon"` - Qobuz bool `json:"qobuz"` - Deezer bool `json:"deezer"` - YouTube bool `json:"youtube"` - TidalURL string `json:"tidal_url,omitempty"` - AmazonURL string `json:"amazon_url,omitempty"` - QobuzURL string `json:"qobuz_url,omitempty"` - DeezerURL string `json:"deezer_url,omitempty"` - YouTubeURL string `json:"youtube_url,omitempty"` - DeezerID string `json:"deezer_id,omitempty"` - QobuzID string `json:"qobuz_id,omitempty"` - TidalID string `json:"tidal_id,omitempty"` - YouTubeID string `json:"youtube_id,omitempty"` -} - -var ( - globalSongLinkClient *SongLinkClient - songLinkClientOnce sync.Once - songLinkRegion = "US" - songLinkRegionMu sync.RWMutex - songLinkSearchByISRC = func(ctx context.Context, isrc string) (*TrackMetadata, error) { - return GetDeezerClient().SearchByISRC(ctx, isrc) - } - songLinkCheckAvailabilityFromDeezer = func(s *SongLinkClient, deezerTrackID string) (*TrackAvailability, error) { - return s.CheckAvailabilityFromDeezer(deezerTrackID) - } -) - -func NewSongLinkClient() *SongLinkClient { - songLinkClientOnce.Do(func() { - globalSongLinkClient = &SongLinkClient{ - fallbackResolver: defaultPlatformResolverFallbacks, - } - }) - return globalSongLinkClient -} - -func normalizeSongLinkRegion(region string) string { - normalized := strings.ToUpper(strings.TrimSpace(region)) - if len(normalized) != 2 { - return "US" - } - for _, ch := range normalized { - if ch < 'A' || ch > 'Z' { - return "US" - } - } - return normalized -} - -func SetSongLinkRegion(region string) { - normalized := normalizeSongLinkRegion(region) - songLinkRegionMu.Lock() - songLinkRegion = normalized - songLinkRegionMu.Unlock() -} - -func GetSongLinkRegion() string { - songLinkRegionMu.RLock() - region := songLinkRegion - songLinkRegionMu.RUnlock() - return region -} - -// resolveTrackPlatforms resolves a music URL through the active resolver -// chain. SongLinkClient remains as the compatibility facade used by the Dart -// and native bridges, but retired resolver services are not contacted. -func (s *SongLinkClient) resolveTrackPlatforms(inputURL string) (map[string]songLinkPlatformLink, error) { - value, err, _ := s.resolutionFlight.Do(inputURL, func() (any, error) { - return s.resolveTrackPlatformsUncoalesced(inputURL) - }) - if err != nil { - return nil, err - } - return cloneSongLinkPlatformLinks(value.(map[string]songLinkPlatformLink)), nil -} - -func (s *SongLinkClient) resolveTrackPlatformsUncoalesced(inputURL string) (map[string]songLinkPlatformLink, error) { - fallbackResolver := s.fallbackResolver - if fallbackResolver == nil { - fallbackResolver = defaultPlatformResolverFallbacks - } - ctx, cancel := context.WithTimeout(context.Background(), resolverFallbackTimeout) - defer cancel() - additional, additionalErr := fallbackResolver.Resolve(ctx, inputURL, resolverMetadata{}) - if additionalErr == nil && len(additional.Links) > 0 { - LogInfo("PlatformResolver", "Resolver chain returned %d platform links", len(additional.Links)) - return additional.Links, nil - } - if additionalErr == nil { - additionalErr = fmt.Errorf("additional resolvers returned no platform links") - } - - return nil, fmt.Errorf("platform resolvers failed: %w", additionalErr) -} - -// resolveTrackPlatformsByPlatform resolves using platform + type + id. -func (s *SongLinkClient) resolveTrackPlatformsByPlatform(platform, entityType, entityID string) (map[string]songLinkPlatformLink, error) { - inputURL, err := resolverURLFromPlatformID(platform, entityType, entityID) - if err != nil { - return nil, err - } - return s.resolveTrackPlatforms(inputURL) -} - -func cloneSongLinkPlatformLinks(links map[string]songLinkPlatformLink) map[string]songLinkPlatformLink { - if links == nil { - return nil - } - cloned := make(map[string]songLinkPlatformLink, len(links)) - for platform, link := range links { - cloned[platform] = link - } - return cloned -} - -const ( - trackAvailabilityCacheTTL = 30 * time.Minute - trackAvailabilityNegCacheTTL = 5 * time.Minute - trackAvailabilityCacheMax = 500 -) - -type trackAvailabilityCacheEntry struct { - availability *TrackAvailability - err bool - expiresAt time.Time -} - -var ( - trackAvailabilityCacheMu sync.Mutex - trackAvailabilityCache = map[string]trackAvailabilityCacheEntry{} -) - -// CheckTrackAvailability resolves platform availability for a track. Results are -// cached in memory (keyed by region + spotifyID/ISRC) to spare the song.link -// path its 9 req/min budget. This is an extra layer beneath the Dart-side -// cached-invoke and is safe: lookups are idempotent. Negative results use a -// shorter TTL so transient failures recover quickly. -func (s *SongLinkClient) CheckTrackAvailability(spotifyTrackID string, isrc string) (*TrackAvailability, error) { - spotifyTrackID = strings.TrimSpace(spotifyTrackID) - isrc = strings.ToUpper(strings.TrimSpace(isrc)) - - var idKey string - switch { - case spotifyTrackID != "": - idKey = "spotify:" + spotifyTrackID - case isrc != "": - idKey = "isrc:" + isrc - default: - return nil, fmt.Errorf("spotify track ID and ISRC are empty") - } - key := GetSongLinkRegion() + "|" + idKey - - if cached, hit, cachedErr := trackAvailabilityCacheLookup(key); hit { - if cachedErr { - return nil, fmt.Errorf("track availability unavailable (cached)") - } - return cloneTrackAvailability(cached), nil - } - - value, err, _ := s.availabilityFlight.Do(key, func() (any, error) { - // Another caller may have populated the cache while this caller was - // waiting to become the singleflight owner. - if cached, hit, cachedErr := trackAvailabilityCacheLookup(key); hit { - if cachedErr { - return nil, fmt.Errorf("track availability unavailable (cached)") - } - return cached, nil - } - - var availability *TrackAvailability - var resolveErr error - switch { - case spotifyTrackID != "": - availability, resolveErr = s.checkTrackAvailabilityFromSpotify(spotifyTrackID) - default: - availability, resolveErr = s.checkTrackAvailabilityFromISRC(isrc) - } - trackAvailabilityCacheStore(key, availability, resolveErr) - return availability, resolveErr - }) - if err != nil { - return nil, err - } - return cloneTrackAvailability(value.(*TrackAvailability)), nil -} - -const trackPlatformLinksCacheMax = 200 - -type trackPlatformLinksCacheEntry struct { - links map[string]string - err bool - expiresAt time.Time -} - -var ( - trackPlatformLinksCacheMu sync.Mutex - trackPlatformLinksCache = map[string]trackPlatformLinksCacheEntry{} -) - -// GetTrackPlatformLinks returns every streaming-platform URL song.link knows -// for a track, keyed by song.link platform ID. Cached in memory like -// CheckTrackAvailability to spare the same request budget. -func (s *SongLinkClient) GetTrackPlatformLinks(spotifyTrackID string, isrc string) (map[string]string, error) { - spotifyTrackID = strings.TrimSpace(spotifyTrackID) - isrc = strings.ToUpper(strings.TrimSpace(isrc)) - - var idKey string - switch { - case spotifyTrackID != "": - idKey = "spotify:" + spotifyTrackID - case isrc != "": - idKey = "isrc:" + isrc - default: - return nil, fmt.Errorf("spotify track ID and ISRC are empty") - } - key := GetSongLinkRegion() + "|" + idKey - - if links, hit, cachedErr := trackPlatformLinksCacheLookup(key); hit { - if cachedErr { - return nil, fmt.Errorf("track platform links unavailable (cached)") - } - return links, nil - } - - value, err, _ := s.platformLinksFlight.Do(key, func() (any, error) { - if links, hit, cachedErr := trackPlatformLinksCacheLookup(key); hit { - if cachedErr { - return nil, fmt.Errorf("track platform links unavailable (cached)") - } - return links, nil - } - links, fetchErr := s.fetchTrackPlatformLinks(spotifyTrackID, isrc) - trackPlatformLinksCacheStore(key, links, fetchErr) - return links, fetchErr - }) - if err != nil { - return nil, err - } - return cloneStringMap(value.(map[string]string)), nil -} - -func (s *SongLinkClient) fetchTrackPlatformLinks(spotifyTrackID string, isrc string) (map[string]string, error) { - var raw map[string]songLinkPlatformLink - var err error - if spotifyTrackID != "" { - raw, err = s.resolveTrackPlatforms( - fmt.Sprintf("https://open.spotify.com/track/%s", spotifyTrackID), - ) - } else { - ctx, cancel := context.WithTimeout(context.Background(), SongLinkTimeout) - defer cancel() - track, isrcErr := songLinkSearchByISRC(ctx, isrc) - if isrcErr != nil { - return nil, fmt.Errorf("failed to resolve Deezer track from ISRC %s: %w", isrc, isrcErr) - } - deezerTrackID := songLinkExtractDeezerTrackID(track) - if deezerTrackID == "" { - return nil, fmt.Errorf("failed to resolve Deezer track ID from ISRC %s", isrc) - } - raw, err = s.resolveTrackPlatforms( - fmt.Sprintf("https://www.deezer.com/track/%s", deezerTrackID), - ) - } - if err != nil { - return nil, err - } - - links := make(map[string]string, len(raw)) - for platform, link := range raw { - if url := normalizeShareURL(link.URL); url != "" { - links[platform] = url - } - } - if len(links) == 0 { - return nil, fmt.Errorf("no platform links found") - } - return links, nil -} - -func trackPlatformLinksCacheLookup(key string) (map[string]string, bool, bool) { - trackPlatformLinksCacheMu.Lock() - defer trackPlatformLinksCacheMu.Unlock() - e, ok := trackPlatformLinksCache[key] - if !ok { - return nil, false, false - } - if time.Now().After(e.expiresAt) { - delete(trackPlatformLinksCache, key) - return nil, false, false - } - return cloneStringMap(e.links), true, e.err -} - -func trackPlatformLinksCacheStore(key string, links map[string]string, err error) { - ttl := trackAvailabilityCacheTTL - if err != nil { - ttl = trackAvailabilityNegCacheTTL - } - trackPlatformLinksCacheMu.Lock() - defer trackPlatformLinksCacheMu.Unlock() - if _, exists := trackPlatformLinksCache[key]; !exists && len(trackPlatformLinksCache) >= trackPlatformLinksCacheMax { - var oldestKey string - var oldest time.Time - first := true - for k, e := range trackPlatformLinksCache { - if first || e.expiresAt.Before(oldest) { - oldest, oldestKey, first = e.expiresAt, k, false - } - } - delete(trackPlatformLinksCache, oldestKey) - } - trackPlatformLinksCache[key] = trackPlatformLinksCacheEntry{ - links: cloneStringMap(links), - err: err != nil, - expiresAt: time.Now().Add(ttl), - } -} - -func cloneStringMap(m map[string]string) map[string]string { - if m == nil { - return nil - } - c := make(map[string]string, len(m)) - for k, v := range m { - c[k] = v - } - return c -} - -func cloneTrackAvailability(a *TrackAvailability) *TrackAvailability { - if a == nil { - return nil - } - c := *a - return &c -} - -func trackAvailabilityCacheLookup(key string) (*TrackAvailability, bool, bool) { - trackAvailabilityCacheMu.Lock() - defer trackAvailabilityCacheMu.Unlock() - e, ok := trackAvailabilityCache[key] - if !ok { - return nil, false, false - } - if time.Now().After(e.expiresAt) { - delete(trackAvailabilityCache, key) - return nil, false, false - } - return e.availability, true, e.err -} - -func trackAvailabilityCacheStore(key string, availability *TrackAvailability, err error) { - ttl := trackAvailabilityCacheTTL - if err != nil { - ttl = trackAvailabilityNegCacheTTL - } - trackAvailabilityCacheMu.Lock() - defer trackAvailabilityCacheMu.Unlock() - if _, exists := trackAvailabilityCache[key]; !exists && len(trackAvailabilityCache) >= trackAvailabilityCacheMax { - var oldestKey string - var oldest time.Time - first := true - for k, e := range trackAvailabilityCache { - if first || e.expiresAt.Before(oldest) { - oldest, oldestKey, first = e.expiresAt, k, false - } - } - delete(trackAvailabilityCache, oldestKey) - } - trackAvailabilityCache[key] = trackAvailabilityCacheEntry{ - availability: availability, - err: err != nil, - expiresAt: time.Now().Add(ttl), - } -} - -func (s *SongLinkClient) checkTrackAvailabilityFromSpotify(spotifyTrackID string) (*TrackAvailability, error) { - spotifyURL := fmt.Sprintf("https://open.spotify.com/track/%s", spotifyTrackID) - links, err := s.resolveTrackPlatforms(spotifyURL) - if err != nil { - return nil, fmt.Errorf("platform resolution failed for Spotify %s: %w", spotifyTrackID, err) - } - return buildTrackAvailabilityFromSongLinkLinks(spotifyTrackID, links), nil -} - -func (s *SongLinkClient) checkTrackAvailabilityFromISRC(isrc string) (*TrackAvailability, error) { - ctx, cancel := context.WithTimeout(context.Background(), SongLinkTimeout) - defer cancel() - - track, err := songLinkSearchByISRC(ctx, isrc) - if err != nil { - return nil, fmt.Errorf("failed to resolve Deezer track from ISRC %s: %w", isrc, err) - } - - deezerTrackID := songLinkExtractDeezerTrackID(track) - if deezerTrackID == "" { - return nil, fmt.Errorf("failed to resolve Deezer track ID from ISRC %s", isrc) - } - - availability, err := songLinkCheckAvailabilityFromDeezer(s, deezerTrackID) - if err != nil { - return nil, fmt.Errorf("failed to resolve SongLink availability from ISRC %s via Deezer %s: %w", isrc, deezerTrackID, err) - } - - return availability, nil -} - -func songLinkExtractDeezerTrackID(track *TrackMetadata) string { - if track == nil { - return "" - } - - if deezerID, ok := strings.CutPrefix(strings.TrimSpace(track.SpotifyID), "deezer:"); ok { - deezerID = strings.TrimSpace(deezerID) - if deezerID != "" { - return deezerID - } - } - - if deezerID := extractDeezerIDFromURL(strings.TrimSpace(track.ExternalURL)); deezerID != "" { - return deezerID - } - - return "" -} - -func (s *SongLinkClient) GetStreamingURLs(spotifyTrackID string) (map[string]string, error) { - availability, err := s.CheckTrackAvailability(spotifyTrackID, "") - if err != nil { - return nil, err - } - - urls := make(map[string]string) - if availability.TidalURL != "" { - urls["tidal"] = availability.TidalURL - } - if availability.AmazonURL != "" { - urls["amazon"] = availability.AmazonURL - } - - return urls, nil -} - -func extractDeezerIDFromURL(deezerURL string) string { - parts := strings.Split(deezerURL, "/") - if len(parts) > 0 { - lastPart := parts[len(parts)-1] - if idx := strings.Index(lastPart, "?"); idx > 0 { - lastPart = lastPart[:idx] - } - return lastPart - } - return "" -} - -// extractQobuzIDFromURL extracts Qobuz track ID from URL. -// URL formats: -// - https://www.qobuz.com/us-en/album/.../12345678 (album page with track highlight) -// - https://open.qobuz.com/track/12345678 -// - https://www.qobuz.com/track/12345678 -// - https://play.qobuz.com/track/12345678 -func extractQobuzIDFromURL(qobuzURL string) string { - if qobuzURL == "" { - return "" - } - - if strings.Contains(qobuzURL, "/track/") { - parts := strings.Split(qobuzURL, "/track/") - if len(parts) > 1 { - idPart := parts[1] - if idx := strings.Index(idPart, "?"); idx > 0 { - idPart = idPart[:idx] - } - if idx := strings.Index(idPart, "/"); idx > 0 { - idPart = idPart[:idx] - } - idPart = strings.TrimSpace(idPart) - if idPart != "" && isNumeric(idPart) { - return idPart - } - } - } - - // Try to extract from album URL with track highlight (e.g. ?trackId=12345678) - if strings.Contains(qobuzURL, "trackId=") { - parts := strings.Split(qobuzURL, "trackId=") - if len(parts) > 1 { - idPart := parts[1] - if idx := strings.Index(idPart, "&"); idx > 0 { - idPart = idPart[:idx] - } - idPart = strings.TrimSpace(idPart) - if idPart != "" && isNumeric(idPart) { - return idPart - } - } - } - - // Last resort: get last numeric segment from URL - parts := strings.Split(qobuzURL, "/") - for i := len(parts) - 1; i >= 0; i-- { - part := parts[i] - if idx := strings.Index(part, "?"); idx > 0 { - part = part[:idx] - } - part = strings.TrimSpace(part) - if part != "" && isNumeric(part) { - return part - } - } - - return "" -} - -func extractTidalIDFromURL(tidalURL string) string { - if tidalURL == "" { - return "" - } - - if strings.Contains(tidalURL, "/track/") { - parts := strings.Split(tidalURL, "/track/") - if len(parts) > 1 { - idPart := parts[1] - if idx := strings.Index(idPart, "?"); idx > 0 { - idPart = idPart[:idx] - } - if idx := strings.Index(idPart, "/"); idx > 0 { - idPart = idPart[:idx] - } - idPart = strings.TrimSpace(idPart) - if idPart != "" && isNumeric(idPart) { - return idPart - } - } - } - - return "" -} - -func extractYouTubeIDFromURL(youtubeURL string) string { - if youtubeURL == "" { - return "" - } - - if strings.Contains(youtubeURL, "youtu.be/") { - parts := strings.Split(youtubeURL, "youtu.be/") - if len(parts) >= 2 { - idPart := parts[1] - if idx := strings.Index(idPart, "?"); idx > 0 { - idPart = idPart[:idx] - } - if idx := strings.Index(idPart, "&"); idx > 0 { - idPart = idPart[:idx] - } - return strings.TrimSpace(idPart) - } - } - - parsed, err := url.Parse(youtubeURL) - if err != nil { - return "" - } - - if v := parsed.Query().Get("v"); v != "" { - return v - } - - if strings.Contains(parsed.Path, "/embed/") { - parts := strings.Split(parsed.Path, "/embed/") - if len(parts) >= 2 { - return strings.Split(parts[1], "/")[0] - } - } - - return "" -} - -func (s *SongLinkClient) GetDeezerIDFromSpotify(spotifyTrackID string) (string, error) { - availability, err := s.CheckTrackAvailability(spotifyTrackID, "") - if err != nil { - return "", err - } - - if !availability.Deezer || availability.DeezerID == "" { - return "", fmt.Errorf("track not found on Deezer") - } - - return availability.DeezerID, nil -} - -func (s *SongLinkClient) GetYouTubeURLFromSpotify(spotifyTrackID string) (string, error) { - availability, err := s.CheckTrackAvailability(spotifyTrackID, "") - if err != nil { - return "", err - } - - if !availability.YouTube || availability.YouTubeURL == "" { - return "", fmt.Errorf("track not found on YouTube") - } - - return availability.YouTubeURL, nil -} - -type AlbumAvailability struct { - SpotifyID string `json:"spotify_id"` - Deezer bool `json:"deezer"` - DeezerURL string `json:"deezer_url,omitempty"` - DeezerID string `json:"deezer_id,omitempty"` -} - -func (s *SongLinkClient) CheckAlbumAvailability(spotifyAlbumID string) (*AlbumAvailability, error) { - spotifyURL := fmt.Sprintf("https://open.spotify.com/album/%s", spotifyAlbumID) - links, err := s.resolveTrackPlatforms(spotifyURL) - if err != nil { - return nil, fmt.Errorf("platform resolution failed for album %s: %w", spotifyAlbumID, err) - } - - availability := &AlbumAvailability{ - SpotifyID: spotifyAlbumID, - } - - if deezerLink, ok := links["deezer"]; ok && deezerLink.URL != "" { - availability.Deezer = true - availability.DeezerURL = deezerLink.URL - availability.DeezerID = extractDeezerIDFromURL(deezerLink.URL) - } - - return availability, nil -} - -func (s *SongLinkClient) GetDeezerAlbumIDFromSpotify(spotifyAlbumID string) (string, error) { - availability, err := s.CheckAlbumAvailability(spotifyAlbumID) - if err != nil { - return "", err - } - - if !availability.Deezer || availability.DeezerID == "" { - return "", fmt.Errorf("album not found on Deezer") - } - - return availability.DeezerID, nil -} - -// This is useful when we have Deezer metadata and want to find the track on other platforms -func (s *SongLinkClient) CheckAvailabilityFromDeezer(deezerTrackID string) (*TrackAvailability, error) { - if deezerTrackID == "" { - return nil, fmt.Errorf("deezer track ID is empty") - } - - return s.checkAvailabilityFromDeezerSongLink(deezerTrackID) -} - -func (s *SongLinkClient) checkAvailabilityFromDeezerSongLink(deezerTrackID string) (*TrackAvailability, error) { - deezerURL := fmt.Sprintf("https://www.deezer.com/track/%s", deezerTrackID) - links, err := s.resolveTrackPlatforms(deezerURL) - if err != nil { - return nil, fmt.Errorf("resolve failed for Deezer %s: %w", deezerTrackID, err) - } - - availability := buildTrackAvailabilityFromSongLinkLinks("", links) - // Ensure Deezer is always marked available since we started from a Deezer URL - availability.Deezer = true - availability.DeezerID = deezerTrackID - if availability.DeezerURL == "" { - availability.DeezerURL = deezerURL - } - return availability, nil -} - -// platform: "spotify", "deezer", "tidal", "amazonMusic", "appleMusic", "youtube", etc. -// entityType: "song" or "album" -// entityID: the ID on that platform -func (s *SongLinkClient) CheckAvailabilityByPlatform(platform, entityType, entityID string) (*TrackAvailability, error) { - if entityID == "" { - return nil, fmt.Errorf("%s ID is empty", platform) - } - - links, err := s.resolveTrackPlatformsByPlatform(platform, entityType, entityID) - if err != nil { - return nil, fmt.Errorf("resolve failed for %s %s: %w", platform, entityID, err) - } - - return buildTrackAvailabilityFromSongLinkLinks("", links), nil -} - -func buildTrackAvailabilityFromSongLinkLinks(spotifyTrackID string, links map[string]songLinkPlatformLink) *TrackAvailability { - availability := &TrackAvailability{ - SpotifyID: spotifyTrackID, - } - - if availability.SpotifyID == "" { - if spotifyLink, ok := links["spotify"]; ok && spotifyLink.URL != "" { - availability.SpotifyID = extractSpotifyIDFromURL(spotifyLink.URL) - } - } - if tidalLink, ok := links["tidal"]; ok && tidalLink.URL != "" { - availability.Tidal = true - availability.TidalURL = tidalLink.URL - availability.TidalID = extractTidalIDFromURL(tidalLink.URL) - } - if amazonLink, ok := links["amazonMusic"]; ok && amazonLink.URL != "" { - availability.Amazon = true - availability.AmazonURL = amazonLink.URL - } - if qobuzLink, ok := links["qobuz"]; ok && qobuzLink.URL != "" { - availability.Qobuz = true - availability.QobuzURL = qobuzLink.URL - availability.QobuzID = extractQobuzIDFromURL(qobuzLink.URL) - } - if deezerLink, ok := links["deezer"]; ok && deezerLink.URL != "" { - availability.Deezer = true - availability.DeezerURL = deezerLink.URL - availability.DeezerID = extractDeezerIDFromURL(deezerLink.URL) - } - if ytMusicLink, ok := links["youtubeMusic"]; ok && ytMusicLink.URL != "" { - availability.YouTube = true - availability.YouTubeURL = ytMusicLink.URL - availability.YouTubeID = extractYouTubeIDFromURL(ytMusicLink.URL) - } - if !availability.YouTube { - if youtubeLink, ok := links["youtube"]; ok && youtubeLink.URL != "" { - availability.YouTube = true - availability.YouTubeURL = youtubeLink.URL - availability.YouTubeID = extractYouTubeIDFromURL(youtubeLink.URL) - } - } - - return availability -} - -func extractSpotifyIDFromURL(spotifyURL string) string { - parts := strings.Split(spotifyURL, "/track/") - if len(parts) > 1 { - idPart := parts[1] - if idx := strings.Index(idPart, "?"); idx > 0 { - idPart = idPart[:idx] - } - return idPart - } - return "" -} - -func (s *SongLinkClient) GetSpotifyIDFromDeezer(deezerTrackID string) (string, error) { - availability, err := s.CheckAvailabilityFromDeezer(deezerTrackID) - if err != nil { - return "", err - } - - if availability.SpotifyID == "" { - return "", fmt.Errorf("track not found on Spotify") - } - - return availability.SpotifyID, nil -} - -func (s *SongLinkClient) GetTidalURLFromDeezer(deezerTrackID string) (string, error) { - availability, err := s.CheckAvailabilityFromDeezer(deezerTrackID) - if err != nil { - return "", err - } - - if !availability.Tidal || availability.TidalURL == "" { - return "", fmt.Errorf("track not found on Tidal") - } - - return availability.TidalURL, nil -} - -func (s *SongLinkClient) GetAmazonURLFromDeezer(deezerTrackID string) (string, error) { - availability, err := s.CheckAvailabilityFromDeezer(deezerTrackID) - if err != nil { - return "", err - } - - if !availability.Amazon || availability.AmazonURL == "" { - return "", fmt.Errorf("track not found on Amazon Music") - } - - return availability.AmazonURL, nil -} - -func (s *SongLinkClient) GetYouTubeURLFromDeezer(deezerTrackID string) (string, error) { - availability, err := s.CheckAvailabilityFromDeezer(deezerTrackID) - if err != nil { - return "", err - } - - if !availability.YouTube || availability.YouTubeURL == "" { - return "", fmt.Errorf("track not found on YouTube") - } - - return availability.YouTubeURL, nil -} - -func (s *SongLinkClient) CheckAvailabilityFromURL(inputURL string) (*TrackAvailability, error) { - links, err := s.resolveTrackPlatforms(inputURL) - if err != nil { - return nil, fmt.Errorf("resolve failed for URL %s: %w", inputURL, err) - } - - return buildTrackAvailabilityFromSongLinkLinks("", links), nil -} diff --git a/go_backend/songlink_test.go b/go_backend/songlink_test.go deleted file mode 100644 index 687996fe..00000000 --- a/go_backend/songlink_test.go +++ /dev/null @@ -1,197 +0,0 @@ -package gobackend - -import ( - "context" - "io" - "net/http" - "sync" - "sync/atomic" - "testing" - "time" -) - -type roundTripFunc func(*http.Request) (*http.Response, error) - -func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { - return fn(req) -} - -type blockingPlatformResolver struct { - calls atomic.Int32 - started chan struct{} - release chan struct{} - result resolverResult -} - -func (r *blockingPlatformResolver) Resolve(context.Context, string, resolverMetadata) (resolverResult, error) { - if r.calls.Add(1) == 1 { - close(r.started) - } - <-r.release - return r.result, nil -} - -func TestIdenticalPlatformResolverRequestsAreCoalesced(t *testing.T) { - resolver := &blockingPlatformResolver{ - started: make(chan struct{}), - release: make(chan struct{}), - result: resolverResult{Links: map[string]songLinkPlatformLink{ - "deezer": {URL: "https://www.deezer.com/track/123"}, - }}, - } - client := &SongLinkClient{fallbackResolver: resolver} - - const workers = 16 - start := make(chan struct{}) - errs := make(chan error, workers) - var wg sync.WaitGroup - wg.Add(workers) - for range workers { - go func() { - defer wg.Done() - <-start - links, err := client.resolveTrackPlatforms("https://open.spotify.com/track/coalesced") - if err == nil && links["deezer"].URL == "" { - err = io.ErrUnexpectedEOF - } - errs <- err - }() - } - close(start) - - select { - case <-resolver.started: - case <-time.After(time.Second): - t.Fatal("resolver did not start") - } - time.Sleep(20 * time.Millisecond) - close(resolver.release) - wg.Wait() - close(errs) - for err := range errs { - if err != nil { - t.Fatalf("coalesced resolution failed: %v", err) - } - } - if got := resolver.calls.Load(); got != 1 { - t.Fatalf("resolver calls = %d, want 1", got) - } -} - -func TestGetRetryAfterDurationMissingHeaderReturnsZero(t *testing.T) { - resp := &http.Response{Header: make(http.Header)} - if got := getRetryAfterDuration(resp); got != 0 { - t.Fatalf("getRetryAfterDuration() = %v, want 0", got) - } -} - -func resetTrackAvailabilityCache() { - trackAvailabilityCacheMu.Lock() - trackAvailabilityCache = map[string]trackAvailabilityCacheEntry{} - trackAvailabilityCacheMu.Unlock() -} - -// testResolverResult is an in-memory fixture; none of these URLs are fetched. -func testResolverResult() resolverResult { - return resolverResult{Links: map[string]songLinkPlatformLink{ - "spotify": {URL: "https://open.spotify.com/track/testspotifyid"}, - "deezer": {URL: "https://www.deezer.com/track/101"}, - "amazonMusic": {URL: "https://music.amazon.com/tracks/TESTASIN"}, - "tidal": {URL: "https://listen.tidal.com/track/202"}, - "qobuz": {URL: "https://open.qobuz.com/track/303"}, - "youtubeMusic": {URL: "https://music.youtube.com/watch?v=testvideoid1"}, - }} -} - -func TestCheckTrackAvailabilityFromSpotifyUsesActiveResolverChain(t *testing.T) { - resetTrackAvailabilityCache() - client := &SongLinkClient{fallbackResolver: &stubPlatformResolver{result: testResolverResult()}} - - availability, err := client.CheckTrackAvailability("testspotifyid", "") - if err != nil { - t.Fatalf("CheckTrackAvailability() error = %v", err) - } - if availability.SpotifyID != "testspotifyid" || availability.DeezerID != "101" { - t.Fatalf("availability IDs = %+v", availability) - } - if !availability.Deezer || !availability.Amazon || !availability.Tidal || !availability.Qobuz || !availability.YouTube { - t.Fatalf("availability flags = %+v", availability) - } - if availability.YouTubeID != "testvideoid1" { - t.Fatalf("YouTubeID = %q", availability.YouTubeID) - } -} - -type inputCapturingResolver struct { - input string - result resolverResult -} - -func (r *inputCapturingResolver) Resolve(_ context.Context, input string, _ resolverMetadata) (resolverResult, error) { - r.input = input - return r.result, nil -} - -func TestResolveTrackPlatformsByPlatformBuildsDirectURL(t *testing.T) { - resolver := &inputCapturingResolver{result: testResolverResult()} - client := &SongLinkClient{fallbackResolver: resolver} - - links, err := client.resolveTrackPlatformsByPlatform("spotify", "song", "testspotifyid") - if err != nil { - t.Fatalf("resolveTrackPlatformsByPlatform() error = %v", err) - } - if resolver.input != "https://open.spotify.com/track/testspotifyid" { - t.Fatalf("resolver input = %q", resolver.input) - } - if links["deezer"].URL == "" { - t.Fatalf("resolver links = %#v", links) - } -} - -func TestCheckTrackAvailabilityCachesResult(t *testing.T) { - resetTrackAvailabilityCache() - resolver := &stubPlatformResolver{result: resolverResult{Links: map[string]songLinkPlatformLink{ - "spotify": {URL: "https://open.spotify.com/track/cachedid"}, - "deezer": {URL: "https://www.deezer.com/track/111"}, - }}} - client := &SongLinkClient{fallbackResolver: resolver} - - first, err := client.CheckTrackAvailability("cachedid", "") - if err != nil { - t.Fatalf("first CheckTrackAvailability() error = %v", err) - } - second, err := client.CheckTrackAvailability("cachedid", "") - if err != nil { - t.Fatalf("second CheckTrackAvailability() error = %v", err) - } - if resolver.calls != 1 { - t.Fatalf("resolver calls = %d, want 1", resolver.calls) - } - if first == second || second.DeezerID != "111" { - t.Fatalf("cached result = %+v", second) - } -} - -func TestCheckTrackAvailabilityNegativeCacheTTL(t *testing.T) { - resetTrackAvailabilityCache() - entry := trackAvailabilityCacheEntry{err: true, expiresAt: time.Now().Add(-time.Second)} - key := GetSongLinkRegion() + "|spotify:expiredneg" - trackAvailabilityCacheMu.Lock() - trackAvailabilityCache[key] = entry - trackAvailabilityCacheMu.Unlock() - - if _, hit, _ := trackAvailabilityCacheLookup(key); hit { - t.Fatal("expired negative entry should not be a cache hit") - } -} - -func TestCheckAvailabilityFromDeezerUsesActiveResolverChain(t *testing.T) { - client := &SongLinkClient{fallbackResolver: &stubPlatformResolver{result: testResolverResult()}} - availability, err := client.CheckAvailabilityFromDeezer("908604612") - if err != nil { - t.Fatalf("CheckAvailabilityFromDeezer() error = %v", err) - } - if !availability.Deezer || availability.DeezerID != "908604612" || availability.SpotifyID != "testspotifyid" { - t.Fatalf("availability = %+v", availability) - } -} diff --git a/go_backend/title_match_utils.go b/go_backend/title_match_utils.go deleted file mode 100644 index 8dd55f61..00000000 --- a/go_backend/title_match_utils.go +++ /dev/null @@ -1,573 +0,0 @@ -package gobackend - -import ( - "regexp" - "strings" - "unicode" - - "golang.org/x/text/unicode/norm" -) - -func writeNormalizedArtistRune(b *strings.Builder, r rune) { - switch r { - case 'đ': - b.WriteString("dj") - case 'ß': - b.WriteString("ss") - case 'æ': - b.WriteString("ae") - case 'œ': - b.WriteString("oe") - default: - b.WriteRune(r) - } -} - -func normalizeLooseTitle(title string) string { - trimmed := strings.TrimSpace(strings.ToLower(title)) - if trimmed == "" { - return "" - } - - var b strings.Builder - b.Grow(len(trimmed)) - - for _, r := range trimmed { - switch { - case unicode.IsLetter(r), unicode.IsNumber(r): - b.WriteRune(r) - case unicode.IsSpace(r): - b.WriteByte(' ') - case r == '/', r == '\\', r == '_', r == '-', r == '|', r == '.', r == '&', r == '+': - b.WriteByte(' ') - default: - } - } - - return strings.Join(strings.Fields(b.String()), " ") -} - -func normalizeLooseArtistName(name string) string { - trimmed := strings.TrimSpace(strings.ToLower(name)) - if trimmed == "" { - return "" - } - - decomposed := norm.NFD.String(trimmed) - - var b strings.Builder - b.Grow(len(decomposed)) - - for _, r := range decomposed { - switch { - case unicode.Is(unicode.Mn, r), unicode.Is(unicode.Mc, r), unicode.Is(unicode.Me, r): - continue - case unicode.IsLetter(r), unicode.IsNumber(r): - writeNormalizedArtistRune(&b, r) - case unicode.IsSpace(r): - b.WriteByte(' ') - case r == '/', r == '\\', r == '_', r == '-', r == '|', r == '.', r == '&', r == '+': - b.WriteByte(' ') - default: - } - } - - return strings.Join(strings.Fields(b.String()), " ") -} - -func hasAlphaNumericRunes(value string) bool { - for _, r := range value { - if unicode.IsLetter(r) || unicode.IsNumber(r) { - return true - } - } - return false -} - -func normalizeSymbolOnlyTitle(title string) string { - trimmed := strings.TrimSpace(strings.ToLower(title)) - if trimmed == "" { - return "" - } - - var b strings.Builder - b.Grow(len(trimmed)) - - for _, r := range trimmed { - switch { - case unicode.IsLetter(r), unicode.IsNumber(r), unicode.IsSpace(r), unicode.IsPunct(r): - continue - // Drop combining marks such as emoji variation selectors. - case unicode.Is(unicode.Mn, r), unicode.Is(unicode.Mc, r), unicode.Is(unicode.Me, r): - continue - default: - b.WriteRune(r) - } - } - - return b.String() -} - -func artistsMatch(expectedArtist, foundArtist string) bool { - normExpected := normalizeLooseArtistName(expectedArtist) - normFound := normalizeLooseArtistName(foundArtist) - - if normExpected == normFound { - return true - } - - if strings.Contains(normExpected, normFound) || - strings.Contains(normFound, normExpected) { - return true - } - - expectedArtists := splitArtists(expectedArtist) - foundArtists := splitArtists(foundArtist) - - for _, expected := range expectedArtists { - for _, found := range foundArtists { - if expected == found { - return true - } - if strings.Contains(expected, found) || - strings.Contains(found, expected) { - return true - } - if sameWordsUnordered(expected, found) { - return true - } - } - } - - return isLatinScript(expectedArtist) != isLatinScript(foundArtist) -} - -func splitArtists(artists string) []string { - normalized := strings.ToLower(artists) - normalized = strings.ReplaceAll(normalized, " feat. ", "|") - normalized = strings.ReplaceAll(normalized, " feat ", "|") - normalized = strings.ReplaceAll(normalized, " ft. ", "|") - normalized = strings.ReplaceAll(normalized, " ft ", "|") - normalized = strings.ReplaceAll(normalized, " & ", "|") - normalized = strings.ReplaceAll(normalized, " and ", "|") - normalized = strings.ReplaceAll(normalized, ",", "|") - normalized = strings.ReplaceAll(normalized, ";", "|") - normalized = strings.ReplaceAll(normalized, " x ", "|") - - parts := strings.Split(normalized, "|") - result := make([]string, 0, len(parts)) - for _, part := range parts { - trimmed := normalizeLooseArtistName(part) - if trimmed != "" { - result = append(result, trimmed) - } - } - return result -} - -func sameWordsUnordered(a, b string) bool { - wordsA := strings.Fields(a) - wordsB := strings.Fields(b) - if len(wordsA) != len(wordsB) || len(wordsA) == 0 { - return false - } - - sortedA := make([]string, len(wordsA)) - sortedB := make([]string, len(wordsB)) - copy(sortedA, wordsA) - copy(sortedB, wordsB) - - for i := 0; i < len(sortedA)-1; i++ { - for j := i + 1; j < len(sortedA); j++ { - if sortedA[i] > sortedA[j] { - sortedA[i], sortedA[j] = sortedA[j], sortedA[i] - } - if sortedB[i] > sortedB[j] { - sortedB[i], sortedB[j] = sortedB[j], sortedB[i] - } - } - } - - for i := range sortedA { - if sortedA[i] != sortedB[i] { - return false - } - } - return true -} - -func titlesMatch(expectedTitle, foundTitle string) bool { - normExpected := strings.ToLower(strings.TrimSpace(expectedTitle)) - normFound := strings.ToLower(strings.TrimSpace(foundTitle)) - - if normExpected == normFound { - return true - } - - if strings.Contains(normExpected, normFound) || - strings.Contains(normFound, normExpected) { - return true - } - - cleanExpected := cleanTitle(normExpected) - cleanFound := cleanTitle(normFound) - if cleanExpected == cleanFound { - return true - } - - if cleanExpected != "" && cleanFound != "" { - if strings.Contains(cleanExpected, cleanFound) || - strings.Contains(cleanFound, cleanExpected) { - return true - } - } - - coreExpected := extractCoreTitle(normExpected) - coreFound := extractCoreTitle(normFound) - if coreExpected != "" && coreFound != "" && coreExpected == coreFound { - return true - } - - looseExpected := normalizeLooseTitle(normExpected) - looseFound := normalizeLooseTitle(normFound) - if looseExpected != "" && looseFound != "" { - if looseExpected == looseFound { - return true - } - if strings.Contains(looseExpected, looseFound) || - strings.Contains(looseFound, looseExpected) { - return true - } - } - - if (!hasAlphaNumericRunes(expectedTitle) || !hasAlphaNumericRunes(foundTitle)) && - strings.TrimSpace(expectedTitle) != "" && - strings.TrimSpace(foundTitle) != "" { - expectedSymbols := normalizeSymbolOnlyTitle(expectedTitle) - foundSymbols := normalizeSymbolOnlyTitle(foundTitle) - if expectedSymbols != "" && - foundSymbols != "" && - expectedSymbols == foundSymbols { - return true - } - } - - return false -} - -var trackTitleAnnotationPattern = regexp.MustCompile(`(?i)[(\[]\s*(?:(?:feat\.?|ft\.?|featuring)\s+[^)\]]+|from\s+["“][^)\]]+["”]\s*)[)\]]`) - -func normalizeTrackIdentityTitle(title string) string { - return normalizeLooseTitle(trackTitleAnnotationPattern.ReplaceAllString(title, " ")) -} - -func trackTitlesMatch(expectedTitle, foundTitle string) bool { - expectedTitle = trackTitleAnnotationPattern.ReplaceAllString(expectedTitle, " ") - foundTitle = trackTitleAnnotationPattern.ReplaceAllString(foundTitle, " ") - expected := normalizeLooseTitle(expectedTitle) - found := normalizeLooseTitle(foundTitle) - if expected != "" && expected == found { - return true - } - - // Version words identify recordings; punctuation around them does not. - for _, title := range []string{expected, found} { - for _, word := range strings.Fields(title) { - switch word { - case "mix", "remix", "live", "acoustic", "demo", "instrumental", - "karaoke", "edit", "extended", "slowed", "sped": - return false - } - } - } - return titlesMatch(expectedTitle, foundTitle) -} - -func extractCoreTitle(title string) string { - parenIdx := strings.Index(title, "(") - bracketIdx := strings.Index(title, "[") - dashIdx := strings.Index(title, " - ") - - cutIdx := len(title) - if parenIdx > 0 && parenIdx < cutIdx { - cutIdx = parenIdx - } - if bracketIdx > 0 && bracketIdx < cutIdx { - cutIdx = bracketIdx - } - if dashIdx > 0 && dashIdx < cutIdx { - cutIdx = dashIdx - } - - return strings.TrimSpace(title[:cutIdx]) -} - -func cleanTitle(title string) string { - cleaned := title - - versionPatterns := []string{ - "remaster", "remastered", "deluxe", "bonus", "single", - "album version", "radio edit", "original mix", "extended", - "club mix", "remix", "live", "acoustic", "demo", - } - - for { - startParen := strings.LastIndex(cleaned, "(") - endParen := strings.LastIndex(cleaned, ")") - if startParen >= 0 && endParen > startParen { - content := strings.ToLower(cleaned[startParen+1 : endParen]) - isVersionIndicator := false - for _, pattern := range versionPatterns { - if strings.Contains(content, pattern) { - isVersionIndicator = true - break - } - } - if isVersionIndicator { - cleaned = strings.TrimSpace(cleaned[:startParen]) + cleaned[endParen+1:] - continue - } - } - break - } - - for { - startBracket := strings.LastIndex(cleaned, "[") - endBracket := strings.LastIndex(cleaned, "]") - if startBracket >= 0 && endBracket > startBracket { - content := strings.ToLower(cleaned[startBracket+1 : endBracket]) - isVersionIndicator := false - for _, pattern := range versionPatterns { - if strings.Contains(content, pattern) { - isVersionIndicator = true - break - } - } - if isVersionIndicator { - cleaned = strings.TrimSpace(cleaned[:startBracket]) + cleaned[endBracket+1:] - continue - } - } - break - } - - dashPatterns := []string{ - " - remaster", " - remastered", " - single version", " - radio edit", - " - live", " - acoustic", " - demo", " - remix", - } - for _, pattern := range dashPatterns { - if strings.HasSuffix(strings.ToLower(cleaned), pattern) { - cleaned = cleaned[:len(cleaned)-len(pattern)] - } - } - - for strings.Contains(cleaned, " ") { - cleaned = strings.ReplaceAll(cleaned, " ", " ") - } - - return strings.TrimSpace(cleaned) -} - -func isLatinScript(value string) bool { - for _, r := range value { - if r < 128 { - continue - } - if (r >= 0x0100 && r <= 0x024F) || - (r >= 0x1E00 && r <= 0x1EFF) || - (r >= 0x00C0 && r <= 0x00FF) { - continue - } - if (r >= 0x4E00 && r <= 0x9FFF) || - (r >= 0x3040 && r <= 0x309F) || - (r >= 0x30A0 && r <= 0x30FF) || - (r >= 0xAC00 && r <= 0xD7AF) || - (r >= 0x0600 && r <= 0x06FF) || - (r >= 0x0400 && r <= 0x04FF) { - return false - } - } - return true -} - -type resolvedTrackInfo struct { - Title string - ArtistName string - AlbumName string - ISRC string - Duration int - SkipNameVerification bool -} - -func exactLooseIdentityMatch(expected, found string, normalize func(string) string) bool { - normExpected := normalize(expected) - normFound := normalize(found) - if normExpected != "" && normFound != "" { - return normExpected == normFound - } - return strings.EqualFold(strings.TrimSpace(expected), strings.TrimSpace(found)) -} - -func durationMatchesRequest(req DownloadRequest, resolved resolvedTrackInfo) bool { - expectedDurationSec := req.DurationMS / 1000 - if expectedDurationSec <= 0 || resolved.Duration <= 0 { - return false - } - diff := expectedDurationSec - resolved.Duration - if diff < 0 { - diff = -diff - } - return diff <= 10 -} - -func hasStrongTrackIdentity(req DownloadRequest, resolved resolvedTrackInfo) bool { - if req.TrackName == "" || resolved.Title == "" || - req.ArtistName == "" || resolved.ArtistName == "" { - return false - } - - titleExact := exactLooseIdentityMatch(req.TrackName, resolved.Title, normalizeTrackIdentityTitle) - if !titleExact { - return false - } - - artistExact := exactLooseIdentityMatch( - req.ArtistName, - resolved.ArtistName, - normalizeLooseArtistName, - ) - return artistExact || (artistsMatch(req.ArtistName, resolved.ArtistName) && - durationMatchesRequest(req, resolved)) -} - -func trackMatchesRequest(req DownloadRequest, resolved resolvedTrackInfo, logPrefix string) bool { - exactISRCMatch := req.ISRC != "" && - resolved.ISRC != "" && - strings.EqualFold(strings.TrimSpace(req.ISRC), strings.TrimSpace(resolved.ISRC)) - conflictingISRC := req.ISRC != "" && resolved.ISRC != "" && !exactISRCMatch - - if !exactISRCMatch && !resolved.SkipNameVerification { - if req.ArtistName != "" && resolved.ArtistName != "" && - !artistsMatch(req.ArtistName, resolved.ArtistName) { - GoLog("[%s] Verification failed: artist mismatch — expected '%s', got '%s'\n", - logPrefix, req.ArtistName, resolved.ArtistName) - return false - } - - if req.TrackName != "" && resolved.Title != "" && - !trackTitlesMatch(req.TrackName, resolved.Title) { - GoLog("[%s] Verification failed: title mismatch — expected '%s', got '%s'\n", - logPrefix, req.TrackName, resolved.Title) - return false - } - - if req.AlbumName != "" && resolved.AlbumName != "" && - !titlesMatch(req.AlbumName, resolved.AlbumName) { - if conflictingISRC || !hasStrongTrackIdentity(req, resolved) { - GoLog("[%s] Verification failed: album mismatch — expected '%s', got '%s'\n", - logPrefix, req.AlbumName, resolved.AlbumName) - return false - } - GoLog("[%s] Verification accepted album mismatch for matching track identity — expected '%s', got '%s'\n", - logPrefix, req.AlbumName, resolved.AlbumName) - } - } - - expectedDurationSec := req.DurationMS / 1000 - if expectedDurationSec > 0 && resolved.Duration > 0 { - diff := expectedDurationSec - resolved.Duration - if diff < 0 { - diff = -diff - } - if diff > 10 { - // Catalog durations can disagree even for the same recording. Require - // both its ISRC and matching names; a preview still cannot qualify. - if exactISRCMatch && req.TrackName != "" && resolved.Title != "" && - exactLooseIdentityMatch(req.TrackName, resolved.Title, normalizeTrackIdentityTitle) && - req.ArtistName != "" && resolved.ArtistName != "" && - artistsMatch(req.ArtistName, resolved.ArtistName) && - !(resolved.Duration <= 35 && expectedDurationSec > 45) { - GoLog("[%s] Accepted catalog duration difference for matching ISRC and recording names: expected %ds, got %ds\n", - logPrefix, expectedDurationSec, resolved.Duration) - return true - } - GoLog("[%s] Verification failed: duration mismatch — expected %ds, got %ds\n", - logPrefix, expectedDurationSec, resolved.Duration) - return false - } - } - - return true -} - -// selectBestMetadataEnrichmentTrack only returns a provider result when it is -// safe to copy missing tags into a download request. Search ordering alone is -// not evidence of identity: providers can put covers, remixes, or unrelated -// same-title recordings first. -func selectBestMetadataEnrichmentTrack(req DownloadRequest, tracks []ExtTrackMetadata) *ExtTrackMetadata { - var best *ExtTrackMetadata - bestScore := -1 << 30 - expectedISRC := strings.TrimSpace(req.ISRC) - - for i := range tracks { - track := &tracks[i] - candidateISRC := strings.TrimSpace(track.ISRC) - exactISRCMatch := expectedISRC != "" && candidateISRC != "" && - strings.EqualFold(expectedISRC, candidateISRC) - if expectedISRC != "" && candidateISRC != "" && !exactISRCMatch { - GoLog("[MetadataEnrichment] Rejected %s result with conflicting ISRC %s\n", track.ProviderID, candidateISRC) - continue - } - - resolved := resolvedTrackInfo{ - Title: track.Name, - ArtistName: track.Artists, - AlbumName: track.AlbumName, - ISRC: track.ISRC, - Duration: track.DurationMS / 1000, - } - if !trackMatchesRequest(req, resolved, "MetadataEnrichment") { - continue - } - if !exactISRCMatch && !hasStrongTrackIdentity(req, resolved) { - GoLog("[MetadataEnrichment] Rejected low-confidence result: %s - %s\n", track.Name, track.Artists) - continue - } - - score := 2000 - if exactISRCMatch { - score += 10000 - } - if exactLooseIdentityMatch(req.TrackName, track.Name, normalizeLooseTitle) { - score += 400 - } - if exactLooseIdentityMatch(req.ArtistName, track.Artists, normalizeLooseArtistName) { - score += 320 - } - if req.AlbumName != "" && track.AlbumName != "" && titlesMatch(req.AlbumName, track.AlbumName) { - score += 120 - } - if durationMatchesRequest(req, resolved) { - score += 80 - } - if track.ISRC != "" { - score += 40 - } - if track.AlbumName != "" { - score += 30 - } - if track.ReleaseDate != "" { - score += 30 - } - if track.TrackNumber > 0 { - score += 10 - } - - if best == nil || score > bestScore { - best = track - bestScore = score - } - } - - return best -} diff --git a/go_backend/title_match_utils_test.go b/go_backend/title_match_utils_test.go deleted file mode 100644 index 08f3960c..00000000 --- a/go_backend/title_match_utils_test.go +++ /dev/null @@ -1,405 +0,0 @@ -package gobackend - -import "testing" - -func TestNormalizeLooseTitle_Separators(t *testing.T) { - got := normalizeLooseTitle("Doctor / Cops") - if got != "doctor cops" { - t.Fatalf("expected doctor cops, got %q", got) - } - - got = normalizeLooseTitle("Doctor _ Cops") - if got != "doctor cops" { - t.Fatalf("expected doctor cops, got %q", got) - } -} - -func TestNormalizeLooseTitle_EmojiAndSymbols(t *testing.T) { - got := normalizeLooseTitle("Music Of The Spheres 🌎✨") - if got != "music of the spheres" { - t.Fatalf("expected music of the spheres, got %q", got) - } -} - -func TestTrackMatchesRequest_SongLinkBypassesArtistAndTitle(t *testing.T) { - req := DownloadRequest{ - TrackName: "Ringišpil", - ArtistName: "Djordje Balasevic", - } - resolved := resolvedTrackInfo{ - Title: "Completely Different Title", - ArtistName: "Totally Different Artist", - SkipNameVerification: true, - } - - if !trackMatchesRequest(req, resolved, "test") { - t.Fatal("expected SongLink-resolved track to bypass artist/title verification") - } -} - -func TestTrackMatchesRequest_SongLinkStillChecksDuration(t *testing.T) { - req := DownloadRequest{ - TrackName: "Ringišpil", - ArtistName: "Djordje Balasevic", - DurationMS: 180000, - } - resolved := resolvedTrackInfo{ - Title: "Completely Different Title", - ArtistName: "Totally Different Artist", - Duration: 240, - SkipNameVerification: true, - } - - if trackMatchesRequest(req, resolved, "test") { - t.Fatal("expected SongLink-resolved track with large duration mismatch to be rejected") - } -} - -func TestTrackMatchesRequestRejectsDifferentAlbumWithoutExactISRC(t *testing.T) { - req := DownloadRequest{ - TrackName: "Bewafa", - ArtistName: "Imran Khan", - AlbumName: "Unforgettable", - ISRC: "GBUM70901234", - } - resolved := resolvedTrackInfo{ - Title: "Bewafa", - ArtistName: "Imran Khan, Tarandeep Singh", - AlbumName: "Bewafa", - ISRC: "QZXYZ2600001", - } - - if trackMatchesRequest(req, resolved, "test") { - t.Fatal("expected same-title cover from a different album to be rejected") - } -} - -func TestTrackMatchesRequestAcceptsDifferentEditionWithExactISRC(t *testing.T) { - req := DownloadRequest{ - TrackName: "Song", - AlbumName: "Original Album", - ISRC: "USRC17607839", - } - resolved := resolvedTrackInfo{ - Title: "Song", - AlbumName: "Deluxe Collection", - ISRC: "usrc17607839", - } - - if !trackMatchesRequest(req, resolved, "test") { - t.Fatal("expected an exact ISRC match to accept another release edition") - } -} - -func TestTrackMatchesRequestAcceptsSameTrackFromDifferentRelease(t *testing.T) { - req := DownloadRequest{ - TrackName: "Crossing Field", - ArtistName: "LiSA", - AlbumName: "Crossing Field - EP", - DurationMS: 233000, - } - resolved := resolvedTrackInfo{ - Title: "Crossing Field", - ArtistName: "LiSA", - AlbumName: "LANDSPACE", - Duration: 233, - } - - if !trackMatchesRequest(req, resolved, "test") { - t.Fatal("expected the same track to be accepted across release albums") - } -} - -func TestTrackMatchesRequestRejectsAlbumMismatchForDifferentVersion(t *testing.T) { - req := DownloadRequest{ - TrackName: "Song (Live)", - ArtistName: "Artist", - AlbumName: "Live at the Theatre", - } - resolved := resolvedTrackInfo{ - Title: "Song", - ArtistName: "Artist", - AlbumName: "Studio Album", - } - - if trackMatchesRequest(req, resolved, "test") { - t.Fatal("expected an album mismatch to reject a different track version") - } -} - -func TestTrackMatchesRequestRejectsConflictingISRCDespiteStrongNames(t *testing.T) { - req := DownloadRequest{ - TrackName: "Song", - ArtistName: "Artist", - AlbumName: "Original Album", - ISRC: "USAAA2600001", - } - resolved := resolvedTrackInfo{ - Title: "Song", - ArtistName: "Artist", - AlbumName: "Other Album", - ISRC: "USAAA2600002", - } - - if trackMatchesRequest(req, resolved, "test") { - t.Fatal("expected conflicting ISRCs to keep album verification strict") - } -} - -func TestTrackMatchesRequestRejectsDurationMismatchAcrossReleases(t *testing.T) { - req := DownloadRequest{ - TrackName: "Crossing Field", - ArtistName: "LiSA", - AlbumName: "Crossing Field - EP", - DurationMS: 233000, - } - resolved := resolvedTrackInfo{ - Title: "Crossing Field", - ArtistName: "LiSA", - AlbumName: "LANDSPACE", - Duration: 280, - } - - if trackMatchesRequest(req, resolved, "test") { - t.Fatal("expected a large duration mismatch to reject another recording") - } -} - -func TestTitlesMatch_SeparatorVariants(t *testing.T) { - if !titlesMatch("Doctor / Cops", "Doctor _ Cops") { - t.Fatal("expected titlesMatch to accept / vs _ variant") - } -} - -func TestTrackMatchingPreservesArtistCreditBoundaries(t *testing.T) { - req := DownloadRequest{ - TrackName: "Signal - Remix", ArtistName: "Composer, Lead Singer & Lyric Writer", - AlbumName: "Original Soundtrack", DurationMS: 234000, - } - for _, artist := range []string{ - "Composer, Lead Singer, Guest Writer", - "Lead Singer & Composer", - "GUEST WRITER; LEAD SINGER", - } { - t.Run(artist, func(t *testing.T) { - resolved := resolvedTrackInfo{ - Title: "Signal (Remix)", ArtistName: artist, - AlbumName: "Original Soundtrack", Duration: 234, - } - if !trackMatchesRequest(req, resolved, "test") { - t.Fatal("matching recording rejected because contributor credits differ") - } - tracks := []ExtTrackMetadata{{ - Name: resolved.Title, Artists: artist, AlbumName: "Collection", - DurationMS: 234000, ProviderID: "provider", - }} - if selectBestMetadataEnrichmentTrack(req, tracks) == nil { - t.Fatal("matching recording rejected during metadata enrichment") - } - }) - } -} - -func TestTrackMetadataTolerancePreservesRecordingIdentity(t *testing.T) { - req := DownloadRequest{ - TrackName: "Signal - Remix", ArtistName: "Composer, Lead Singer & Lyric Writer", - AlbumName: "Original Soundtrack", DurationMS: 234000, - } - for _, tc := range []struct { - name string - title string - artist string - duration int - want bool - }{ - {"punctuation", "Signal (Remix)", req.ArtistName, 234, true}, - {"original", "Signal", req.ArtistName, 234, false}, - {"named mix", "Signal (Club Mix)", req.ArtistName, 234, false}, - {"live remix", "Signal (Remix Live)", req.ArtistName, 234, false}, - {"other artist", "Signal (Remix)", "Unrelated Singer", 234, false}, - {"other duration", "Signal (Remix)", "Composer, Lead Singer, Guest Writer", 305, false}, - } { - t.Run(tc.name, func(t *testing.T) { - got := trackMatchesRequest(req, resolvedTrackInfo{ - Title: tc.title, ArtistName: tc.artist, AlbumName: req.AlbumName, Duration: tc.duration, - }, "test") - if got != tc.want { - t.Fatalf("trackMatchesRequest = %v, want %v", got, tc.want) - } - }) - } -} - -func TestTrackIdentityIgnoresCreditAndSoundtrackAnnotations(t *testing.T) { - for _, tc := range []struct { - name string - expected string - found string - want bool - }{ - {"soundtrack", "Signal", `Signal (From "Original Soundtrack")`, true}, - {"mix credit", "Signal - Tiger Style Mix", "Signal (feat. Guest) [Tiger Style Mix]", true}, - {"mix soundtrack", "Signal - Tiger Style Mix", `Signal (Tiger Style Mix) [From "Original Soundtrack"]`, true}, - {"different mix", "Signal - Tiger Style Mix", "Signal (feat. Guest) [Club Mix]", false}, - {"original and mix", "Signal", "Signal (feat. Guest) [Tiger Style Mix]", false}, - {"unrelated title", "Signal", `Another Song (From "Signal")`, false}, - } { - t.Run(tc.name, func(t *testing.T) { - req := DownloadRequest{ - TrackName: tc.expected, ArtistName: "Composer, Singer & Writer", - AlbumName: "Original Soundtrack", DurationMS: 280000, - } - resolved := resolvedTrackInfo{ - Title: tc.found, ArtistName: "Singer & Composer", - AlbumName: "Collection", Duration: 283, - } - if got := trackMatchesRequest(req, resolved, "test"); got != tc.want { - t.Fatalf("trackMatchesRequest = %v, want %v", got, tc.want) - } - tracks := []ExtTrackMetadata{{ - Name: resolved.Title, Artists: resolved.ArtistName, AlbumName: resolved.AlbumName, - DurationMS: resolved.Duration * 1000, ProviderID: "provider", - }} - if got := selectBestMetadataEnrichmentTrack(req, tracks) != nil; got != tc.want { - t.Fatalf("metadata enrichment match = %v, want %v", got, tc.want) - } - resolved.Duration = 244 - if trackMatchesRequest(req, resolved, "test") { - t.Fatal("title annotations must not bypass a duration mismatch") - } - }) - } -} - -func TestTrackIdentityResolvesConflictingCatalogDurations(t *testing.T) { - for _, tc := range []struct { - title string - expected int - found int - }{ - {"Signal", 280000, 244}, - {"Signal (Tiger Style Mix)", 243000, 280}, - } { - t.Run(tc.title, func(t *testing.T) { - req := DownloadRequest{ - TrackName: tc.title, ArtistName: "Composer, Singer & Writer", AlbumName: "Soundtrack", - ISRC: "USAAA0000001", DurationMS: tc.expected, - } - resolved := resolvedTrackInfo{ - Title: tc.title, ArtistName: "Singer & Composer", AlbumName: "Collection", - ISRC: req.ISRC, Duration: tc.found, - } - if !trackMatchesRequest(req, resolved, "test") { - t.Fatal("matching ISRC and recording names should resolve inconsistent catalog durations") - } - for _, isrc := range []string{"", "USAAA0000002"} { - resolved.ISRC = isrc - if trackMatchesRequest(req, resolved, "test") { - t.Fatal("duration discrepancy requires an exact ISRC") - } - } - resolved.ISRC = req.ISRC - resolved.Duration = 30 - if trackMatchesRequest(req, resolved, "test") { - t.Fatal("a preview must not qualify as a catalog duration discrepancy") - } - resolved.Duration = tc.found - resolved.Title += " (Live)" - if trackMatchesRequest(req, resolved, "test") { - t.Fatal("inconsistent names and durations must not qualify on ISRC alone") - } - }) - } -} - -func TestTitlesMatch_EmojiStrict(t *testing.T) { - if titlesMatch("🪐", "Higher Power") { - t.Fatal("expected emoji title not to match unrelated textual title") - } - if !titlesMatch("🪐", "🪐") { - t.Fatal("expected identical emoji titles to match") - } -} - -func TestSelectBestMetadataEnrichmentTrackSkipsWrongFirstResult(t *testing.T) { - req := DownloadRequest{ - TrackName: "Song", - ArtistName: "Original Artist", - DurationMS: 180000, - } - tracks := []ExtTrackMetadata{ - { - Name: "Song", - Artists: "Cover Band", - AlbumName: "Covers", - DurationMS: 180000, - ProviderID: "first", - }, - { - Name: "Song", - Artists: "Original Artist", - AlbumName: "Original Album", - ReleaseDate: "2026-01-01", - ISRC: "USAAA2600001", - DurationMS: 180000, - ProviderID: "second", - }, - } - - best := selectBestMetadataEnrichmentTrack(req, tracks) - if best == nil || best.ProviderID != "second" { - t.Fatalf("best metadata match = %#v, want second result", best) - } -} - -func TestSelectBestMetadataEnrichmentTrackRejectsConflictingISRC(t *testing.T) { - req := DownloadRequest{ - TrackName: "Song", - ArtistName: "Artist", - ISRC: "USAAA2600001", - } - tracks := []ExtTrackMetadata{{ - Name: "Song", - Artists: "Artist", - AlbumName: "Album", - ISRC: "USAAA2600002", - ProviderID: "provider", - }} - - if best := selectBestMetadataEnrichmentTrack(req, tracks); best != nil { - t.Fatalf("expected conflicting ISRC to be rejected, got %#v", best) - } -} - -func TestSelectBestMetadataEnrichmentTrackAcceptsExactISRC(t *testing.T) { - req := DownloadRequest{ - TrackName: "Localized Song Name", - ArtistName: "Localized Artist Name", - ISRC: "USAAA2600001", - } - tracks := []ExtTrackMetadata{{ - Name: "Original Song Name", - Artists: "Original Artist Name", - AlbumName: "Album", - ISRC: "usaaa2600001", - ProviderID: "provider", - }} - - if best := selectBestMetadataEnrichmentTrack(req, tracks); best == nil { - t.Fatal("expected exact ISRC to provide a confident metadata match") - } -} - -func TestSelectBestMetadataEnrichmentTrackRejectsWeakArtistMatch(t *testing.T) { - req := DownloadRequest{TrackName: "Song", ArtistName: "Artist"} - tracks := []ExtTrackMetadata{{ - Name: "Song", - Artists: "Artist feat. Someone Else", - AlbumName: "Album", - ProviderID: "provider", - }} - - if best := selectBestMetadataEnrichmentTrack(req, tracks); best != nil { - t.Fatalf("expected fuzzy artist match without duration to be rejected, got %#v", best) - } -} diff --git a/go_backend/wav_aiff.go b/go_backend/wav_aiff.go deleted file mode 100644 index 2eb97e98..00000000 --- a/go_backend/wav_aiff.go +++ /dev/null @@ -1,1012 +0,0 @@ -package gobackend - -// WAV (RIFF) and AIFF/AIFC support: quality probing, tag reading/writing, and -// cover-art extraction. These containers are not handled by go-flac, so chunks -// are parsed/written by hand here. -// -// Tags are stored as an embedded ID3v2.4 tag (UTF-8): WAV uses a lowercase -// "id3 " chunk, AIFF uses an uppercase "ID3 " chunk. ID3v2.4 is chosen because -// the existing ID3 reader (parseID3v23Frames with version=4) reads synchsafe -// frame sizes and UTF-8 text, so anything we write is read back losslessly. -// -// Reading also recognises a WAV "LIST"/"INFO" block as a fallback for files -// that carry only RIFF INFO tags (common from other taggers). - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "math" - "os" - "path/filepath" - "strconv" - "strings" -) - -// WAVQuality / AIFFQuality mirror the other GetXQuality result shapes. -type WAVQuality struct { - SampleRate int - BitDepth int - Channels int - Duration int -} - -const ( - wavMaxMetaChunk = 16 * 1024 * 1024 // safety cap for buffering a metadata chunk - id3ChunkWAV = "id3 " - id3ChunkAIFF = "ID3 " - // Other format tags for reference: 0x0001 PCM, 0x0003 IEEE float. - wavFormatExtensn = 0xFFFE -) - -func putUint32(dst []byte, le bool, v uint32) { - if le { - binary.LittleEndian.PutUint32(dst, v) - } else { - binary.BigEndian.PutUint32(dst, v) - } -} - -func readUint32(b []byte, le bool) uint32 { - if le { - return binary.LittleEndian.Uint32(b) - } - return binary.BigEndian.Uint32(b) -} - -func synchsafeEncode(n int) []byte { - return []byte{ - byte((n >> 21) & 0x7f), - byte((n >> 14) & 0x7f), - byte((n >> 7) & 0x7f), - byte(n & 0x7f), - } -} - -func synchsafeDecode(b []byte) int { - if len(b) < 4 { - return 0 - } - return int(b[0])<<21 | int(b[1])<<14 | int(b[2])<<7 | int(b[3]) -} - -// parseExtendedFloat80 decodes an 80-bit IEEE 754 extended float (used by the -// AIFF COMM chunk for the sample rate). -func parseExtendedFloat80(b []byte) float64 { - if len(b) < 10 { - return 0 - } - sign := 1.0 - if b[0]&0x80 != 0 { - sign = -1.0 - } - exponent := int(b[0]&0x7f)<<8 | int(b[1]) - var mantissa uint64 - for i := 2; i < 10; i++ { - mantissa = mantissa<<8 | uint64(b[i]) - } - if exponent == 0 && mantissa == 0 { - return 0 - } - return sign * float64(mantissa) * math.Pow(2, float64(exponent-16383-63)) -} - -type wavProbe struct { - sampleRate int - bitDepth int - channels int - byteRate int - dataSize int64 - id3 []byte - info map[string]string -} - -// streamProbeWAV walks the top-level RIFF chunks, buffering only the small -// metadata chunks (fmt/id3/LIST) and skipping the large data chunk. -func streamProbeWAV(f *os.File) (*wavProbe, error) { - header := make([]byte, 12) - if _, err := io.ReadFull(f, header); err != nil { - return nil, err - } - if string(header[0:4]) != "RIFF" || string(header[8:12]) != "WAVE" { - return nil, fmt.Errorf("not a WAVE file") - } - - p := &wavProbe{info: map[string]string{}} - hdr := make([]byte, 8) - for { - if _, err := io.ReadFull(f, hdr); err != nil { - break - } - id := string(hdr[0:4]) - size := readUint32(hdr[4:8], true) - pad := int64(size) & 1 - - switch id { - case "fmt ": - buf := make([]byte, size) - if _, err := io.ReadFull(f, buf); err != nil { - return p, nil - } - if len(buf) >= 16 { - format := binary.LittleEndian.Uint16(buf[0:2]) - p.channels = int(binary.LittleEndian.Uint16(buf[2:4])) - p.sampleRate = int(binary.LittleEndian.Uint32(buf[4:8])) - p.byteRate = int(binary.LittleEndian.Uint32(buf[8:12])) - p.bitDepth = int(binary.LittleEndian.Uint16(buf[14:16])) - if format == wavFormatExtensn && len(buf) >= 26 { - // Valid bits per sample lives in the extension; the real - // PCM format tag is in the GUID, but bitDepth from the - // container field is sufficient for display. - if vb := int(binary.LittleEndian.Uint16(buf[18:20])); vb > 0 { - p.bitDepth = vb - } - } - } - if pad == 1 { - f.Seek(pad, io.SeekCurrent) - } - case "data": - p.dataSize = int64(size) - f.Seek(int64(size)+pad, io.SeekCurrent) - case id3ChunkWAV, "ID3 ": - if size > 0 && size <= wavMaxMetaChunk { - buf := make([]byte, size) - if _, err := io.ReadFull(f, buf); err == nil { - p.id3 = buf - } - if pad == 1 { - f.Seek(pad, io.SeekCurrent) - } - } else { - f.Seek(int64(size)+pad, io.SeekCurrent) - } - case "LIST": - if size > 0 && size <= wavMaxMetaChunk { - buf := make([]byte, size) - if _, err := io.ReadFull(f, buf); err == nil { - parseRIFFInfo(buf, p.info) - } - if pad == 1 { - f.Seek(pad, io.SeekCurrent) - } - } else { - f.Seek(int64(size)+pad, io.SeekCurrent) - } - default: - f.Seek(int64(size)+pad, io.SeekCurrent) - } - } - return p, nil -} - -// parseRIFFInfo reads a LIST/INFO block ("INFO" + sub-chunks like INAM, IART). -func parseRIFFInfo(buf []byte, out map[string]string) { - if len(buf) < 4 || string(buf[0:4]) != "INFO" { - return - } - pos := 4 - for pos+8 <= len(buf) { - id := string(buf[pos : pos+4]) - size := int(binary.LittleEndian.Uint32(buf[pos+4 : pos+8])) - pos += 8 - if size <= 0 || pos+size > len(buf) { - break - } - val := strings.TrimRight(string(buf[pos:pos+size]), "\x00") - out[id] = strings.TrimSpace(val) - pos += size - if size&1 == 1 { - pos++ - } - } -} - -func wavMetadataFromProbe(p *wavProbe) *AudioMetadata { - if p == nil { - return nil - } - if len(p.id3) > 0 { - if meta, err := readID3v2FromBytes(p.id3); err == nil && meta != nil && - (meta.Title != "" || meta.Artist != "" || meta.Album != "") { - return meta - } - } - if len(p.info) > 0 { - meta := &AudioMetadata{ - Title: p.info["INAM"], - Artist: p.info["IART"], - Album: p.info["IPRD"], - Genre: cleanGenre(p.info["IGNR"]), - Date: p.info["ICRD"], - Comment: p.info["ICMT"], - Copyright: p.info["ICOP"], - Composer: p.info["IMUS"], - } - if n, err := strconv.Atoi(strings.TrimSpace(p.info["ITRK"])); err == nil { - meta.TrackNumber = n - } - if meta.Date != "" && len(meta.Date) >= 4 { - meta.Year = meta.Date[:4] - } - if meta.Title != "" || meta.Artist != "" || meta.Album != "" { - return meta - } - } - return nil -} - -// GetWAVQuality probes PCM parameters and computes duration from the data size. -func GetWAVQuality(filePath string) (*WAVQuality, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer f.Close() - - p, err := streamProbeWAV(f) - if err != nil { - return nil, err - } - q := &WAVQuality{ - SampleRate: p.sampleRate, - BitDepth: p.bitDepth, - Channels: p.channels, - } - if p.byteRate > 0 && p.dataSize > 0 { - q.Duration = int(p.dataSize / int64(p.byteRate)) - } else if p.sampleRate > 0 && p.channels > 0 && p.bitDepth > 0 && p.dataSize > 0 { - bytesPerSec := int64(p.sampleRate * p.channels * p.bitDepth / 8) - if bytesPerSec > 0 { - q.Duration = int(p.dataSize / bytesPerSec) - } - } - return q, nil -} - -// ReadWAVTags reads tags from a WAV file (ID3 chunk preferred, RIFF INFO fallback). -func ReadWAVTags(filePath string) (*AudioMetadata, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer f.Close() - - p, err := streamProbeWAV(f) - if err != nil { - return nil, err - } - meta := wavMetadataFromProbe(p) - if meta == nil { - return nil, fmt.Errorf("no WAV tags found") - } - return meta, nil -} - -type aiffProbe struct { - sampleRate int - bitDepth int - channels int - numFrames int64 - id3 []byte - nameChunk string - authChunk string - annoChunk string - copyrightChunk string -} - -func streamProbeAIFF(f *os.File) (*aiffProbe, error) { - header := make([]byte, 12) - if _, err := io.ReadFull(f, header); err != nil { - return nil, err - } - form := string(header[8:12]) - if string(header[0:4]) != "FORM" || (form != "AIFF" && form != "AIFC") { - return nil, fmt.Errorf("not an AIFF file") - } - - p := &aiffProbe{} - hdr := make([]byte, 8) - for { - if _, err := io.ReadFull(f, hdr); err != nil { - break - } - id := string(hdr[0:4]) - size := readUint32(hdr[4:8], false) - pad := int64(size) & 1 - - switch id { - case "COMM": - buf := make([]byte, size) - if _, err := io.ReadFull(f, buf); err != nil { - return p, nil - } - if len(buf) >= 18 { - p.channels = int(binary.BigEndian.Uint16(buf[0:2])) - p.numFrames = int64(binary.BigEndian.Uint32(buf[2:6])) - p.bitDepth = int(binary.BigEndian.Uint16(buf[6:8])) - p.sampleRate = int(parseExtendedFloat80(buf[8:18]) + 0.5) - } - if pad == 1 { - f.Seek(pad, io.SeekCurrent) - } - case id3ChunkAIFF, "id3 ": - if size > 0 && size <= wavMaxMetaChunk { - buf := make([]byte, size) - if _, err := io.ReadFull(f, buf); err == nil { - p.id3 = buf - } - if pad == 1 { - f.Seek(pad, io.SeekCurrent) - } - } else { - f.Seek(int64(size)+pad, io.SeekCurrent) - } - case "NAME", "AUTH", "ANNO", "(c) ": - if size > 0 && size <= wavMaxMetaChunk { - buf := make([]byte, size) - if _, err := io.ReadFull(f, buf); err == nil { - val := strings.TrimRight(strings.TrimSpace(string(buf)), "\x00") - switch id { - case "NAME": - p.nameChunk = val - case "AUTH": - p.authChunk = val - case "ANNO": - p.annoChunk = val - case "(c) ": - p.copyrightChunk = val - } - } - if pad == 1 { - f.Seek(pad, io.SeekCurrent) - } - } else { - f.Seek(int64(size)+pad, io.SeekCurrent) - } - default: - f.Seek(int64(size)+pad, io.SeekCurrent) - } - } - return p, nil -} - -func aiffMetadataFromProbe(p *aiffProbe) *AudioMetadata { - if p == nil { - return nil - } - if len(p.id3) > 0 { - if meta, err := readID3v2FromBytes(p.id3); err == nil && meta != nil && - (meta.Title != "" || meta.Artist != "" || meta.Album != "") { - return meta - } - } - if p.nameChunk != "" || p.authChunk != "" { - meta := &AudioMetadata{ - Title: p.nameChunk, - Artist: p.authChunk, - Comment: p.annoChunk, - Copyright: p.copyrightChunk, - } - return meta - } - return nil -} - -// GetAIFFQuality probes PCM parameters and computes duration from frame count. -func GetAIFFQuality(filePath string) (*WAVQuality, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer f.Close() - - p, err := streamProbeAIFF(f) - if err != nil { - return nil, err - } - q := &WAVQuality{ - SampleRate: p.sampleRate, - BitDepth: p.bitDepth, - Channels: p.channels, - } - if p.sampleRate > 0 && p.numFrames > 0 { - q.Duration = int(p.numFrames / int64(p.sampleRate)) - } - return q, nil -} - -// ReadAIFFTags reads tags from an AIFF file (ID3 chunk preferred, AIFF text chunks fallback). -func ReadAIFFTags(filePath string) (*AudioMetadata, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer f.Close() - - p, err := streamProbeAIFF(f) - if err != nil { - return nil, err - } - meta := aiffMetadataFromProbe(p) - if meta == nil { - return nil, fmt.Errorf("no AIFF tags found") - } - return meta, nil -} - -// readID3v2FromBytes parses an in-memory ID3v2 tag (the contents of a WAV "id3 " -// or AIFF "ID3 " chunk) by reusing the existing frame parsers. -func readID3v2FromBytes(data []byte) (*AudioMetadata, error) { - if len(data) < 10 || string(data[0:3]) != "ID3" { - return nil, fmt.Errorf("no ID3v2 header") - } - majorVersion := data[3] - flags := data[5] - unsync := (flags & 0x80) != 0 - extendedHeader := (flags & 0x40) != 0 - footerPresent := (flags & 0x10) != 0 - - size := synchsafeDecode(data[6:10]) - if size <= 0 || 10+size > len(data) { - size = len(data) - 10 - } - tagData := data[10 : 10+size] - - if footerPresent && len(tagData) >= 10 { - footerStart := len(tagData) - 10 - if footerStart >= 0 && string(tagData[footerStart:footerStart+3]) == "3DI" { - tagData = tagData[:footerStart] - } - } - if extendedHeader { - if skip := extendedHeaderSize(tagData, majorVersion); skip > 0 && skip < len(tagData) { - tagData = tagData[skip:] - } - } - - metadata := &AudioMetadata{} - if majorVersion == 2 { - parseID3v22Frames(tagData, metadata, unsync) - } else { - parseID3v23Frames(tagData, metadata, majorVersion, unsync) - } - return metadata, nil -} - -// extractAPICFromID3 returns the first embedded picture (APIC/PIC) and its MIME. -func extractAPICFromID3(tag []byte) ([]byte, string) { - if len(tag) < 10 || string(tag[0:3]) != "ID3" { - return nil, "" - } - ver := tag[3] - size := synchsafeDecode(tag[6:10]) - if size <= 0 || 10+size > len(tag) { - size = len(tag) - 10 - } - data := tag[10 : 10+size] - - pos := 0 - for { - if ver == 2 { - if pos+6 > len(data) || data[pos] == 0 { - break - } - id := string(data[pos : pos+3]) - fsz := int(data[pos+3])<<16 | int(data[pos+4])<<8 | int(data[pos+5]) - if fsz <= 0 || pos+6+fsz > len(data) { - break - } - if id == "PIC" { - return parseAPICFrame(data[pos+6:pos+6+fsz], ver) - } - pos += 6 + fsz - continue - } - - if pos+10 > len(data) || data[pos] == 0 { - break - } - id := string(data[pos : pos+4]) - var fsz int - if ver == 4 { - fsz = synchsafeDecode(data[pos+4 : pos+8]) - } else { - fsz = int(binary.BigEndian.Uint32(data[pos+4 : pos+8])) - } - if fsz <= 0 || pos+10+fsz > len(data) { - break - } - if id == "APIC" { - return parseAPICFrame(data[pos+10:pos+10+fsz], ver) - } - pos += 10 + fsz - } - return nil, "" -} - -// buildID3v24Tag builds a UTF-8 ID3v2.4 tag from metadata plus optional cover. -func buildID3v24Tag(meta *AudioMetadata, coverData []byte, coverMIME string) []byte { - var frames bytes.Buffer - - writeFrame := func(id string, payload []byte) { - frames.WriteString(id) - frames.Write(synchsafeEncode(len(payload))) - frames.Write([]byte{0, 0}) - frames.Write(payload) - } - writeText := func(id, val string) { - if strings.TrimSpace(val) == "" { - return - } - payload := append([]byte{0x03}, []byte(val)...) - writeFrame(id, payload) - } - - writeText("TIT2", meta.Title) - writeText("TPE1", meta.Artist) - writeText("TALB", meta.Album) - writeText("TPE2", meta.AlbumArtist) - writeText("TCON", meta.Genre) - writeText("TCOM", meta.Composer) - writeText("TPUB", meta.Label) - writeText("TCOP", meta.Copyright) - writeText("TSRC", meta.ISRC) - - date := meta.Date - if date == "" { - date = meta.Year - } - writeText("TDRC", date) - - if meta.TrackNumber > 0 { - if meta.TotalTracks > 0 { - writeText("TRCK", fmt.Sprintf("%d/%d", meta.TrackNumber, meta.TotalTracks)) - } else { - writeText("TRCK", strconv.Itoa(meta.TrackNumber)) - } - } - if meta.DiscNumber > 0 { - if meta.TotalDiscs > 0 { - writeText("TPOS", fmt.Sprintf("%d/%d", meta.DiscNumber, meta.TotalDiscs)) - } else { - writeText("TPOS", strconv.Itoa(meta.DiscNumber)) - } - } - - if strings.TrimSpace(meta.Comment) != "" { - // COMM: encoding + language(3) + short desc(null) + text - payload := []byte{0x03} - payload = append(payload, []byte("eng")...) - payload = append(payload, 0x00) // empty description - payload = append(payload, []byte(meta.Comment)...) - writeFrame("COMM", payload) - } - if strings.TrimSpace(meta.Lyrics) != "" { - payload := []byte{0x03} - payload = append(payload, []byte("eng")...) - payload = append(payload, 0x00) - payload = append(payload, []byte(meta.Lyrics)...) - writeFrame("USLT", payload) - } - - // ReplayGain as TXXX (description\0value), UTF-8. - writeTXXX := func(desc, val string) { - if strings.TrimSpace(val) == "" { - return - } - payload := []byte{0x03} - payload = append(payload, []byte(desc)...) - payload = append(payload, 0x00) - payload = append(payload, []byte(val)...) - writeFrame("TXXX", payload) - } - writeTXXX("REPLAYGAIN_TRACK_GAIN", meta.ReplayGainTrackGain) - writeTXXX("REPLAYGAIN_TRACK_PEAK", meta.ReplayGainTrackPeak) - writeTXXX("REPLAYGAIN_ALBUM_GAIN", meta.ReplayGainAlbumGain) - writeTXXX("REPLAYGAIN_ALBUM_PEAK", meta.ReplayGainAlbumPeak) - if meta.Explicit { - writeTXXX("ITUNESADVISORY", "1") - } - writeTXXX("RELEASETYPE", meta.AlbumType) - writeTXXX("BARCODE", meta.UPC) - if strings.EqualFold(strings.TrimSpace(meta.AlbumType), "compilation") { - writeText("TCMP", "1") - } - - if len(coverData) > 0 { - if strings.TrimSpace(coverMIME) == "" { - coverMIME = "image/jpeg" - } - // APIC: encoding + mime(null) + picture-type(0x03 front) + desc(null) + data - payload := []byte{0x03} - payload = append(payload, []byte(coverMIME)...) - payload = append(payload, 0x00) - payload = append(payload, 0x03) - payload = append(payload, 0x00) - payload = append(payload, coverData...) - writeFrame("APIC", payload) - } - - body := frames.Bytes() - var out bytes.Buffer - out.WriteString("ID3") - out.Write([]byte{0x04, 0x00}) // v2.4.0 - out.WriteByte(0x00) // flags - out.Write(synchsafeEncode(len(body))) - out.Write(body) - return out.Bytes() -} - -// writeID3Chunk rewrites filePath, replacing any existing tag chunk (chunkID, -// matched case-insensitively) with a fresh ID3v2.4 chunk appended at the end. -// The audio data and all other chunks are preserved; container size is patched. -func writeID3Chunk(filePath, expectMagic, chunkID string, le bool, id3 []byte) error { - in, err := os.Open(filePath) - if err != nil { - return err - } - defer in.Close() - - header := make([]byte, 12) - if _, err := io.ReadFull(in, header); err != nil { - return err - } - if string(header[0:4]) != expectMagic { - return fmt.Errorf("unexpected container magic %q", string(header[0:4])) - } - - tmpPath := filePath + ".tagtmp" - out, err := os.Create(tmpPath) - if err != nil { - return err - } - cleanup := func() { - out.Close() - os.Remove(tmpPath) - } - - if _, err := out.Write(header); err != nil { - cleanup() - return err - } - - var bodyLen int64 = 4 // the 4-byte form type after the size field - hdr := make([]byte, 8) - for { - n, rerr := io.ReadFull(in, hdr) - if n < 8 { - break - } - if rerr != nil { - break - } - id := string(hdr[0:4]) - size := readUint32(hdr[4:8], le) - pad := int64(size) & 1 - - if strings.EqualFold(id, chunkID) { - if _, err := in.Seek(int64(size)+pad, io.SeekCurrent); err != nil { - cleanup() - return err - } - continue - } - - if _, err := out.Write(hdr); err != nil { - cleanup() - return err - } - if _, err := io.CopyN(out, in, int64(size)+pad); err != nil { - cleanup() - return err - } - bodyLen += 8 + int64(size) + pad - } - - newSize := len(id3) - chunkHdr := make([]byte, 8) - copy(chunkHdr[0:4], chunkID) - putUint32(chunkHdr[4:8], le, uint32(newSize)) - if _, err := out.Write(chunkHdr); err != nil { - cleanup() - return err - } - if _, err := out.Write(id3); err != nil { - cleanup() - return err - } - if newSize&1 == 1 { - if _, err := out.Write([]byte{0}); err != nil { - cleanup() - return err - } - } - bodyLen += 8 + int64(newSize) + int64(newSize&1) - - // Patch the container size field (bytes 4..8). - sizeBuf := make([]byte, 4) - putUint32(sizeBuf, le, uint32(bodyLen)) - if _, err := out.WriteAt(sizeBuf, 4); err != nil { - cleanup() - return err - } - - if err := out.Sync(); err != nil { - out.Close() - os.Remove(tmpPath) - return err - } - if err := out.Close(); err != nil { - os.Remove(tmpPath) - return err - } - in.Close() - - if err := os.Rename(tmpPath, filePath); err != nil { - os.Remove(tmpPath) - return err - } - syncDir(filepath.Dir(filePath)) - return nil -} - -func loadCoverForTag(fields map[string]string) ([]byte, string, error) { - coverPath := strings.TrimSpace(fields["cover_path"]) - if coverPath == "" { - return nil, "", nil - } - data, err := os.ReadFile(coverPath) - if err != nil { - return nil, "", fmt.Errorf("read cover art: %w", err) - } - if len(data) == 0 { - return nil, "", fmt.Errorf("cover art is empty") - } - mime := "image/jpeg" - switch { - case len(data) >= 8 && data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47: - mime = "image/png" - case len(data) >= 12 && string(data[0:4]) == "RIFF" && string(data[8:12]) == "WEBP": - mime = "image/webp" - case len(data) >= 6 && (string(data[0:6]) == "GIF87a" || string(data[0:6]) == "GIF89a"): - mime = "image/gif" - default: - switch strings.ToLower(filepath.Ext(coverPath)) { - case ".png": - mime = "image/png" - case ".webp": - mime = "image/webp" - case ".gif": - mime = "image/gif" - } - } - return data, mime, nil -} - -func audioMetadataFromEditFields(fields map[string]string) *AudioMetadata { - atoi := func(k string) int { - n := 0 - if v, ok := fields[k]; ok && strings.TrimSpace(v) != "" { - fmt.Sscanf(strings.TrimSpace(v), "%d", &n) - } - return n - } - return &AudioMetadata{ - Title: fields["title"], - Artist: fields["artist"], - Album: fields["album"], - AlbumArtist: fields["album_artist"], - Date: fields["date"], - TrackNumber: atoi("track_number"), - TotalTracks: atoi("track_total"), - DiscNumber: atoi("disc_number"), - TotalDiscs: atoi("disc_total"), - ISRC: fields["isrc"], - Lyrics: fields["lyrics"], - Genre: fields["genre"], - Label: fields["label"], - Copyright: fields["copyright"], - Composer: fields["composer"], - Comment: fields["comment"], - AlbumType: fields["album_type"], - Explicit: isTruthyTagValue(fields["explicit"]), - UPC: fields["upc"], - ReplayGainTrackGain: fields["replaygain_track_gain"], - ReplayGainTrackPeak: fields["replaygain_track_peak"], - ReplayGainAlbumGain: fields["replaygain_album_gain"], - ReplayGainAlbumPeak: fields["replaygain_album_peak"], - } -} - -// mergeWAVEditFields merges edit fields onto existing tags so untouched fields -// (and cover art, when no new cover is provided) are preserved. -func mergeEditFieldsOntoExisting(existing *AudioMetadata, fields map[string]string) *AudioMetadata { - meta := audioMetadataFromEditFields(fields) - if existing == nil { - return meta - } - // Only overwrite fields that are present as keys in the edit set; otherwise - // keep the existing value. An empty value with the key present clears it. - keep := func(key, newVal, oldVal string) string { - if _, ok := fields[key]; ok { - return newVal - } - return oldVal - } - meta.Title = keep("title", meta.Title, existing.Title) - meta.Artist = keep("artist", meta.Artist, existing.Artist) - meta.Album = keep("album", meta.Album, existing.Album) - meta.AlbumArtist = keep("album_artist", meta.AlbumArtist, existing.AlbumArtist) - meta.Genre = keep("genre", meta.Genre, existing.Genre) - meta.Composer = keep("composer", meta.Composer, existing.Composer) - meta.Label = keep("label", meta.Label, existing.Label) - meta.Copyright = keep("copyright", meta.Copyright, existing.Copyright) - meta.ISRC = keep("isrc", meta.ISRC, existing.ISRC) - meta.Lyrics = keep("lyrics", meta.Lyrics, existing.Lyrics) - meta.Comment = keep("comment", meta.Comment, existing.Comment) - meta.AlbumType = keep("album_type", meta.AlbumType, existing.AlbumType) - meta.UPC = keep("upc", meta.UPC, existing.UPC) - if _, ok := fields["explicit"]; !ok { - meta.Explicit = existing.Explicit - } - meta.Date = keep("date", meta.Date, existing.Date) - if _, ok := fields["track_number"]; !ok { - meta.TrackNumber = existing.TrackNumber - } - if _, ok := fields["track_total"]; !ok { - meta.TotalTracks = existing.TotalTracks - } - if _, ok := fields["disc_number"]; !ok { - meta.DiscNumber = existing.DiscNumber - } - if _, ok := fields["disc_total"]; !ok { - meta.TotalDiscs = existing.TotalDiscs - } - if _, ok := fields["replaygain_track_gain"]; !ok { - meta.ReplayGainTrackGain = existing.ReplayGainTrackGain - } - if _, ok := fields["replaygain_track_peak"]; !ok { - meta.ReplayGainTrackPeak = existing.ReplayGainTrackPeak - } - if _, ok := fields["replaygain_album_gain"]; !ok { - meta.ReplayGainAlbumGain = existing.ReplayGainAlbumGain - } - if _, ok := fields["replaygain_album_peak"]; !ok { - meta.ReplayGainAlbumPeak = existing.ReplayGainAlbumPeak - } - return meta -} - -// WriteWAVTags writes/merges tags into a WAV file's "id3 " chunk. -func WriteWAVTags(filePath string, fields map[string]string) error { - existing, _ := ReadWAVTags(filePath) - meta := mergeEditFieldsOntoExisting(existing, fields) - - coverData, coverMIME, err := loadCoverForTag(fields) - if err != nil { - return err - } - if coverData == nil { - // Preserve an existing embedded cover when no new one is supplied. - if f, err := os.Open(filePath); err == nil { - if p, perr := streamProbeWAV(f); perr == nil && len(p.id3) > 0 { - coverData, coverMIME = extractAPICFromID3(p.id3) - } - f.Close() - } - } - - tag := buildID3v24Tag(meta, coverData, coverMIME) - return writeID3Chunk(filePath, "RIFF", id3ChunkWAV, true, tag) -} - -// WriteAIFFTags writes/merges tags into an AIFF file's "ID3 " chunk. -func WriteAIFFTags(filePath string, fields map[string]string) error { - existing, _ := ReadAIFFTags(filePath) - meta := mergeEditFieldsOntoExisting(existing, fields) - - coverData, coverMIME, err := loadCoverForTag(fields) - if err != nil { - return err - } - if coverData == nil { - if f, err := os.Open(filePath); err == nil { - if p, perr := streamProbeAIFF(f); perr == nil && len(p.id3) > 0 { - coverData, coverMIME = extractAPICFromID3(p.id3) - } - f.Close() - } - } - - tag := buildID3v24Tag(meta, coverData, coverMIME) - return writeID3Chunk(filePath, "FORM", id3ChunkAIFF, false, tag) -} - -func scanWAVFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - if metadata, err := ReadWAVTags(filePath); err == nil && metadata != nil { - applyAudioMetadataToScan(metadata, result) - } - if quality, err := GetWAVQuality(filePath); err == nil && quality != nil { - result.BitDepth = quality.BitDepth - result.SampleRate = quality.SampleRate - result.Duration = quality.Duration - } - result.Bitrate = 0 // lossless PCM - result.Format = "wav" - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - return result, nil -} - -func scanAIFFFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - if metadata, err := ReadAIFFTags(filePath); err == nil && metadata != nil { - applyAudioMetadataToScan(metadata, result) - } - if quality, err := GetAIFFQuality(filePath); err == nil && quality != nil { - result.BitDepth = quality.BitDepth - result.SampleRate = quality.SampleRate - result.Duration = quality.Duration - } - result.Bitrate = 0 // lossless PCM - result.Format = "aiff" - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - return result, nil -} - -func applyAudioMetadataToScan(metadata *AudioMetadata, result *LibraryScanResult) { - result.TrackName = metadata.Title - result.ArtistName = metadata.Artist - result.AlbumName = metadata.Album - result.AlbumArtist = metadata.AlbumArtist - result.ISRC = metadata.ISRC - result.TrackNumber = metadata.TrackNumber - result.TotalTracks = metadata.TotalTracks - result.DiscNumber = metadata.DiscNumber - result.TotalDiscs = metadata.TotalDiscs - if metadata.Date != "" { - result.ReleaseDate = metadata.Date - } else { - result.ReleaseDate = metadata.Year - } - result.Genre = metadata.Genre - result.Composer = metadata.Composer - result.Label = metadata.Label - result.Copyright = metadata.Copyright - result.Comment = metadata.Comment - result.AlbumType = metadata.AlbumType - result.Explicit = metadata.Explicit - result.HasLyrics = rawLyricsHasUsableContent(metadata.Lyrics) - result.UPC = metadata.UPC -} - -// extractWAVAIFFCover returns embedded cover art (from the ID3 chunk) for a -// WAV or AIFF file, or an error when none is present. -func extractWAVAIFFCover(filePath string) ([]byte, string, error) { - ext := strings.ToLower(filepath.Ext(filePath)) - f, err := os.Open(filePath) - if err != nil { - return nil, "", err - } - defer f.Close() - - var id3 []byte - switch ext { - case ".aiff", ".aif", ".aifc": - if p, perr := streamProbeAIFF(f); perr == nil { - id3 = p.id3 - } - default: - if p, perr := streamProbeWAV(f); perr == nil { - id3 = p.id3 - } - } - if len(id3) == 0 { - return nil, "", fmt.Errorf("no embedded cover") - } - data, mime := extractAPICFromID3(id3) - if len(data) == 0 { - return nil, "", fmt.Errorf("no embedded cover") - } - return data, mime, nil -} diff --git a/go_backend/wav_aiff_test.go b/go_backend/wav_aiff_test.go deleted file mode 100644 index 0c60553c..00000000 --- a/go_backend/wav_aiff_test.go +++ /dev/null @@ -1,375 +0,0 @@ -package gobackend - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "os" - "os/exec" - "path/filepath" - "testing" -) - -func writeTestWAV(t *testing.T, path string) { - t.Helper() - var body bytes.Buffer - body.WriteString("WAVE") - - fmtData := make([]byte, 16) - binary.LittleEndian.PutUint16(fmtData[0:2], 1) - binary.LittleEndian.PutUint16(fmtData[2:4], 2) - binary.LittleEndian.PutUint32(fmtData[4:8], 44100) - binary.LittleEndian.PutUint32(fmtData[8:12], 44100*2*2) - binary.LittleEndian.PutUint16(fmtData[12:14], 4) - binary.LittleEndian.PutUint16(fmtData[14:16], 16) - writeTestRIFFChunk(&body, "fmt ", fmtData, true) - writeTestRIFFChunk(&body, "data", []byte{0, 0, 0, 0}, true) - - var out bytes.Buffer - out.WriteString("RIFF") - if err := binary.Write(&out, binary.LittleEndian, uint32(body.Len())); err != nil { - t.Fatal(err) - } - out.Write(body.Bytes()) - if err := os.WriteFile(path, out.Bytes(), 0600); err != nil { - t.Fatal(err) - } -} - -func writeTestAIFF(t *testing.T, path string) { - t.Helper() - var body bytes.Buffer - body.WriteString("AIFF") - - comm := make([]byte, 18) - binary.BigEndian.PutUint16(comm[0:2], 2) - binary.BigEndian.PutUint32(comm[2:6], 1) - binary.BigEndian.PutUint16(comm[6:8], 16) - copy(comm[8:18], []byte{0x40, 0x0e, 0xac, 0x44, 0, 0, 0, 0, 0, 0}) - writeTestRIFFChunk(&body, "COMM", comm, false) - writeTestRIFFChunk(&body, "SSND", make([]byte, 12), false) - - var out bytes.Buffer - out.WriteString("FORM") - if err := binary.Write(&out, binary.BigEndian, uint32(body.Len())); err != nil { - t.Fatal(err) - } - out.Write(body.Bytes()) - if err := os.WriteFile(path, out.Bytes(), 0600); err != nil { - t.Fatal(err) - } -} - -func writeTestRIFFChunk(out *bytes.Buffer, id string, data []byte, littleEndian bool) { - out.WriteString(id) - if littleEndian { - _ = binary.Write(out, binary.LittleEndian, uint32(len(data))) - } else { - _ = binary.Write(out, binary.BigEndian, uint32(len(data))) - } - out.Write(data) - if len(data)&1 == 1 { - out.WriteByte(0) - } -} - -func TestWAVAIFFMetadataAndCoverRoundTrip(t *testing.T) { - cover := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4} - fields := map[string]string{ - "title": "Judul Lagu", - "artist": "Artis", - "album": "Album", - "album_artist": "Album Artis", - "date": "2026-08-01", - "track_number": "3", - "track_total": "12", - "disc_number": "1", - "disc_total": "2", - "genre": "Pop", - "isrc": "TEST12345678", - "lyrics": "Baris pertama\nBaris kedua", - "label": "Label", - "copyright": "Copyright", - "composer": "Komposer", - "comment": "Komentar", - "replaygain_track_gain": "-5.00 dB", - "replaygain_track_peak": "0.987654", - "replaygain_album_gain": "-4.00 dB", - "replaygain_album_peak": "0.998877", - } - - formats := []struct { - name string - ext string - write func(*testing.T, string) - tags func(string) (*AudioMetadata, error) - edit func(string, map[string]string) error - method string - }{ - {name: "WAV", ext: ".wav", write: writeTestWAV, tags: ReadWAVTags, edit: WriteWAVTags, method: "native_wav"}, - {name: "AIFF", ext: ".aiff", write: writeTestAIFF, tags: ReadAIFFTags, edit: WriteAIFFTags, method: "native_aiff"}, - } - - for _, format := range formats { - format := format - t.Run(format.name, func(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "track"+format.ext) - coverPath := filepath.Join(dir, "cover.png") - format.write(t, path) - if err := os.WriteFile(coverPath, cover, 0600); err != nil { - t.Fatal(err) - } - - writeFields := make(map[string]string, len(fields)+1) - for key, value := range fields { - writeFields[key] = value - } - writeFields["cover_path"] = coverPath - metadataJSON, err := json.Marshal(writeFields) - if err != nil { - t.Fatal(err) - } - responseJSON, err := EditFileMetadata(path, string(metadataJSON)) - if err != nil { - t.Fatalf("EditFileMetadata: %v", err) - } - var response map[string]any - if err := json.Unmarshal([]byte(responseJSON), &response); err != nil { - t.Fatalf("decode EditFileMetadata response: %v", err) - } - if response["success"] != true || response["method"] != format.method { - t.Fatalf("EditFileMetadata response = %v", response) - } - - meta, err := format.tags(path) - if err != nil { - t.Fatalf("read tags: %v", err) - } - assertWAVAIFFTestMetadata(t, meta) - - extracted, mime, err := extractWAVAIFFCover(path) - if err != nil { - t.Fatalf("extract cover: %v", err) - } - if mime != "image/png" || !bytes.Equal(extracted, cover) { - t.Fatalf("cover = %q (%x), want image/png (%x)", mime, extracted, cover) - } - - outputCover := filepath.Join(dir, "extracted.bin") - if err := ExtractCoverToFile(path, outputCover); err != nil { - t.Fatalf("ExtractCoverToFile: %v", err) - } - outputBytes, err := os.ReadFile(outputCover) - if err != nil || !bytes.Equal(outputBytes, cover) { - t.Fatalf("ExtractCoverToFile bytes = %x, err=%v", outputBytes, err) - } - - // A partial Edit Metadata update must retain every untouched field and - // the existing cover instead of rebuilding a sparse tag. - if err := format.edit(path, map[string]string{"title": "Judul Baru"}); err != nil { - t.Fatalf("partial edit: %v", err) - } - meta, err = format.tags(path) - if err != nil { - t.Fatalf("read edited tags: %v", err) - } - if meta.Title != "Judul Baru" || meta.Artist != "Artis" || meta.TotalTracks != 12 { - t.Fatalf("partial edit lost metadata: %+v", meta) - } - extracted, _, err = extractWAVAIFFCover(path) - if err != nil || !bytes.Equal(extracted, cover) { - t.Fatalf("partial edit lost cover: %x, err=%v", extracted, err) - } - }) - } -} - -func TestWAVAIFFTagWriteRejectsMissingRequestedCover(t *testing.T) { - path := filepath.Join(t.TempDir(), "track.wav") - writeTestWAV(t, path) - before, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - - err = WriteWAVTags(path, map[string]string{ - "title": "Must not be partially written", - "cover_path": filepath.Join(t.TempDir(), "missing.jpg"), - }) - if err == nil { - t.Fatal("WriteWAVTags succeeded with an inaccessible requested cover") - } - after, readErr := os.ReadFile(path) - if readErr != nil { - t.Fatal(readErr) - } - if !bytes.Equal(before, after) { - t.Fatal("WAV changed even though requested cover could not be read") - } -} - -func TestWAVAIFFRemainExternallyDecodableAfterTagging(t *testing.T) { - ffprobePath, err := exec.LookPath("ffprobe") - if err != nil { - t.Skip("ffprobe not available") - } - if _, err := exec.LookPath("ffmpeg"); err != nil { - t.Skip("ffmpeg not available") - } - - dir := t.TempDir() - coverPath := filepath.Join(dir, "cover.png") - runFFmpegTestCommand( - t, - "-y", - "-f", - "lavfi", - "-i", - "color=c=blue:s=64x64:d=1", - "-frames:v", - "1", - coverPath, - ) - - formats := []struct { - name string - extension string - codec string - writeTags func(string, map[string]string) error - littleSize bool - }{ - {name: "WAV16", extension: ".wav", codec: "pcm_s16le", writeTags: WriteWAVTags, littleSize: true}, - {name: "WAV24", extension: ".wav", codec: "pcm_s24le", writeTags: WriteWAVTags, littleSize: true}, - {name: "AIFF16", extension: ".aiff", codec: "pcm_s16be", writeTags: WriteAIFFTags}, - {name: "AIFF24", extension: ".aiff", codec: "pcm_s24be", writeTags: WriteAIFFTags}, - } - - for _, format := range formats { - format := format - t.Run(format.name, func(t *testing.T) { - path := filepath.Join(dir, format.name+format.extension) - runFFmpegTestCommand( - t, - "-y", - "-f", - "lavfi", - "-i", - "sine=frequency=440:sample_rate=48000:duration=1", - "-ac", - "2", - "-c:a", - format.codec, - "-metadata", - "title=Container fallback", - path, - ) - - if err := format.writeTags(path, map[string]string{ - "title": "Externally decodable", - "artist": "SpotiFLAC Mobile", - "track_number": "1", - "track_total": "1", - "cover_path": coverPath, - }); err != nil { - t.Fatalf("write native tags: %v", err) - } - - contents, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if len(contents) < 12 { - t.Fatalf("container is too small: %d bytes", len(contents)) - } - var declaredSize uint32 - if format.littleSize { - declaredSize = binary.LittleEndian.Uint32(contents[4:8]) - } else { - declaredSize = binary.BigEndian.Uint32(contents[4:8]) - } - if int64(declaredSize)+8 != int64(len(contents)) { - t.Fatalf( - "container size = %d, file size = %d", - int64(declaredSize)+8, - len(contents), - ) - } - - probe := exec.Command( - ffprobePath, - "-v", - "error", - "-select_streams", - "a:0", - "-show_entries", - "stream=codec_name,sample_rate,channels", - "-of", - "json", - path, - ) - probeOutput, err := probe.CombinedOutput() - if err != nil { - t.Fatalf("ffprobe rejected tagged file: %v\n%s", err, probeOutput) - } - var probeResult struct { - Streams []struct { - CodecName string `json:"codec_name"` - SampleRate string `json:"sample_rate"` - Channels int `json:"channels"` - } `json:"streams"` - } - if err := json.Unmarshal(probeOutput, &probeResult); err != nil { - t.Fatalf("decode ffprobe output: %v\n%s", err, probeOutput) - } - if len(probeResult.Streams) != 1 { - t.Fatalf("ffprobe streams = %v", probeResult.Streams) - } - stream := probeResult.Streams[0] - if stream.CodecName != format.codec || stream.SampleRate != "48000" || stream.Channels != 2 { - t.Fatalf("ffprobe stream = %+v, want %s/48000 Hz/stereo", stream, format.codec) - } - - decode := ffmpegCommand( - "-v", - "error", - "-i", - path, - "-map", - "0:a:0", - "-f", - "null", - "-", - ) - if output, err := decode.CombinedOutput(); err != nil { - t.Fatalf("ffmpeg could not decode tagged file: %v\n%s", err, output) - } - }) - } -} - -func assertWAVAIFFTestMetadata(t *testing.T, meta *AudioMetadata) { - t.Helper() - if meta.Title != "Judul Lagu" || - meta.Artist != "Artis" || - meta.Album != "Album" || - meta.AlbumArtist != "Album Artis" || - meta.Date != "2026-08-01" || - meta.TrackNumber != 3 || - meta.TotalTracks != 12 || - meta.DiscNumber != 1 || - meta.TotalDiscs != 2 || - meta.Genre != "Pop" || - meta.ISRC != "TEST12345678" || - meta.Lyrics != "Baris pertama\nBaris kedua" || - meta.Label != "Label" || - meta.Copyright != "Copyright" || - meta.Composer != "Komposer" || - meta.Comment != "Komentar" || - meta.ReplayGainTrackGain != "-5.00 dB" || - meta.ReplayGainTrackPeak != "0.987654" || - meta.ReplayGainAlbumGain != "-4.00 dB" || - meta.ReplayGainAlbumPeak != "0.998877" { - t.Fatalf("metadata did not round-trip: %+v", meta) - } -} diff --git a/ios/Flutter/Debug.xcconfig b/ios/Flutter/Debug.xcconfig index dc020e11..ec97fc6f 100644 --- a/ios/Flutter/Debug.xcconfig +++ b/ios/Flutter/Debug.xcconfig @@ -1,6 +1,2 @@ #include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig" #include "Generated.xcconfig" - -// Go backend framework (Gobackend.xcframework) -FRAMEWORK_SEARCH_PATHS=$(inherited) $(PROJECT_DIR)/Frameworks -OTHER_LDFLAGS=$(inherited) -framework Gobackend diff --git a/ios/Flutter/Release.xcconfig b/ios/Flutter/Release.xcconfig index d1bd6e25..c4855bfe 100644 --- a/ios/Flutter/Release.xcconfig +++ b/ios/Flutter/Release.xcconfig @@ -1,6 +1,2 @@ #include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig" #include "Generated.xcconfig" - -// Go backend framework (Gobackend.xcframework) -FRAMEWORK_SEARCH_PATHS=$(inherited) $(PROJECT_DIR)/Frameworks -OTHER_LDFLAGS=$(inherited) -framework Gobackend diff --git a/ios/Podfile b/ios/Podfile index 82b716ea..d9308044 100644 --- a/ios/Podfile +++ b/ios/Podfile @@ -63,10 +63,17 @@ target 'Runner' do use_modular_headers! flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__)) -end + rust_pod_dir = File.expand_path('../rust_backend/target/ios', __dir__) + unless File.directory?(File.join(rust_pod_dir, 'SpotiFLACBackendFFI.xcframework')) && + File.file?(File.join(rust_pod_dir, 'SpotiFLACBackend.podspec')) && + File.file?(File.join(rust_pod_dir, 'SpotiFLACBackend.swift')) + raise 'Run bash scripts/build_rust_backend.sh ios before installing Rust application pods' + end + pod 'SpotiFLACBackend', :path => '../rust_backend/target/ios' -target 'RunnerTests' do - inherit! :search_paths + target 'RunnerTests' do + inherit! :search_paths + end end post_install do |installer| diff --git a/ios/Podfile.lock b/ios/Podfile.lock index 843ccae0..5837dddc 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -2,21 +2,26 @@ PODS: - Flutter (1.0.0) - open_filex (0.0.2): - Flutter + - SpotiFLACBackend (0.1.0) DEPENDENCIES: - Flutter (from `Flutter`) - open_filex (from `.symlinks/plugins/open_filex/ios`) + - SpotiFLACBackend (from `../rust_backend/target/ios`) EXTERNAL SOURCES: Flutter: :path: Flutter open_filex: :path: ".symlinks/plugins/open_filex/ios" + SpotiFLACBackend: + :path: "../rust_backend/target/ios" SPEC CHECKSUMS: Flutter: 71a624a5bc0c04062bf19101d501e466baf2fb47 - open_filex: 432f3cd11432da3e39f47fcc0df2b1603854eff1 + open_filex: 6e26e659846ec990262224a12ef1c528bb4edbe4 + SpotiFLACBackend: e0c81656e8e4c31e86b9f14bbb8e245fb3a14011 -PODFILE CHECKSUM: 56111b15e504c4f967a8ce631d1fc11455de8985 +PODFILE CHECKSUM: 6ca58d78915f3e4fcb64938b9167b26e33d44a8a COCOAPODS: 1.16.2 diff --git a/ios/Runner.xcodeproj/project.pbxproj b/ios/Runner.xcodeproj/project.pbxproj index 1eeee458..4632f304 100644 --- a/ios/Runner.xcodeproj/project.pbxproj +++ b/ios/Runner.xcodeproj/project.pbxproj @@ -7,20 +7,19 @@ objects = { /* Begin PBXBuildFile section */ - D08A594730CF000100000001 /* DownloadProgressSubscription.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08A594730CF000100000002 /* DownloadProgressSubscription.swift */; }; 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */ = {isa = PBXBuildFile; fileRef = 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */; }; 331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 331C807B294A618700263BE5 /* RunnerTests.swift */; }; 3B3967161E833CAA004F5970 /* AppFrameworkInfo.plist in Resources */ = {isa = PBXBuildFile; fileRef = 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */; }; 68CFDDBCDE3A5678CC34B903 /* Pods_Runner.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 4DCB20F642541E82BB2C12EC /* Pods_Runner.framework */; }; 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 74858FAE1ED2DC5600515810 /* AppDelegate.swift */; }; 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */ = {isa = PBXBuildFile; productRef = 78A3181F2AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage */; }; - A11CE1012F00000000000001 /* Gobackend.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = A11CE1032F00000000000001 /* Gobackend.xcframework */; }; - A11CE1022F00000000000001 /* Gobackend.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = A11CE1032F00000000000001 /* Gobackend.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 97C146FC1CF9000F007C117D /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FA1CF9000F007C117D /* Main.storyboard */; }; 97C146FE1CF9000F007C117D /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FD1CF9000F007C117D /* Assets.xcassets */; }; 97C147011CF9000F007C117D /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */; }; A11CE0012F00000000000001 /* ExtensionCallbackParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = A11CE0022F00000000000001 /* ExtensionCallbackParser.swift */; }; - BD09D99380BF342C986CFAD6 /* libPods-RunnerTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 0448E6A8F05C4FAF8461F7C9 /* libPods-RunnerTests.a */; }; + A7FCE2361461C6C5FBC10719 /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 403DAD1CF5A7C09D11A461EF /* Pods_RunnerTests.framework */; }; + C03400000000000000000001 /* CoreBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = C03400000000000000000002 /* CoreBackend.swift */; }; + D08A594730CF000100000001 /* DownloadProgressSubscription.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08A594730CF000100000002 /* DownloadProgressSubscription.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -40,7 +39,6 @@ dstPath = ""; dstSubfolderSpec = 10; files = ( - A11CE1022F00000000000001 /* Gobackend.xcframework in Embed Frameworks */, ); name = "Embed Frameworks"; runOnlyForDeploymentPostprocessing = 0; @@ -48,13 +46,12 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ - D08A594730CF000100000002 /* DownloadProgressSubscription.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DownloadProgressSubscription.swift; sourceTree = "<group>"; }; - 0448E6A8F05C4FAF8461F7C9 /* libPods-RunnerTests.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-RunnerTests.a"; sourceTree = BUILT_PRODUCTS_DIR; }; 1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = "<group>"; }; 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = "<group>"; }; 331C807B294A618700263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = "<group>"; }; 331C8081294A63A400263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = AppFrameworkInfo.plist; path = Flutter/AppFrameworkInfo.plist; sourceTree = "<group>"; }; + 403DAD1CF5A7C09D11A461EF /* Pods_RunnerTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_RunnerTests.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 4DCB20F642541E82BB2C12EC /* Pods_Runner.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Runner.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "Runner-Bridging-Header.h"; sourceTree = "<group>"; }; 74858FAE1ED2DC5600515810 /* AppDelegate.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; }; @@ -69,11 +66,12 @@ 97C147001CF9000F007C117D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; }; 97C147021CF9000F007C117D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; }; 97EA1417A7115E3D3E4EBA2A /* Pods-RunnerTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.release.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.release.xcconfig"; sourceTree = "<group>"; }; - A11CE1032F00000000000001 /* Gobackend.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Frameworks/Gobackend.xcframework; sourceTree = "<group>"; }; A11CE0022F00000000000001 /* ExtensionCallbackParser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExtensionCallbackParser.swift; sourceTree = "<group>"; }; B38518FC01BA7990B1B952FC /* Pods-Runner.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.profile.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.profile.xcconfig"; sourceTree = "<group>"; }; B7F67C6787B0752DCCCC62F3 /* Pods-RunnerTests.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.profile.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.profile.xcconfig"; sourceTree = "<group>"; }; + C03400000000000000000002 /* CoreBackend.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreBackend.swift; sourceTree = "<group>"; }; CE79C187C33AB874567BE0FA /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = "<group>"; }; + D08A594730CF000100000002 /* DownloadProgressSubscription.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DownloadProgressSubscription.swift; sourceTree = "<group>"; }; FB3B03A8181A430FB2E2796D /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = "<group>"; }; /* End PBXFileReference section */ @@ -82,7 +80,6 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - A11CE1012F00000000000001 /* Gobackend.xcframework in Frameworks */, 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */, 68CFDDBCDE3A5678CC34B903 /* Pods_Runner.framework in Frameworks */, ); @@ -92,7 +89,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - BD09D99380BF342C986CFAD6 /* libPods-RunnerTests.a in Frameworks */, + A7FCE2361461C6C5FBC10719 /* Pods_RunnerTests.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -164,6 +161,7 @@ 1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */, 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */, 74858FAE1ED2DC5600515810 /* AppDelegate.swift */, + C03400000000000000000002 /* CoreBackend.swift */, D08A594730CF000100000002 /* DownloadProgressSubscription.swift */, A11CE0022F00000000000001 /* ExtensionCallbackParser.swift */, 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */, @@ -174,9 +172,8 @@ C6C88C23490A8D792BED2082 /* Frameworks */ = { isa = PBXGroup; children = ( - A11CE1032F00000000000001 /* Gobackend.xcframework */, 4DCB20F642541E82BB2C12EC /* Pods_Runner.framework */, - 0448E6A8F05C4FAF8461F7C9 /* libPods-RunnerTests.a */, + 403DAD1CF5A7C09D11A461EF /* Pods_RunnerTests.framework */, ); name = Frameworks; sourceTree = "<group>"; @@ -400,6 +397,7 @@ buildActionMask = 2147483647; files = ( 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */, + C03400000000000000000001 /* CoreBackend.swift in Sources */, D08A594730CF000100000001 /* DownloadProgressSubscription.swift in Sources */, A11CE0012F00000000000001 /* ExtensionCallbackParser.swift in Sources */, 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */, diff --git a/ios/Runner/AppDelegate.swift b/ios/Runner/AppDelegate.swift index 6d3a38fc..36f075e1 100644 --- a/ios/Runner/AppDelegate.swift +++ b/ios/Runner/AppDelegate.swift @@ -2,7 +2,6 @@ import AuthenticationServices import Flutter import UIKit import UniformTypeIdentifiers -import Gobackend @main @objc class AppDelegate: FlutterAppDelegate { @@ -12,18 +11,20 @@ import Gobackend private let LARGE_JSON_RESULT_FILE_KEY = "__json_file" private let LARGE_JSON_RESULT_FILE_THRESHOLD_BYTES = 256 * 1024 private let streamQueue = DispatchQueue(label: "com.zarz.spotiflac.progress_stream", qos: .utility) - private let downloadProgressSubscription = DownloadProgressSubscription { sequence, timeout in - GobackendWaitForAllDownloadProgressDelta(sequence, timeout) as String? ?? "" - } + private lazy var downloadProgressSubscription = DownloadProgressSubscription(connect: { [weak self] in + guard let self else { throw NSError(domain: "CoreBackend", code: 1) } + return try self.coreBackend.openDownloadProgress() + }) private var libraryScanProgressTimer: DispatchSourceTimer? private var libraryScanProgressEventSink: FlutterEventSink? private var lastLibraryScanProgressPayload: String? private var libraryScanProgressGeneration: UInt64 = 0 private var backendChannel: FlutterMethodChannel? + private let coreBackend: CoreBackend = createCoreBackend() private var pendingSessionGrantEvents: [[String: Any]] = [] private let securityScopedAccessLock = NSLock() - private var securityScopedAccesses: [String: URL] = [:] + private var securityScopedAccesses: [String: (URL, CoreDirectoryScope)] = [:] /// Pending Flutter result for the native folder picker private var pendingDirectoryPickerResult: FlutterResult? @@ -42,9 +43,6 @@ import Gobackend _ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? ) -> Bool { - if let version = Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String { - GobackendSetAppVersion(version) - } let controller = window?.rootViewController as! FlutterViewController let channel = FlutterMethodChannel( @@ -110,97 +108,27 @@ import Gobackend private func handleExtensionOAuthRedirect(url: URL) -> Bool { guard let route = ExtensionCallbackParser.parse(url) else { return false } streamQueue.async { - var err: NSError? - var response: String? - let extensionId: String - if route.isSessionGrant { - extensionId = GobackendResolveExtensionCallbackState( - route.state, - &err - ) - } else { - extensionId = GobackendConsumeExtensionCallbackState( - route.state, - &err - ) - } - guard err == nil, !extensionId.isEmpty else { - NSLog("SpotiFLAC Mobile: Rejected invalid or expired extension callback") - return - } - if route.isSessionGrant { - GobackendSetExtensionSessionGrantByID(extensionId, route.code) - response = GobackendInvokeExtensionActionJSON( - extensionId, - "completeGrant", - &err - ) - } else { - GobackendSetExtensionAuthCodeByID(extensionId, route.code) - response = GobackendInvokeExtensionActionJSON( - extensionId, - "completeSpotifyLogin", - &err - ) - } - if err == nil && route.isSessionGrant { - do { - try self.requireSuccessfulExtensionAction( - extensionId: extensionId, - actionName: "completeGrant", - response: response - ) - } catch { - err = error as NSError + var extensionId = "" + do { + try self.coreBackend.completeAuthCallback(state: route.state, code: route.code, sessionGrant: route.isSessionGrant) { + extensionId = $0 } - } - if let err = err { - NSLog( - "SpotiFLAC Mobile: Extension callback failed (code \(err.code))") - } else if route.isSessionGrant { - DispatchQueue.main.async { [weak self] in - self?.notifySessionGrantCompleted( - extensionId: extensionId - ) + if route.isSessionGrant { + DispatchQueue.main.async { [weak self] in + self?.notifySessionGrantCompleted(extensionId: extensionId) + } + } + } catch { + if extensionId.isEmpty { + NSLog("SpotiFLAC Mobile: Rejected invalid or expired extension callback") + } else { + NSLog("SpotiFLAC Mobile: Extension callback failed (code \((error as NSError).code))") } } } return true } - private func requireSuccessfulExtensionAction( - extensionId: String, - actionName: String, - response: String? - ) throws { - let text = response ?? "" - guard let data = text.data(using: .utf8), - let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { - throw NSError( - domain: "SpotiFLAC", - code: 1, - userInfo: [ - NSLocalizedDescriptionKey: - "Extension \(actionName) for \(extensionId) returned invalid JSON: \(String(text.prefix(240)))" - ] - ) - } - if (obj["success"] as? Bool) == true { - return - } - let error = - (obj["error"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? - (obj["message"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? - String(text.prefix(240)) - throw NSError( - domain: "SpotiFLAC", - code: 2, - userInfo: [ - NSLocalizedDescriptionKey: "Extension \(actionName) failed for \(extensionId): \(error)" - ] - ) - } - private func notifySessionGrantCompleted(extensionId: String) { let payload: [String: Any] = [ "extension_id": extensionId, @@ -248,7 +176,7 @@ import Gobackend timer.schedule(deadline: .now(), repeating: .milliseconds(800)) timer.setEventHandler { [weak self] in guard let self, self.libraryScanProgressGeneration == generation else { return } - let payload = GobackendGetLibraryScanProgressJSON() as String? ?? "{}" + let payload = (try? self.coreBackend.getLibraryScanProgress()) ?? "{}" if payload == self.lastLibraryScanProgressPayload { return } @@ -299,6 +227,22 @@ import Gobackend } private func handleMethodCall(call: FlutterMethodCall, result: @escaping FlutterResult) { + let osMethods: Set<String> = ["getBackendImplementations", "startWebAuthSession", "beginBackgroundDownloadTask", "endBackgroundDownloadTask", + "pickIosDirectory", "createIosBookmarkFromPath", "resolveIosBookmark", "startAccessingIosBookmark", "stopAccessingIosBookmark", "downloadCoverToFile", "releaseMemory", "releaseMemoryUnderPressure", + "setLibraryCoverCacheDir", "scanLibraryFolder", "scanLibraryFolderToNDJSONFile", "scanLibraryFolderIncremental", + "getLibraryScanProgress", "cancelLibraryScan", "parseCueSheet", "extractCoverToFile", + "rewriteSplitArtistTags", "writeM4AFreeformTags", "ensureAC4Config", "writeAC4Metadata", "reEnrichFile"] + if coreBackend.routesApplication && !osMethods.contains(call.method) { + DispatchQueue.global(qos: .userInitiated).async { + do { + let response = try self.coreBackend.invokeApplication(method: call.method, arguments: call.arguments) + DispatchQueue.main.async { result(response) } + } catch { + DispatchQueue.main.async { result(FlutterError(code: "ERROR", message: error.localizedDescription, details: nil)) } + } + } + return + } switch call.method { case "beginBackgroundDownloadTask": downloadsActive = true @@ -408,10 +352,11 @@ import Gobackend withName: "SpotiFLACDownloads" ) { [weak self] in if self?.downloadsActive == true { + NSLog("SpotiFLAC: download background task expired") // Flutter channel delivery is asynchronous and iOS may suspend - // us immediately after this callback. Cancel live Go requests + // us immediately after this callback. Cancel live requests // synchronously first; Dart then persists/requeues the items. - let cancelledItemIDs = GobackendCancelAllActiveDownloads() + let cancelledItemIDs = (try? self?.coreBackend.cancelActiveDownloads()) ?? "[]" self?.backendChannel?.invokeMethod( "iosBackgroundDownloadExpired", arguments: cancelledItemIDs @@ -429,652 +374,131 @@ import Gobackend } private func invokeGoMethod(call: FlutterMethodCall) throws -> Any? { - var error: NSError? switch call.method { case "downloadByStrategy": let requestJson = call.arguments as! String - let response = GobackendDownloadByStrategy(requestJson, &error) - if let error = error { throw error } - return response + return try coreBackend.downloadByStrategy(requestJson: requestJson) - case "getAllDownloadProgress": - let response = GobackendGetAllDownloadProgress() - return parseJsonPayload(response as String? ?? "{}") - - case "clearItemProgress": - let args = call.arguments as! [String: Any] - let itemId = args["item_id"] as! String - GobackendClearItemProgress(itemId) - return nil - case "cancelDownload": + case "acquireDownloadDirectory": let args = call.arguments as! [String: Any] - let itemId = args["item_id"] as! String - GobackendCancelDownload(itemId) - return nil + try coreBackend.openDownloadDirectory(path: args["path"] as! String).close() + return "" - case "resetDownloadCancel": - let args = call.arguments as! [String: Any] - let itemId = args["item_id"] as! String - GobackendResetDownloadCancel(itemId) - return nil + case "releaseDownloadDirectory": return nil - case "setDownloadDirectory": - let args = call.arguments as! [String: Any] - let path = args["path"] as! String - GobackendSetDownloadDirectory(path, &error) - if let error = error { throw error } - return nil - case "setNetworkCompatibilityOptions", "setSongLinkNetworkOptions": - let args = call.arguments as! [String: Any] - let allowHTTP = args["allow_http"] as? Bool ?? false - let insecureTLS = args["insecure_tls"] as? Bool ?? false - GobackendSetNetworkCompatibilityOptions(allowHTTP, insecureTLS) - return nil + case "getBackendImplementations": + return [ + "filename": coreBackend.implementation, + "file_metadata": coreBackend.fileMetadataImplementation(path: (call.arguments as? [String: Any])?["file_path"] as? String ?? ""), + "extensions": coreBackend.routesApplication ? "rust" : "go", + "downloads": coreBackend.implementation, + ] - case "setAllowPrivateNetwork": - let args = call.arguments as! [String: Any] - let allowed = args["allowed"] as? Bool ?? false - GobackendSetAllowPrivateNetwork(allowed) - return nil - - case "checkDuplicatesBatch": - let args = call.arguments as! [String: Any] - let outputDir = args["output_dir"] as! String - let tracksJson = args["tracks"] as? String ?? "[]" - let response = GobackendCheckDuplicatesBatch(outputDir, tracksJson, &error) - if let error = error { throw error } - return response - - case "preBuildDuplicateIndex": - let args = call.arguments as! [String: Any] - let outputDir = args["output_dir"] as! String - GobackendPreBuildDuplicateIndex(outputDir, &error) - if let error = error { throw error } - return nil - - case "invalidateDuplicateIndex": - let args = call.arguments as! [String: Any] - let outputDir = args["output_dir"] as! String - GobackendInvalidateDuplicateIndex(outputDir) - return nil - case "buildFilename": let args = call.arguments as! [String: Any] let template = args["template"] as! String let metadata = args["metadata"] as! String - let response = GobackendBuildFilename(template, metadata, &error) - if let error = error { throw error } - return response + return try coreBackend.buildFilename(template: template, metadataJson: metadata) case "sanitizeFilename": let args = call.arguments as! [String: Any] let filename = args["filename"] as! String - let response = GobackendSanitizeFilename(filename) - return response - - case "getLyricsLRC": - let args = call.arguments as! [String: Any] - let spotifyId = args["spotify_id"] as! String - let trackName = args["track_name"] as! String - let artistName = args["artist_name"] as! String - let filePath = args["file_path"] as? String ?? "" - let durationMs = args["duration_ms"] as? Int64 ?? 0 - let response = GobackendGetLyricsLRC(spotifyId, trackName, artistName, filePath, durationMs, &error) - if let error = error { throw error } - return response + return coreBackend.sanitizeFilename(filename: filename) + - case "getLyricsLRCWithSource": - let args = call.arguments as! [String: Any] - let spotifyId = args["spotify_id"] as! String - let trackName = args["track_name"] as! String - let artistName = args["artist_name"] as! String - let filePath = args["file_path"] as? String ?? "" - let durationMs = args["duration_ms"] as? Int64 ?? 0 - let response = GobackendGetLyricsLRCWithSource(spotifyId, trackName, artistName, filePath, durationMs, &error) - if let error = error { throw error } - return response - - case "embedLyricsToFile": - let args = call.arguments as! [String: Any] - let filePath = args["file_path"] as! String - let lyrics = args["lyrics"] as! String - let response = GobackendEmbedLyricsToFile(filePath, lyrics, &error) - if let error = error { throw error } - return response - case "rewriteSplitArtistTags": let args = call.arguments as! [String: Any] let filePath = args["file_path"] as! String let artist = args["artist"] as! String let albumArtist = args["album_artist"] as! String - let response = GobackendRewriteSplitArtistTagsExport(filePath, artist, albumArtist, &error) - if let error = error { throw error } - return response - - case "cleanupConnections": - GobackendCleanupConnections() - return nil + return try coreBackend.rewriteSplitArtistTags(path: filePath, artist: artist, albumArtist: albumArtist) + + case "writeM4AFreeformTags": + let args = call.arguments as! [String: Any] + return try coreBackend.writeM4aFreeformTags(path: args["file_path"] as! String, metadataJson: args["metadata_json"] as? String ?? "{}") + + case "ensureAC4Config": + let args = call.arguments as! [String: Any] + return try coreBackend.ensureAc4Config(path: args["file_path"] as! String, reference: args["source_path"] as? String ?? "") + + case "writeAC4Metadata": + let args = call.arguments as! [String: Any] + return try coreBackend.writeAc4Metadata(path: args["file_path"] as! String, metadataJson: args["metadata_json"] as? String ?? "{}", coverPath: args["cover_path"] as? String ?? "") + case "downloadCoverToFile": let args = call.arguments as! [String: Any] let coverURL = args["cover_url"] as! String let outputPath = args["output_path"] as! String - let maxDimension = max(0, (args["max_dimension"] as? NSNumber)?.intValue ?? 0) - GobackendDownloadCoverToFileSized(coverURL, outputPath, maxDimension, &error) - if let error = error { throw error } + let maxDimension = max(0, (args["max_dimension"] as? NSNumber)?.int64Value ?? 0) + let temporary = outputPath.isEmpty ? try coreBackend.createTemporaryMediaFile(prefix: "cover_", suffix: ".jpg") : nil + do { + try coreBackend.downloadCoverToFileSized(url: coverURL, outputPath: temporary?.path ?? outputPath, maxDimension: maxDimension) + } catch { + if let temporary = temporary { try? FileManager.default.removeItem(at: temporary) } + throw error + } + if let temporary = temporary { + return String(decoding: try JSONSerialization.data(withJSONObject: ["success": true, "file_path": temporary.path]), as: UTF8.self) + } return "{\"success\":true}" case "extractCoverToFile": let args = call.arguments as! [String: Any] let audioPath = args["audio_path"] as! String let outputPath = args["output_path"] as! String - GobackendExtractCoverToFile(audioPath, outputPath, &error) - if let error = error { throw error } + try coreBackend.extractCoverToFile(audioPath: audioPath, outputPath: outputPath) return "{\"success\":true}" - case "fetchAndSaveLyrics": - let args = call.arguments as! [String: Any] - let trackName = args["track_name"] as! String - let artistName = args["artist_name"] as! String - let spotifyId = args["spotify_id"] as! String - let durationMs = args["duration_ms"] as? Int64 ?? 0 - let outputPath = args["output_path"] as! String - let audioFilePath = args["audio_file_path"] as? String ?? "" - GobackendFetchAndSaveLyrics(trackName, artistName, spotifyId, durationMs, outputPath, audioFilePath, &error) - if let error = error { throw error } - return "{\"success\":true}" case "reEnrichFile": let args = call.arguments as! [String: Any] let requestJson = args["request_json"] as? String ?? "{}" - let response = GobackendReEnrichFile(requestJson, &error) - if let error = error { throw error } - return response + return try coreBackend.reEnrichFile(requestJson: requestJson) case "readFileMetadata": let args = call.arguments as! [String: Any] let filePath = args["file_path"] as! String - let response = GobackendReadFileMetadataWithHint(filePath, args["display_name"] as? String ?? "", &error) - if let error = error { throw error } - return response + return try coreBackend.readFileMetadata(path: filePath, hint: args["display_name"] as? String ?? "") case "editFileMetadata": let args = call.arguments as! [String: Any] let filePath = args["file_path"] as! String let metadataJson = args["metadata_json"] as? String ?? "{}" - let response = GobackendEditFileMetadata(filePath, metadataJson, &error) - if let error = error { throw error } - return response - - case "getProviderMetadata": - let args = call.arguments as! [String: Any] - let providerId = args["provider_id"] as! String - let resourceType = args["resource_type"] as! String - let resourceId = args["resource_id"] as! String - let response = GobackendGetProviderMetadataJSON(providerId, resourceType, resourceId, &error) - if let error = error { throw error } - return response + return try coreBackend.editFileMetadata(path: filePath, metadataJson: metadataJson) - case "searchDeezerByISRC": - let args = call.arguments as! [String: Any] - let isrc = args["isrc"] as! String - let itemId = args["item_id"] as? String ?? "" - let response = GobackendSearchDeezerByISRCForItemID(isrc, itemId, &error) - if let error = error { throw error } - return response - - case "getDeezerExtendedMetadata": - let args = call.arguments as! [String: Any] - let trackId = args["track_id"] as! String - let response = GobackendGetDeezerExtendedMetadata(trackId, &error) - if let error = error { throw error } - return response - - case "convertSpotifyToDeezer": - let args = call.arguments as! [String: Any] - let resourceType = args["resource_type"] as! String - let spotifyId = args["spotify_id"] as! String - let response = GobackendConvertSpotifyToDeezer(resourceType, spotifyId, &error) - if let error = error { throw error } - return response - - case "getSpotifyIDFromDeezerTrack": - let args = call.arguments as! [String: Any] - let deezerTrackId = args["deezer_track_id"] as! String - let response = GobackendGetSpotifyIDFromDeezerTrack(deezerTrackId, &error) - if let error = error { throw error } - return response - - case "getTidalURLFromDeezerTrack": - let args = call.arguments as! [String: Any] - let deezerTrackId = args["deezer_track_id"] as! String - let response = GobackendGetTidalURLFromDeezerTrack(deezerTrackId, &error) - if let error = error { throw error } - return response - - case "getTrackCacheSize": - let response = GobackendGetTrackCacheSize() - return response - - case "clearTrackCache": - GobackendClearTrackIDCache() - return nil - - case "getLogsSince": - let args = call.arguments as! [String: Any] - let index = args["index"] as? Int ?? 0 - let response = GobackendGetLogsSince(Int(index)) - return response - - case "clearLogs": - GobackendClearLogs() - return nil case "releaseMemory": - GobackendReleaseMemory() + try coreBackend.releaseIdleResources() return nil case "releaseMemoryUnderPressure": - GobackendReleaseMemoryUnderPressure() + try coreBackend.releaseMemoryUnderPressure() + NSLog("SpotiFLAC: Backend memory pressure release completed") return nil - case "getGoRuntimeMetrics": - return GobackendGetRuntimeMetricsJSON() - - case "setLoggingEnabled": - let args = call.arguments as! [String: Any] - let enabled = args["enabled"] as? Bool ?? false - GobackendSetLoggingEnabled(enabled) - return nil - - case "initExtensionSystem": - let args = call.arguments as! [String: Any] - let extensionsDir = args["extensions_dir"] as! String - let dataDir = args["data_dir"] as! String - let masterKey = args["master_key"] as! String - GobackendSetExtensionStorageMasterKey(masterKey, &error) - if let error = error { throw error } - GobackendInitExtensionSystem(extensionsDir, dataDir, &error) - if let error = error { throw error } - return nil - - case "loadExtensionsFromDir": - let args = call.arguments as! [String: Any] - let dirPath = args["dir_path"] as! String - let response = GobackendLoadExtensionsFromDir(dirPath, &error) - if let error = error { throw error } - return response - - case "loadExtensionFromPath": - let args = call.arguments as! [String: Any] - let filePath = args["file_path"] as! String - let response = GobackendLoadExtensionFromPath(filePath, &error) - if let error = error { throw error } - return response - - case "unloadExtension": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - GobackendUnloadExtensionByID(extensionId, &error) - if let error = error { throw error } - return nil - - case "getInstalledExtensions": - let response = GobackendGetInstalledExtensions(&error) - if let error = error { throw error } - return response - - case "setExtensionEnabled": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let enabled = args["enabled"] as? Bool ?? false - GobackendSetExtensionEnabledByID(extensionId, enabled, &error) - if let error = error { throw error } - return nil - - case "setProviderPriority": - let args = call.arguments as! [String: Any] - let priorityJson = args["priority"] as! String - GobackendSetProviderPriorityJSON(priorityJson, &error) - if let error = error { throw error } - return nil - - case "getProviderPriority": - let response = GobackendGetProviderPriorityJSON(&error) - if let error = error { throw error } - return response - case "setDownloadFallbackExtensionIds": - let args = call.arguments as! [String: Any] - let extensionIdsJson = args["extension_ids"] as? String ?? "" - GobackendSetExtensionFallbackProviderIDsJSON(extensionIdsJson, &error) - if let error = error { throw error } - return nil - - case "setMetadataProviderPriority": - let args = call.arguments as! [String: Any] - let priorityJson = args["priority"] as! String - GobackendSetMetadataProviderPriorityJSON(priorityJson, &error) - if let error = error { throw error } - return nil - - case "getMetadataProviderPriority": - let response = GobackendGetMetadataProviderPriorityJSON(&error) - if let error = error { throw error } - return response - - case "getExtensionSettings": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let response = GobackendGetExtensionSettingsJSON(extensionId, &error) - if let error = error { throw error } - return response - case "checkExtensionHealth": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let response = GobackendCheckExtensionHealthJSON(extensionId, &error) - if let error = error { throw error } - return response - - case "setExtensionSettings": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let settingsJson = args["settings"] as! String - GobackendSetExtensionSettingsJSON(extensionId, settingsJson, &error) - if let error = error { throw error } - return nil - - case "invokeExtensionAction": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let actionName = args["action"] as! String - let response = GobackendInvokeExtensionActionJSON(extensionId, actionName, &error) - if let error = error { throw error } - return response - - case "searchTracksWithMetadataProviders": - let args = call.arguments as! [String: Any] - let query = args["query"] as! String - let limit = args["limit"] as? Int ?? 20 - let includeExtensions = args["include_extensions"] as? Bool ?? true - let response = GobackendSearchTracksWithMetadataProvidersJSON( - query, - Int(limit), - includeExtensions, - &error - ) - if let error = error { throw error } - return response - - case "searchTracksWithMetadataProvider": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as? String ?? "" - let query = args["query"] as? String ?? "" - let limit = args["limit"] as? Int ?? 20 - let response = GobackendSearchTracksWithMetadataProviderJSON( - extensionId, - query, - Int(limit), - &error - ) - if let error = error { throw error } - return response - - case "enrichTrackWithExtension": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let trackJson = args["track"] as? String ?? "{}" - let response = GobackendEnrichTrackWithExtensionJSON(extensionId, trackJson, &error) - if let error = error { throw error } - return response - - case "downloadWithExtensions": - let requestJson = call.arguments as! String - let response = GobackendDownloadWithExtensionsJSON(requestJson, &error) - if let error = error { throw error } - return response - - case "removeExtension": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - GobackendRemoveExtensionByID(extensionId, &error) - if let error = error { throw error } - return nil - - case "upgradeExtension": - let args = call.arguments as! [String: Any] - let filePath = args["file_path"] as! String - let response = GobackendUpgradeExtensionFromPath(filePath, &error) - if let error = error { throw error } - return response - - case "checkExtensionUpgrade": - let args = call.arguments as! [String: Any] - let filePath = args["file_path"] as! String - let response = GobackendCheckExtensionUpgradeFromPath(filePath, &error) - if let error = error { throw error } - return response - - case "cleanupExtensions": - GobackendCleanupExtensions() - return nil - - case "getExtensionPendingAuth": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let response = GobackendGetExtensionPendingAuthJSON(extensionId, &error) - if let error = error { throw error } - return response - - case "setExtensionAuthCode": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let authCode = args["auth_code"] as! String - GobackendSetExtensionAuthCodeByID(extensionId, authCode) - return nil - - case "completeExtensionSessionGrant": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let grant = args["grant"] as! String - GobackendSetExtensionSessionGrantByID(extensionId, grant) - let response = GobackendInvokeExtensionActionJSON(extensionId, "completeGrant", &error) - if let error = error { throw error } - try requireSuccessfulExtensionAction( - extensionId: extensionId, - actionName: "completeGrant", - response: response - ) - return true - - case "setExtensionTokens": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let accessToken = args["access_token"] as! String - let refreshToken = args["refresh_token"] as? String ?? "" - let expiresIn = args["expires_in"] as? Int ?? 0 - GobackendSetExtensionTokensByID(extensionId, accessToken, refreshToken, Int(expiresIn)) - return nil - - case "clearExtensionPendingAuth": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - GobackendClearExtensionPendingAuthByID(extensionId) - return nil - - case "isExtensionAuthenticated": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let response = GobackendIsExtensionAuthenticatedByID(extensionId) - return response - - case "getAllPendingAuthRequests": - let response = GobackendGetAllPendingAuthRequestsJSON(&error) - if let error = error { throw error } - return response - - case "getPendingFFmpegCommand": - let args = call.arguments as! [String: Any] - let commandId = args["command_id"] as! String - let response = GobackendGetPendingFFmpegCommandJSON(commandId, &error) - if let error = error { throw error } - return response - - case "setFFmpegCommandResult": - let args = call.arguments as! [String: Any] - let commandId = args["command_id"] as! String - let success = args["success"] as? Bool ?? false - let output = args["output"] as? String ?? "" - let errorMsg = args["error"] as? String ?? "" - GobackendSetFFmpegCommandResult(commandId, success, output, errorMsg) - return nil - - case "getAllPendingFFmpegCommands": - let response = GobackendGetAllPendingFFmpegCommandsJSON(&error) - if let error = error { throw error } - return response - - case "customSearchWithExtension": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let query = args["query"] as! String - let optionsJson = args["options"] as? String ?? "" - let requestId = args["request_id"] as? String ?? "" - let response = GobackendCustomSearchWithExtensionJSONWithRequestID(extensionId, query, optionsJson, requestId, &error) - if let error = error { throw error } - return response - - case "cancelExtensionRequest": - let args = call.arguments as! [String: Any] - let requestId = args["request_id"] as? String ?? "" - GobackendCancelExtensionRequestJSON(requestId) - return nil - - case "handleURLWithExtension": - let args = call.arguments as! [String: Any] - let url = args["url"] as! String - let response = GobackendHandleURLWithExtensionJSON(url, &error) - if let error = error { throw error } - return response - - case "findURLHandler": - let args = call.arguments as! [String: Any] - let url = args["url"] as! String - let response = GobackendFindURLHandlerJSON(url) - return response - - case "getTrackPlatformLinks": - let args = call.arguments as! [String: Any] - let spotifyId = args["spotify_id"] as? String ?? "" - let isrc = args["isrc"] as? String ?? "" - let response = GobackendGetTrackPlatformLinksJSON(spotifyId, isrc, &error) - if let error = error { throw error } - return response - - case "fetchMusicBrainzTags": - let args = call.arguments as! [String: Any] - let isrc = args["isrc"] as? String ?? "" - let albumName = args["album_name"] as? String ?? "" - var genreError: NSError? - let genre = GobackendFetchMusicBrainzGenreByISRC(isrc, &genreError) - var artistError: NSError? - let albumArtist = GobackendFetchMusicBrainzAlbumArtistByISRC(isrc, albumName, &artistError) - let payload: [String: Any] = [ - "genre": genreError == nil ? genre : "", - "album_artist": artistError == nil ? albumArtist : "", - ] - let data = try JSONSerialization.data(withJSONObject: payload) - return String(data: data, encoding: .utf8) ?? "{}" - case "runPostProcessingV2": let args = call.arguments as! [String: Any] let inputJson = args["input"] as? String ?? "" let metadataJson = args["metadata"] as? String ?? "" - let response = GobackendRunPostProcessingV2JSON(inputJson, metadataJson, &error) - if let error = error { throw error } - return response - - case "initExtensionRepo": - let args = call.arguments as! [String: Any] - let cacheDir = args["cache_dir"] as! String - GobackendInitExtensionRepoJSON(cacheDir, &error) - if let error = error { throw error } - return nil - - case "setRepoRegistryUrl": - let args = call.arguments as! [String: Any] - let registryUrl = args["registry_url"] as? String ?? "" - GobackendSetRepoRegistryURLJSON(registryUrl, &error) - if let error = error { throw error } - return nil - - case "getRepoRegistryUrl": - let response = GobackendGetRepoRegistryURLJSON(&error) - if let error = error { throw error } - return response - - case "clearRepoRegistryUrl": - GobackendClearRepoRegistryURLJSON(&error) - if let error = error { throw error } - return nil - - case "getRepoExtensions": - let args = call.arguments as! [String: Any] - let forceRefresh = args["force_refresh"] as? Bool ?? false - let response = GobackendGetRepoExtensionsJSON(forceRefresh, &error) - if let error = error { throw error } - return response - - case "searchRepoExtensions": - let args = call.arguments as! [String: Any] - let query = args["query"] as? String ?? "" - let category = args["category"] as? String ?? "" - let response = GobackendSearchRepoExtensionsJSON(query, category, &error) - if let error = error { throw error } - return response - - case "getRepoCategories": - let response = GobackendGetRepoCategoriesJSON(&error) - if let error = error { throw error } - return response - - case "downloadRepoExtension": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let destDir = args["dest_dir"] as! String - let response = GobackendDownloadRepoExtensionJSON(extensionId, destDir, &error) - if let error = error { throw error } - return response - - case "clearRepoCache": - GobackendClearRepoCacheJSON(&error) - if let error = error { throw error } - return nil - - case "getExtensionHomeFeed": - let args = call.arguments as! [String: Any] - let extensionId = args["extension_id"] as! String - let requestId = args["request_id"] as? String ?? "" - let response = GobackendGetExtensionHomeFeedJSONWithRequestID(extensionId, requestId, &error) - if let error = error { throw error } - return response - + return try coreBackend.runPostProcessing(inputJson: inputJson, metadataJson: metadataJson) + + case "setLibraryCoverCacheDir": let args = call.arguments as! [String: Any] let cacheDir = args["cache_dir"] as! String - GobackendSetLibraryCoverCacheDirJSON(cacheDir) + try coreBackend.setLibraryCoverCacheDirectory(path: cacheDir) return nil case "scanLibraryFolder": let args = call.arguments as! [String: Any] let folderPath = args["folder_path"] as! String - let response = GobackendScanLibraryFolderJSON(folderPath, &error) - if let error = error { throw error } - return bridgeJsonResult(response as String? ?? "[]") + return bridgeJsonResult(try coreBackend.scanLibraryFolder(folder: folderPath)) case "scanLibraryFolderToNDJSONFile": guard @@ -1086,46 +510,23 @@ import Gobackend else { throw invalidArgumentsError(call.method) } - var count = 0 - let succeeded = GobackendScanLibraryFolderToNDJSONFileJSON( - folderPath, - outputPath, - &count, - &error - ) - if let error = error { throw error } - if !succeeded { - throw NSError( - domain: "SpotiFLAC", - code: -1, - userInfo: [NSLocalizedDescriptionKey: "Library scan failed"] - ) - } + let count = try coreBackend.scanLibraryFolderToNdjsonFile(folder: folderPath, output: outputPath) return ["path": outputPath, "count": count] case "scanLibraryFolderIncremental": let args = call.arguments as! [String: Any] let folderPath = args["folder_path"] as! String let existingFiles = args["existing_files"] as? String ?? "{}" - let response = GobackendScanLibraryFolderIncrementalJSON(folderPath, existingFiles, &error) - if let error = error { throw error } - return bridgeJsonResult(response as String? ?? "{}") + return bridgeJsonResult(try coreBackend.scanLibraryFolderIncremental(folder: folderPath, existing: existingFiles)) case "getLibraryScanProgress": - let response = GobackendGetLibraryScanProgressJSON() - return parseJsonPayload(response as String? ?? "{}") + return parseJsonPayload(try coreBackend.getLibraryScanProgress()) case "cancelLibraryScan": - GobackendCancelLibraryScanJSON() + try coreBackend.cancelLibraryScan() return nil - - case "readAudioMetadata": - let args = call.arguments as! [String: Any] - let filePath = args["file_path"] as! String - let response = GobackendReadAudioMetadataJSON(filePath, &error) - if let error = error { throw error } - return response - + + case "resolveIosBookmark": let args = call.arguments as! [String: Any] let bookmarkBase64 = args["bookmark"] as! String @@ -1156,43 +557,13 @@ import Gobackend let args = call.arguments as! [String: Any] let path = args["path"] as! String return try createIosBookmarkFromPath(path) - - case "setLyricsProviders": - let args = call.arguments as! [String: Any] - let providersJson = args["providers_json"] as? String ?? "[]" - GobackendSetLyricsProvidersJSON(providersJson, &error) - if let error = error { throw error } - return "{\"success\":true}" - - case "getLyricsProviders": - let response = GobackendGetLyricsProvidersJSON(&error) - if let error = error { throw error } - return response - - case "getAvailableLyricsProviders": - let response = GobackendGetAvailableLyricsProvidersJSON(&error) - if let error = error { throw error } - return response - - case "setLyricsFetchOptions": - let args = call.arguments as! [String: Any] - let optionsJson = args["options_json"] as? String ?? "{}" - GobackendSetLyricsFetchOptionsJSON(optionsJson, &error) - if let error = error { throw error } - return "{\"success\":true}" - - case "getLyricsFetchOptions": - let response = GobackendGetLyricsFetchOptionsJSON(&error) - if let error = error { throw error } - return response - + + case "parseCueSheet": let args = call.arguments as! [String: Any] let cuePath = args["cue_path"] as! String let audioDir = args["audio_dir"] as? String ?? "" - let response = GobackendParseCueSheet(cuePath, audioDir, &error) - if let error = error { throw error } - return response + return try coreBackend.parseCueSheet(path: cuePath, audioDirectory: audioDir) default: throw NSError( @@ -1347,9 +718,16 @@ import Gobackend ) } + let scope: CoreDirectoryScope + do { + scope = try coreBackend.openDownloadDirectory(path: url.path) + } catch { + url.stopAccessingSecurityScopedResource() + throw error + } let token = UUID().uuidString securityScopedAccessLock.lock() - securityScopedAccesses[token] = url + securityScopedAccesses[token] = (url, scope) securityScopedAccessLock.unlock() return ["path": url.path, "token": token] } @@ -1357,9 +735,10 @@ import Gobackend /// Releases only the lease identified by the caller's token. private func stopAccessingIosBookmark(token: String) { securityScopedAccessLock.lock() - let url = securityScopedAccesses.removeValue(forKey: token) + let lease = securityScopedAccesses.removeValue(forKey: token) securityScopedAccessLock.unlock() - url?.stopAccessingSecurityScopedResource() + lease?.1.close() + lease?.0.stopAccessingSecurityScopedResource() } } diff --git a/ios/Runner/CoreBackend.swift b/ios/Runner/CoreBackend.swift new file mode 100644 index 00000000..bb577bb7 --- /dev/null +++ b/ios/Runner/CoreBackend.swift @@ -0,0 +1,811 @@ +import Foundation +import CryptoKit +import Darwin +import ffmpegkit +import SpotiFLACBackend + +struct CoreFFmpegCommand: Decodable { + let command_id: String + let arguments: [String] + var output_path: String? = nil +} + +final class CoreFFmpegExecution { + let wait: (Int64) throws -> [CoreFFmpegCommand] + let active: (String) throws -> Bool + let complete: (String, Bool, String, String) throws -> Void + let close: () -> Void + private let lock = NSLock() + private var running = true + + init(wait: @escaping (Int64) throws -> [CoreFFmpegCommand], active: @escaping (String) throws -> Bool, + complete: @escaping (String, Bool, String, String) throws -> Void, close: @escaping () -> Void = {}) { + self.wait = wait + self.active = active + self.complete = complete + self.close = close + } + + func run(execute: @escaping ([String], @escaping () -> Bool) -> (Bool, String) = executeCoreFFmpeg, + block: () throws -> String) rethrows -> String { + DispatchQueue.global(qos: .userInitiated).async { + defer { self.close() } + while self.isRunning() { + guard let commands = try? self.wait(1_000) else { return } + // A different operation may own any command in this claimed batch. + for command in commands { + let cancelled = { (try? self.active(command.command_id)) != true } + let result: (Bool, String) + if cancelled() { result = (false, "cancelled") } + else if command.arguments.isEmpty { result = (false, "FFmpeg arguments are empty") } + else { result = executeCoreFFmpegCommand(command: command, cancelled: cancelled, execute: execute) } + try? self.complete(command.command_id, result.0, result.1, result.0 ? "" : result.1) + } + } + } + defer { + lock.lock() + running = false + lock.unlock() + } + return try block() + } + + private func isRunning() -> Bool { + lock.lock() + defer { lock.unlock() } + return running + } +} + +private enum CoreFFmpegStaging { + static let lock = NSLock() + static var directories = Set<String>() + static let prefix = ".spotiflac-ffmpeg-\(Bundle.main.bundleIdentifier ?? "spotiflac")-" + + static func create(for target: URL) throws -> URL { + let directory = target.deletingLastPathComponent().resolvingSymlinksInPath() + lock.lock() + defer { lock.unlock() } + // One app process owns these stages. Sweep before its first command + // in this directory; subsequent/concurrent commands may own live files. + if !directories.contains(directory.path) { + for file in try FileManager.default.contentsOfDirectory( + at: directory, includingPropertiesForKeys: [.isRegularFileKey, .isSymbolicLinkKey] + ) where file.lastPathComponent.hasPrefix(prefix) { + let suffix = file.lastPathComponent.dropFirst(prefix.count) + let token = String(suffix.prefix(36)) + let values = try file.resourceValues(forKeys: [.isRegularFileKey, .isSymbolicLinkKey]) + if UUID(uuidString: token) != nil, suffix.dropFirst(36).hasPrefix("."), + values.isRegularFile == true, values.isSymbolicLink != true { + try FileManager.default.removeItem(at: file) + } + } + directories.insert(directory.path) + } + let staged = directory.appendingPathComponent("\(prefix)\(UUID().uuidString).\(target.pathExtension)") + try Data().write(to: staged, options: .withoutOverwriting) + return staged + } +} + +func executeCoreFFmpegCommand(command: CoreFFmpegCommand, cancelled: @escaping () -> Bool, + execute: ([String], @escaping () -> Bool) -> (Bool, String)) -> (Bool, String) { + if cancelled() { return (false, "cancelled") } + guard let output = command.output_path, !output.isEmpty else { + return execute(command.arguments, cancelled) + } + guard command.arguments.last == output else { return (false, "FFmpeg output does not match command") } + let target = URL(fileURLWithPath: output) + do { + let staged = try CoreFFmpegStaging.create(for: target) + defer { try? FileManager.default.removeItem(at: staged) } + var arguments = command.arguments + arguments[arguments.count - 1] = staged.path + let result = execute(arguments, cancelled) + if cancelled() { return (false, "cancelled") } + if !result.0 { return result } + // POSIX rename replaces the destination atomically on the same filesystem. + guard rename(staged.path, target.path) == 0 else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + return result + } catch { + return (false, error.localizedDescription) + } +} + +private func executeCoreFFmpeg(arguments: [String], cancelled: @escaping () -> Bool) -> (Bool, String) { + if cancelled() { return (false, "cancelled") } + let finished = DispatchSemaphore(value: 0) + guard let session = FFmpegKit.execute(withArgumentsAsync: arguments, withCompleteCallback: { _ in finished.signal() }) else { + return (false, "FFmpeg session could not start") + } + while finished.wait(timeout: .now() + 0.2) == .timedOut { + if cancelled() { + session.cancel() + _ = finished.wait(timeout: .now() + 5) + return (false, "cancelled") + } + } + if cancelled() { return (false, "cancelled") } + return (ReturnCode.isSuccess(session.getReturnCode()), session.getOutput() ?? "") +} + +/// Owns one operation's directory access through finalization. +final class CoreDirectoryScope { + private let lock = NSLock() + private var release: (() -> Void)? + + init(_ release: @escaping () -> Void) { self.release = release } + + func close() { + lock.lock() + defer { lock.unlock() } + let action = release + release = nil + action?() + } + + deinit { close() } +} + +/// Native migration boundary; each process selects one stateful backend. +protocol CoreBackend { + var implementation: String { get } + var routesApplication: Bool { get } + func invokeApplication(method: String, arguments: Any?) throws -> Any? + func completeAuthCallback(state: String, code: String, sessionGrant: Bool, onResolved: (String) -> Void) throws + func openDownloadProgress() throws -> CoreDownloadProgress + func cancelActiveDownloads() throws -> String + func downloadByStrategy(requestJson: String) throws -> String + func runPostProcessing(inputJson: String, metadataJson: String) throws -> String + func buildFilename(template: String, metadataJson: String) throws -> String + func sanitizeFilename(filename: String) -> String + func fileMetadataImplementation(path: String) -> String + func readFileMetadata(path: String, hint: String) throws -> String + func editFileMetadata(path: String, metadataJson: String) throws -> String + func reEnrichFile(requestJson: String) throws -> String + func rewriteSplitArtistTags(path: String, artist: String, albumArtist: String) throws -> String + func extractCoverToFile(audioPath: String, outputPath: String) throws + func writeM4aFreeformTags(path: String, metadataJson: String) throws -> String + func ensureAc4Config(path: String, reference: String) throws -> String + func writeAc4Metadata(path: String, metadataJson: String, coverPath: String) throws -> String + func setLibraryCoverCacheDirectory(path: String) throws + func scanLibraryFolder(folder: String) throws -> String + func scanLibraryFolderToNdjsonFile(folder: String, output: String) throws -> Int + func scanLibraryFolderIncremental(folder: String, existing: String) throws -> String + func getLibraryScanProgress() throws -> String + func cancelLibraryScan() throws + func parseCueSheet(path: String, audioDirectory: String) throws -> String + func openDownloadDirectory(path: String) throws -> CoreDirectoryScope + func createTemporaryMediaFile(prefix: String, suffix: String) throws -> URL + func downloadCoverToFileSized(url: String, outputPath: String, maxDimension: Int64) throws + func releaseIdleResources() throws + func releaseMemoryUnderPressure() throws +} + +extension CoreBackend { + var routesApplication: Bool { false } + + func invokeApplication(method: String, arguments: Any?) throws -> Any? { + throw NSError(domain: "CoreBackend", code: 1, userInfo: [NSLocalizedDescriptionKey: "Application routing is unavailable for \(method)"]) + } +} + + +final class RustCoreBackend: CoreBackend { + static let shared = RustCoreBackend() + let implementation = "rust" + let routesApplication = true + private let root = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0].appendingPathComponent("rust-core-pilot", isDirectory: true) + private let ownerLock = NSLock() + private var manager: ExtensionManager? + private var repository: ExtensionRepository? + private var requests: CancellationRegistry? + private var identity: [String]? + private var loggingEnabled = false + private var allowPrivateNetwork = false + private var allowHttpFallback = false + private var fallbackProviders: [String]? + private var directoryScopes: [String: CoreDirectoryScope] = [:] + private var libraryCoverScope: CoreDirectoryScope? + + private init() {} + + private func withFFmpegCommands(_ current: ExtensionManager, block: () throws -> String) throws -> String { + let commands = try current.environment().ffmpegCommands() + let execution = CoreFFmpegExecution(wait: { timeout in + try JSONDecoder().decode([CoreFFmpegCommand].self, from: Data(commands.waitPending(timeoutMs: timeout).utf8)) + }, active: { id in + try !commands.getCommand(commandId: id).isEmpty + }, complete: { id, success, output, error in + _ = try commands.complete(commandId: id, success: success, output: output, error: error) + }) + return try execution.run(block: block) + } + + func downloadByStrategy(requestJson: String) throws -> String { + let current = try owner() + return try withFFmpegCommands(current) { try current.downloadByStrategy(requestJson: requestJson) } + } + + func runPostProcessing(inputJson: String, metadataJson: String) throws -> String { + let current = try owner() + return try withFFmpegCommands(current) { + try current.runPostProcessing(inputJson: inputJson, metadataJson: metadataJson, timeoutMs: 120_000) + } + } + + private func owner() throws -> ExtensionManager { + ownerLock.lock() + defer { ownerLock.unlock() } + guard let manager = manager else { throw failure("Rust backend is not initialized") } + return manager + } + + private func requestRegistryLocked() -> CancellationRegistry { + if let requests = requests { return requests } + let created = CancellationRegistry(domain: .extensionRequest) + requests = created + return created + } + + private func acquireRequest(_ id: String) throws -> (ExtensionManager, RequestLease) { + ownerLock.lock() + defer { ownerLock.unlock() } + guard let manager = manager else { throw failure("Rust backend is not initialized") } + return (manager, try requestRegistryLocked().acquire(id: id)) + } + + private func repositoryOwner() throws -> ExtensionRepository { + ownerLock.lock() + defer { ownerLock.unlock() } + guard let repository = repository else { throw failure("Extension repository is not initialized") } + return repository + } + + private func initializeRepository(_ cachePath: String) throws { + ownerLock.lock() + defer { ownerLock.unlock() } + guard let manager = manager else { throw failure("Rust backend is not initialized") } + if repository != nil { return } + guard cachePath.hasPrefix("/") else { throw failure("Repository cache directory must be absolute") } + let cache = URL(fileURLWithPath: cachePath).resolvingSymlinksInPath().standardizedFileURL.path + repository = try ExtensionRepository(manager: manager, cacheDirectory: cache) + } + + private func failure(_ message: String) -> NSError { + NSError(domain: "RustCoreBackend", code: 1, userInfo: [NSLocalizedDescriptionKey: message]) + } + + private func directoryAliases(_ value: String, sources: String, data: String) throws -> [String] { + guard value.hasPrefix("/") else { throw failure("Output directories must be absolute") } + let url = URL(fileURLWithPath: value) + let path = url.resolvingSymlinksInPath().standardizedFileURL.path + guard path != "/", ![sources, data].contains(where: { + path == $0 || path.hasPrefix($0 + "/") || $0.hasPrefix(path + "/") + }) else { throw failure("Output directory overlaps extension storage") } + let original = url.standardizedFileURL.path + return path == original ? [path] : [path, original] + } + + private func openDownloadDirectoryLocked(path: String) throws -> CoreDirectoryScope { + guard let manager = manager, let storage = identity else { throw failure("Rust backend is not initialized") } + let paths = try directoryAliases(path, sources: storage[0], data: storage[1]) + let scope = try manager.environment().grantDownloadDirectories(directories: paths) + return CoreDirectoryScope { scope.release() } + } + + func openDownloadDirectory(path: String) throws -> CoreDirectoryScope { + ownerLock.lock() + defer { ownerLock.unlock() } + return try openDownloadDirectoryLocked(path: path) + } + + private func withLibraryDirectories<T>(_ paths: [String], block: (ExtensionManager) throws -> T) throws -> T { + let (current, scope): (ExtensionManager, CoreDirectoryScope) = try { + ownerLock.lock() + defer { ownerLock.unlock() } + guard let manager = manager, let storage = identity else { throw failure("Rust backend is not initialized") } + let aliases = try paths.flatMap { try directoryAliases($0, sources: storage[0], data: storage[1]) } + let grant = try manager.environment().grantDownloadDirectories(directories: Array(Set(aliases))) + return (manager, CoreDirectoryScope { grant.release() }) + }() + defer { scope.close() } + // Progress and cancellation must remain available while the scan runs. + return try block(current) + } + + func setLibraryCoverCacheDirectory(path: String) throws { + ownerLock.lock() + defer { ownerLock.unlock() } + guard let manager = manager else { throw failure("Rust backend is not initialized") } + guard path.isEmpty || path.hasPrefix("/") else { throw failure("Library cover directory must be absolute") } + let directory = path.isEmpty ? nil : URL(fileURLWithPath: path).resolvingSymlinksInPath().standardizedFileURL + if let directory = directory { try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) } + let next = try directory.map { try openDownloadDirectoryLocked(path: $0.path) } + do { try manager.setLibraryCoverCacheDirectory(directory: directory?.path ?? "") } + catch { next?.close(); throw error } + libraryCoverScope?.close() + libraryCoverScope = next + } + + func scanLibraryFolder(folder: String) throws -> String { + try withLibraryDirectories([folder]) { + try $0.scanLibraryFolder(folder: URL(fileURLWithPath: folder).resolvingSymlinksInPath().standardizedFileURL.path, lease: nil) + } + } + + func scanLibraryFolderToNdjsonFile(folder: String, output: String) throws -> Int { + guard output.hasPrefix("/"), URL(fileURLWithPath: output).pathExtension.lowercased() == "ndjson" else { + throw failure("Library scan output must be an absolute NDJSON path") + } + return try withLibraryDirectories([folder]) { current in + // Application support also contains extension storage. Stage inside + // the existing private root instead of granting that whole directory. + let staged = root.appendingPathComponent("files", isDirectory: true) + .appendingPathComponent("library_scan_\(UUID().uuidString).ndjson") + defer { try? FileManager.default.removeItem(at: staged) } + let count = try current.scanLibraryFolderToNdjsonFile( + folder: URL(fileURLWithPath: folder).resolvingSymlinksInPath().standardizedFileURL.path, + output: staged.resolvingSymlinksInPath().path, lease: nil) + guard rename(staged.path, output) == 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + return Int(count) + } + } + + func scanLibraryFolderIncremental(folder: String, existing: String) throws -> String { + try withLibraryDirectories([folder]) { + try $0.scanLibraryFolderIncremental(folder: URL(fileURLWithPath: folder).resolvingSymlinksInPath().standardizedFileURL.path, + existingJson: existing, lease: nil) + } + } + + func getLibraryScanProgress() throws -> String { + ownerLock.lock() + let current = manager + ownerLock.unlock() + return try current?.getLibraryScanProgress() ?? "{}" + } + + func cancelLibraryScan() throws { + ownerLock.lock() + let current = manager + ownerLock.unlock() + try current?.cancelLibraryScan() + } + + func parseCueSheet(path: String, audioDirectory: String) throws -> String { + let cue = URL(fileURLWithPath: path).resolvingSymlinksInPath().standardizedFileURL + let audio = audioDirectory.isEmpty ? cue.deletingLastPathComponent() : URL(fileURLWithPath: audioDirectory).resolvingSymlinksInPath().standardizedFileURL + return try withLibraryDirectories([cue.deletingLastPathComponent().path, audio.path]) { + try $0.parseCueFileJson(path: cue.path, audioDirectory: audio.path, lease: nil) + } + } + + func createTemporaryMediaFile(prefix: String, suffix: String) throws -> URL { + _ = try owner() + let file = root.appendingPathComponent("files", isDirectory: true) + .appendingPathComponent("\(prefix)\(UUID().uuidString)\(suffix)") + try Data().write(to: file, options: .withoutOverwriting) + return file + } + + func downloadCoverToFileSized(url: String, outputPath: String, maxDimension: Int64) throws { + try owner().downloadCoverToFileSized(url: url, + outputPath: URL(fileURLWithPath: outputPath).resolvingSymlinksInPath().standardizedFileURL.path, + maxDimension: maxDimension, lease: nil) + } + + func releaseIdleResources() throws { + try owner().releaseMemory(underPressure: false) + } + + func releaseMemoryUnderPressure() throws { + try owner().releaseMemory(underPressure: true) + } + + private func initializeOwner(_ arguments: [String: Any]) throws { + guard let sourcePath = arguments["extensions_dir"] as? String, + let dataPath = arguments["data_dir"] as? String, + let key = arguments["master_key"] as? String else { + throw failure("Extension storage arguments are required") + } + guard sourcePath.hasPrefix("/"), dataPath.hasPrefix("/") else { + throw failure("Extension storage directories must be absolute") + } + let sources = URL(fileURLWithPath: sourcePath).resolvingSymlinksInPath().standardizedFileURL.path + let data = URL(fileURLWithPath: dataPath).resolvingSymlinksInPath().standardizedFileURL.path + let keyHash = SHA256.hash(data: Data(key.utf8)).map { String(format: "%02x", $0) }.joined() + let requested = [sources, data, keyHash] + ownerLock.lock() + defer { ownerLock.unlock() } + let files = root.appendingPathComponent("files", isDirectory: true) + try FileManager.default.createDirectory(at: files, withIntermediateDirectories: true) + guard arguments["allowed_directories"] == nil || arguments["allowed_directories"] is [String] else { + throw failure("Invalid output directories") + } + var allowedDirectories = [files.resolvingSymlinksInPath().path, files.standardizedFileURL.path] + for value in arguments["allowed_directories"] as? [String] ?? [] { + for alias in try directoryAliases(value, sources: sources, data: data) where !allowedDirectories.contains(alias) { + allowedDirectories.append(alias) + } + } + if manager != nil { + guard identity == requested else { throw failure("Rust backend is already initialized with different storage") } + try manager!.environment().setAllowedDownloadDirectories(directories: allowedDirectories) + return + } + let created = try ExtensionManager.withLyricsSettings( + sourceDirectory: sources, + dataDirectory: data, + masterKey: key, + appVersion: Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "1", + timeoutMs: 30000, + providersJson: arguments["lyrics_providers_json"] as? String ?? "[]", + optionsJson: arguments["lyrics_options_json"] as? String ?? "{}" + ) + do { + let environment = created.environment() + try environment.setAllowedDownloadDirectories(directories: allowedDirectories) + try environment.setAllowPrivateNetwork(allow: allowPrivateNetwork) + try environment.setNetworkCompatibilityOptions(allowHttp: allowHttpFallback, insecureTls: false) + try environment.logBuffer().setEnabled(enabled: loggingEnabled) + try created.setFallbackProviders(ids: fallbackProviders) + } catch { + created.shutdown() + throw error + } + identity = requested + manager = created + } + + private func shutdownOwner() { + ownerLock.lock() + defer { ownerLock.unlock() } + requests?.shutdown() + requests = nil + repository?.shutdown() + repository = nil + manager?.shutdown() + manager = nil + directoryScopes.values.forEach { $0.close() } + directoryScopes.removeAll() + libraryCoverScope?.close() + libraryCoverScope = nil + identity = nil + } + + func fileMetadataImplementation(path: String) -> String { "rust" } + + func readFileMetadata(path: String, hint: String) throws -> String { + try SpotiFLACBackend.readFileMetadata(path: path, hint: hint, lease: nil) + } + + func editFileMetadata(path: String, metadataJson: String) throws -> String { + try owner().editFileMetadata(path: URL(fileURLWithPath: path).resolvingSymlinksInPath().standardizedFileURL.path, metadataJson: metadataJson, lease: nil) + } + + private func mediaPath(_ path: String) -> String { + path.isEmpty ? "" : URL(fileURLWithPath: path).resolvingSymlinksInPath().standardizedFileURL.path + } + + func reEnrichFile(requestJson: String) throws -> String { + guard var request = try JSONSerialization.jsonObject(with: Data(requestJson.utf8)) as? [String: Any] else { + throw failure("Re-enrich request must be an object") + } + if request["preview_only"] as? Bool != true, + let path = request["file_path"] as? String, path.hasPrefix("/") { + request["file_path"] = mediaPath(path) + } + let data = try JSONSerialization.data(withJSONObject: request) + return try owner().reenrichFile(requestJson: String(decoding: data, as: UTF8.self), lease: nil) + } + + func rewriteSplitArtistTags(path: String, artist: String, albumArtist: String) throws -> String { + try owner().rewriteSplitArtistTags(path: mediaPath(path), artist: artist, albumArtist: albumArtist, lease: nil) + } + + func extractCoverToFile(audioPath: String, outputPath: String) throws { + try owner().extractCoverToFile(audioPath: mediaPath(audioPath), outputPath: mediaPath(outputPath), lease: nil) + } + + func writeM4aFreeformTags(path: String, metadataJson: String) throws -> String { + try owner().writeM4aFreeformTags(path: mediaPath(path), metadataJson: metadataJson, lease: nil) + } + + func ensureAc4Config(path: String, reference: String) throws -> String { + try owner().ensureAc4Config(path: mediaPath(path), reference: mediaPath(reference), lease: nil) + } + + func writeAc4Metadata(path: String, metadataJson: String, coverPath: String) throws -> String { + try owner().writeAc4Metadata(path: mediaPath(path), metadataJson: metadataJson, coverPath: mediaPath(coverPath), lease: nil) + } + + func completeAuthCallback(state: String, code: String, sessionGrant: Bool, onResolved: (String) -> Void) throws { + let current = try owner() + let environment = current.environment() + let id = try (sessionGrant ? environment.resolveCallbackState(state: state) : environment.consumeCallbackState(state: state)) + onResolved(id) + if sessionGrant { + try completeSessionGrant(current, id: id, grant: code) + } else { + try environment.setAuthCode(extensionId: id, code: code) + _ = try current.invokeAction(extensionId: id, action: "completeSpotifyLogin") + } + } + + func openDownloadProgress() throws -> CoreDownloadProgress { + let subscription = try owner().environment().downloadState().subscribeProgress() + return CoreDownloadProgress( + wait: { try subscription.waitDelta(since: $0, timeoutMs: $1) }, + close: { subscription.stop() } + ) + } + + func cancelActiveDownloads() throws -> String { + let ids = try owner().environment().downloadState().cancelActiveDownloads() + return String(decoding: try JSONSerialization.data(withJSONObject: ids), as: UTF8.self) + } + + private func completeSessionGrant(_ current: ExtensionManager, id: String, grant: String) throws { + try current.environment().setSessionGrant(extensionId: id, grant: grant) + let response = try current.invokeAction(extensionId: id, action: "completeGrant") + try requireSuccessfulExtensionAction(extensionId: id, actionName: "completeGrant", response: response) + } + + func invokeApplication(method: String, arguments: Any?) throws -> Any? { + let args = arguments as? [String: Any] ?? [:] + func string(_ key: String, _ fallback: String = "") -> String { args[key] as? String ?? fallback } + func ids(_ raw: String) throws -> [String] { + guard let value = try JSONSerialization.jsonObject(with: Data(raw.utf8)) as? [String] else { + throw failure("Expected an array of extension IDs") + } + return value + } + switch method { + case "cancelExtensionRequest": + ownerLock.lock() + defer { ownerLock.unlock() } + try requestRegistryLocked().cancel(id: string("request_id")) + return nil + case "customSearchWithExtension", "getExtensionHomeFeed": + let (current, lease) = try acquireRequest(string("request_id")) + defer { lease.release() } + if method == "customSearchWithExtension" { + return try current.customSearchJson(extensionId: string("extension_id"), query: string("query"), optionsJson: string("options"), lease: lease) + } + return try current.getExtensionHomeFeedJson(extensionId: string("extension_id"), lease: lease) + case "initExtensionSystem": try initializeOwner(args); return nil + case "setDownloadDirectory": + ownerLock.lock() + defer { ownerLock.unlock() } + guard let manager = manager, let storage = identity else { throw failure("Rust backend is not initialized") } + let files = root.appendingPathComponent("files", isDirectory: true) + var paths = [files.resolvingSymlinksInPath().path, files.standardizedFileURL.path] + let path = string("path") + if !path.isEmpty { paths += try directoryAliases(path, sources: storage[0], data: storage[1]) } + try manager.environment().setAllowedDownloadDirectories(directories: paths) + return nil + case "acquireDownloadDirectory": + ownerLock.lock() + defer { ownerLock.unlock() } + let scope = try openDownloadDirectoryLocked(path: string("path")) + let token = UUID().uuidString + directoryScopes[token] = scope + return token + case "releaseDownloadDirectory": + ownerLock.lock() + let scope = directoryScopes.removeValue(forKey: string("token")) + ownerLock.unlock() + scope?.close() + return nil + case "initExtensionRepo": try initializeRepository(string("cache_dir")); return nil + case "getRepoRegistryUrl": return try repositoryOwner().registryUrl() + case "setRepoRegistryUrl": try repositoryOwner().setRegistryUrl(url: string("registry_url")); return nil + case "clearRepoRegistryUrl": try repositoryOwner().clearRegistryUrl(); return nil + case "getRepoExtensions": return try repositoryOwner().extensions(forceRefresh: args["force_refresh"] as? Bool ?? false) + case "searchRepoExtensions": return try repositoryOwner().search(query: string("query"), category: string("category")) + case "getRepoCategories": + return String(decoding: try JSONSerialization.data(withJSONObject: repositoryOwner().categories()), as: UTF8.self) + case "downloadRepoExtension": return try repositoryOwner().download(extensionId: string("extension_id"), destinationDirectory: string("dest_dir")) + case "clearRepoCache": try repositoryOwner().clearCache(); return nil + case "cleanupExtensions": shutdownOwner(); return nil + case "buildFilename": return try buildFilename(template: string("template"), metadataJson: string("metadata", "{}")) + case "sanitizeFilename": return sanitizeFilename(filename: string("filename")) + case "readFileMetadata": return try readFileMetadata(path: string("file_path"), hint: string("display_name")) + case "editFileMetadata": return try editFileMetadata(path: string("file_path"), metadataJson: string("metadata_json", "{}")) + case "getLogsSince": + ownerLock.lock() + defer { ownerLock.unlock() } + return try manager?.environment().logBuffer().since(index: (args["index"] as? NSNumber)?.int64Value ?? 0) ?? "{\"logs\":[],\"next_index\":0}" + case "clearLogs": + ownerLock.lock() + defer { ownerLock.unlock() } + try manager?.environment().logBuffer().clear() + return nil + case "setLoggingEnabled", "setAllowPrivateNetwork", "setDownloadFallbackExtensionIds", "setLyricsProviders", "setLyricsFetchOptions", "setNetworkCompatibilityOptions", "setSongLinkNetworkOptions": + ownerLock.lock() + defer { ownerLock.unlock() } + switch method { + case "setLoggingEnabled": + let enabled = args["enabled"] as? Bool ?? false + try manager?.environment().logBuffer().setEnabled(enabled: enabled) + loggingEnabled = enabled + case "setAllowPrivateNetwork": + let allowed = args["allowed"] as? Bool ?? false + try manager?.environment().setAllowPrivateNetwork(allow: allowed) + allowPrivateNetwork = allowed + case "setNetworkCompatibilityOptions", "setSongLinkNetworkOptions": + let allowed = args["allow_http"] as? Bool ?? false + let insecureTLS = args["insecure_tls"] as? Bool ?? false + try manager?.environment().setNetworkCompatibilityOptions(allowHttp: allowed, insecureTls: insecureTLS) + allowHttpFallback = allowed + case "setDownloadFallbackExtensionIds": + let raw = string("extension_ids").trimmingCharacters(in: .whitespacesAndNewlines) + let value = raw.isEmpty || raw == "null" ? nil : try ids(raw) + try manager?.setFallbackProviders(ids: value) + fallbackProviders = value + case "setLyricsProviders": + let raw = string("providers_json", "[]") + if let manager = manager { try manager.setLyricsProvidersJson(providersJson: raw) } + else { _ = try ids(raw) } + return "{\"success\":true}" + case "setLyricsFetchOptions": + let raw = string("options_json", "{}") + if let manager = manager { try manager.setLyricsFetchOptionsJson(optionsJson: raw) } + else { _ = try JSONSerialization.jsonObject(with: Data(raw.utf8)) } + return "{\"success\":true}" + default: break + } + return nil + default: break + } + let current = try owner() + switch method { + case "loadExtensionsFromDir": + let path = URL(fileURLWithPath: string("dir_path")).resolvingSymlinksInPath().standardizedFileURL.path + ownerLock.lock() + let matches = identity?.first == path + ownerLock.unlock() + guard matches else { throw failure("Extension source directory does not match the initialized owner") } + return try current.loadAll() + case "loadExtensionFromPath": return try current.install(packagePath: string("file_path")) + case "upgradeExtension": return try current.upgrade(packagePath: string("file_path")) + case "checkExtensionUpgrade": return try current.checkUpgrade(packagePath: string("file_path")) + case "getInstalledExtensions": return try current.installed() + case "setExtensionEnabled": try current.setEnabled(extensionId: string("extension_id"), enabled: args["enabled"] as? Bool ?? false); return nil + case "unloadExtension": try current.unload(extensionId: string("extension_id")); return nil + case "removeExtension": try current.remove(extensionId: string("extension_id")); return nil + case "getExtensionSettings": return try current.environment().settings(extensionId: string("extension_id")) + case "setExtensionSettings": try current.updateSettings(extensionId: string("extension_id"), settingsJson: string("settings", "{}")); return nil + case "invokeExtensionAction": return try current.invokeAction(extensionId: string("extension_id"), action: string("action")) + case "checkExtensionHealth": return try current.checkExtensionHealthJson(extensionId: string("extension_id")) + case "searchTracksWithMetadataProviders": + return try current.searchMetadataProviders(query: string("query"), limit: (args["limit"] as? NSNumber)?.int64Value ?? 20, includeExtensions: args["include_extensions"] as? Bool ?? true, itemId: "", timeoutMs: 30000) + case "searchTracksWithMetadataProvider": + return try current.searchMetadataProvider(extensionId: string("extension_id"), query: string("query"), limit: (args["limit"] as? NSNumber)?.int64Value ?? 20, timeoutMs: 30000) + case "getProviderMetadata": + return try current.getProviderMetadataJson(providerId: string("provider_id"), resourceType: string("resource_type"), resourceId: string("resource_id"), lease: nil) + case "findCollectionAcrossExtensions": + return try current.findCollectionAcrossExtensionsJson(requestJson: arguments as? String ?? "{}", lease: nil) + case "enrichTrackWithExtension": return try current.enrichTrackJson(extensionId: string("extension_id"), trackJson: string("track", "{}")) + case "handleURLWithExtension": return try current.handleUrlJson(url: string("url")) + case "findURLHandler": return try current.findUrlHandler(url: string("url")) ?? "" + case "searchDeezerByISRC": return try current.searchDeezerByIsrcForItemId(isrc: string("isrc"), itemId: string("item_id"), lease: nil) + case "getDeezerExtendedMetadata": return try current.getDeezerExtendedMetadata(trackId: string("track_id"), lease: nil) + case "convertSpotifyToDeezer": return try current.convertSpotifyToDeezer(resourceType: string("resource_type"), spotifyId: string("spotify_id"), lease: nil) + case "getSpotifyIDFromDeezerTrack": return try current.getSpotifyIdFromDeezerTrack(trackId: string("deezer_track_id"), lease: nil) + case "getTidalURLFromDeezerTrack": return try current.getTidalUrlFromDeezerTrack(trackId: string("deezer_track_id"), lease: nil) + case "getTrackPlatformLinks": return try current.getTrackPlatformLinksJson(spotifyId: string("spotify_id"), isrc: string("isrc"), lease: nil) + case "fetchMusicBrainzTags": + let genre = (try? current.fetchMusicBrainzGenreByIsrc(isrc: string("isrc"), lease: nil)) ?? "" + let albumArtist = (try? current.fetchMusicBrainzAlbumArtistByIsrc(isrc: string("isrc"), albumName: string("album_name"), lease: nil)) ?? "" + return String(decoding: try JSONSerialization.data(withJSONObject: ["genre": genre, "album_artist": albumArtist]), as: UTF8.self) + case "getTrackCacheSize": return Int(try current.getTrackCacheSize()) + case "readAudioMetadata": return try current.readAudioMetadata(path: string("file_path"), hint: "", cacheKey: "", lease: nil) + case "clearTrackCache": try current.clearTrackIdCache(); return nil + case "setMetadataLanguage": try current.setMetadataLanguage(tag: string("tag")); return nil + case "downloadByStrategy", "downloadWithExtensions": + guard let request = arguments as? String else { + throw NSError(domain: "CoreBackend", code: 1, userInfo: [NSLocalizedDescriptionKey: "Download request must be a JSON string"]) + } + return try withFFmpegCommands(current) { + if method == "downloadWithExtensions" { return try current.downloadWithExtensionsJson(requestJson: request) } + return try current.downloadByStrategy(requestJson: request) + } + case "getAllDownloadProgress": return try current.environment().downloadState().allProgress() + case "cleanupConnections": try current.environment().cleanupConnections(); return nil + case "clearItemProgress": try current.environment().downloadState().clearItemProgress(itemId: string("item_id")); return nil + case "cancelDownload": try current.environment().downloadState().cancelDownload(itemId: string("item_id")); return nil + case "resetDownloadCancel": try current.environment().downloadState().resetDownloadCancel(itemId: string("item_id")); return nil + case "getExtensionPendingAuth": return try current.getExtensionPendingAuthJson(extensionId: string("extension_id")) + case "setExtensionAuthCode": try current.environment().setAuthCode(extensionId: string("extension_id"), code: string("auth_code")); return nil + case "completeExtensionSessionGrant": + try completeSessionGrant(current, id: string("extension_id"), grant: string("grant")) + return true + case "setExtensionTokens": + try current.environment().setAuthTokens(extensionId: string("extension_id"), accessToken: string("access_token"), refreshToken: string("refresh_token"), expiresIn: (args["expires_in"] as? NSNumber)?.int64Value ?? 0) + return nil + case "clearExtensionPendingAuth": try current.environment().clearPendingAuth(extensionId: string("extension_id")); return nil + case "isExtensionAuthenticated": return try current.environment().isAuthenticated(extensionId: string("extension_id")) + case "getAllPendingAuthRequests": return try current.environment().allPendingAuth() + case "getLyricsLRC", "getLyricsLRCWithSource", "fetchAndSaveLyrics": + let request = LyricsRequest( + spotifyId: string("spotify_id"), + track: string("track_name"), + artist: string("artist_name"), + filePath: string(method == "fetchAndSaveLyrics" ? "audio_file_path" : "file_path"), + durationMs: (args["duration_ms"] as? NSNumber)?.int64Value ?? 0 + ) + if method == "getLyricsLRC" { return try current.getLyricsLrc(request: request, lease: nil) } + if method == "getLyricsLRCWithSource" { return try current.getLyricsLrcWithSource(request: request, lease: nil) } + try current.fetchAndSaveLyrics(request: request, outputPath: string("output_path"), lease: nil) + return "{\"success\":true}" + case "getPendingFFmpegCommand": + return try current.environment().ffmpegCommands().getCommand(commandId: string("command_id")) + case "getAllPendingFFmpegCommands": + return try current.environment().ffmpegCommands().pending() + case "setFFmpegCommandResult": + _ = try current.environment().ffmpegCommands().complete( + commandId: string("command_id"), + success: args["success"] as? Bool ?? false, + output: string("output"), + error: string("error") + ) + return nil + case "runPostProcessingV2": + return try withFFmpegCommands(current) { + try current.runPostProcessing(inputJson: string("input"), metadataJson: string("metadata"), timeoutMs: 120_000) + } + case "embedLyricsToFile": + return try current.embedLyricsToFile(path: string("file_path"), lyrics: string("lyrics"), lease: nil) + case "setProviderPriority", "setMetadataProviderPriority": + try current.setProviderPriority(kind: method == "setProviderPriority" ? "download" : "metadata", ids: ids(string("priority", "[]"))) + return nil + case "getProviderPriority", "getMetadataProviderPriority": + let priorities = try JSONSerialization.jsonObject(with: Data(current.providerPriorities().utf8)) as! [String: Any] + let value = priorities[method == "getProviderPriority" ? "download" : "metadata"] as? [String] ?? [] + return String(decoding: try JSONSerialization.data(withJSONObject: value), as: UTF8.self) + case "getLyricsProviders": return try current.getLyricsProvidersJson() + case "getLyricsFetchOptions": return try current.getLyricsFetchOptionsJson() + case "getAvailableLyricsProviders": return try current.getAvailableLyricsProvidersJson() + default: throw failure("Rust application method is not connected yet: \(method)") + } + } + + func buildFilename(template: String, metadataJson: String) throws -> String { + try SpotiFLACBackend.buildFilename(template: template, metadataJson: metadataJson) + } + + func sanitizeFilename(filename: String) -> String { + SpotiFLACBackend.sanitizeFilename(filename: filename) + } +} + +func requireSuccessfulExtensionAction(extensionId: String, actionName: String, response: String?) throws { + let text = response ?? "" + guard let data = text.data(using: .utf8), + let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw NSError( + domain: "SpotiFLAC", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Extension \(actionName) for \(extensionId) returned invalid JSON: \(String(text.prefix(240)))"] + ) + } + if (obj["success"] as? Bool) == true { return } + let error = + (obj["error"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? + (obj["message"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? + String(text.prefix(240)) + throw NSError( + domain: "SpotiFLAC", + code: 2, + userInfo: [NSLocalizedDescriptionKey: "Extension \(actionName) failed for \(extensionId): \(error)"] + ) +} + +func createCoreBackend() -> CoreBackend { + return RustCoreBackend.shared +} diff --git a/ios/Runner/DownloadProgressSubscription.swift b/ios/Runner/DownloadProgressSubscription.swift index fa433473..bb2f9163 100644 --- a/ios/Runner/DownloadProgressSubscription.swift +++ b/ios/Runner/DownloadProgressSubscription.swift @@ -1,5 +1,21 @@ import Foundation +final class CoreDownloadProgress { + private let waiter: (Int64, Int64) throws -> String + private let closer: () -> Void + + init(wait: @escaping (Int64, Int64) throws -> String, close: @escaping () -> Void = {}) { + waiter = wait + closer = close + } + + func waitDelta(_ sequence: Int64, _ timeout: Int64) throws -> String { + try waiter(sequence, timeout) + } + + func close() { closer() } +} + /// Each listener owns its cursor and pending waiter. Lifecycle and delivery run /// on the main queue; a cancelled wait may finish, but can only touch its own state. final class DownloadProgressSubscription { @@ -8,6 +24,7 @@ final class DownloadProgressSubscription { let queue = DispatchQueue(label: "com.zarz.spotiflac.download_progress_subscription", qos: .utility) private let lock = NSLock() private var cancelled = false + private var source: CoreDownloadProgress? // Accessed only on this session's queue. var sequence: Int64 = 0 var lastPayload: String? @@ -15,7 +32,37 @@ final class DownloadProgressSubscription { func cancel() { lock.lock() cancelled = true + let closing = source + source = nil lock.unlock() + closing?.close() + } + + func connection(_ connect: () throws -> CoreDownloadProgress) throws -> CoreDownloadProgress? { + lock.lock() + let stopped = cancelled + let existing = source + lock.unlock() + if stopped { return nil } + if let existing { return existing } + let opened = try connect() + lock.lock() + if cancelled { + lock.unlock() + opened.close() + return nil + } + source = opened + lock.unlock() + return opened + } + + func disconnect() { + lock.lock() + let closing = source + source = nil + lock.unlock() + closing?.close() } var isCancelled: Bool { @@ -25,13 +72,17 @@ final class DownloadProgressSubscription { } } - private let waiter: Waiter + private let connect: () throws -> CoreDownloadProgress private let interval: TimeInterval private var current: Session? - init(interval: TimeInterval = 0.25, waiter: @escaping Waiter) { + convenience init(interval: TimeInterval = 0.25, waiter: @escaping Waiter) { + self.init(interval: interval, connect: { CoreDownloadProgress(wait: waiter) }) + } + + init(interval: TimeInterval = 0.25, connect: @escaping () throws -> CoreDownloadProgress) { self.interval = interval - self.waiter = waiter + self.connect = connect } func start(_ receive: @escaping (Any) -> Void) { @@ -51,10 +102,19 @@ final class DownloadProgressSubscription { deinit { current?.cancel() } private func poll(_ session: Session, receive: @escaping (Any) -> Void) { - let waiter = self.waiter + let connect = self.connect session.queue.async { [weak self] in guard !session.isCancelled else { return } - let payload = waiter(session.sequence, 15_000) + let payload: String + do { + guard let source = try session.connection(connect) else { return } + payload = try source.waitDelta(session.sequence, 15_000) + } catch { + session.disconnect() + session.sequence = 0 + session.lastPayload = nil + payload = "" + } guard !session.isCancelled else { return } if !payload.isEmpty && payload != session.lastPayload, let data = payload.data(using: .utf8), diff --git a/ios/RunnerTests/RunnerTests.swift b/ios/RunnerTests/RunnerTests.swift index 201956e3..a898da65 100644 --- a/ios/RunnerTests/RunnerTests.swift +++ b/ios/RunnerTests/RunnerTests.swift @@ -3,6 +3,305 @@ import XCTest @testable import Runner class RunnerTests: XCTestCase { + private func isSymbolicLink(_ url: URL) -> Bool { + guard let values = try? url.resourceValues(forKeys: [.isSymbolicLinkKey]) else { return false } + return values.isSymbolicLink == true + } + + func testFFmpegPublishesOnlySuccessfulUncancelledOutput() throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: false) + defer { try? FileManager.default.removeItem(at: root) } + let target = root.appendingPathComponent("output.flac") + for outcome in ["cancel", "failure", "success"] { + try Data("original".utf8).write(to: target) + var cancelled = false + var staged: URL? + let command = CoreFFmpegCommand(command_id: outcome, arguments: ["-i", "input.wav", "-y", target.path], output_path: target.path) + let result = executeCoreFFmpegCommand(command: command, cancelled: { cancelled }, execute: { arguments, _ in + XCTAssertEqual(arguments[1], "input.wav") + staged = URL(fileURLWithPath: arguments.last!) + XCTAssertNotEqual(staged, target) + XCTAssertEqual(staged?.pathExtension, "flac") + do { try Data("converted".utf8).write(to: staged!) } + catch { XCTFail("staging write failed: \(error)"); return (false, "write failed") } + cancelled = outcome == "cancel" + return (outcome != "failure", "result") + }) + XCTAssertEqual(result.0, outcome == "success") + XCTAssertEqual(try String(contentsOf: target, encoding: .utf8), outcome == "success" ? "converted" : "original") + XCTAssertFalse(FileManager.default.fileExists(atPath: staged!.path)) + } + } + + func testFFmpegCallerReturnKeepsOtherClaimedCommandAlive() { + let started = DispatchSemaphore(value: 0) + let release = DispatchSemaphore(value: 0) + let closed = expectation(description: "command handle released") + let lock = NSLock() + var isClosed = false + let execution = CoreFFmpegExecution(wait: { _ in + [CoreFFmpegCommand(command_id: "other", arguments: ["convert"])] + }, active: { _ in true }, complete: { id, success, _, _ in + XCTAssertEqual(id, "other") + XCTAssertTrue(success) + }, close: { + lock.lock() + isClosed = true + lock.unlock() + closed.fulfill() + }) + let result = execution.run(execute: { _, cancelled in + started.signal() + XCTAssertEqual(release.wait(timeout: .now() + 2), .success) + XCTAssertFalse(cancelled()) + return (true, "finished") + }) { + XCTAssertEqual(started.wait(timeout: .now() + 2), .success) + return "caller finished" + } + XCTAssertEqual(result, "caller finished") + lock.lock() + XCTAssertFalse(isClosed) + lock.unlock() + release.signal() + wait(for: [closed], timeout: 2) + } + + func testFFmpegFirstExecutionSweepsOnlyOwnOrphanFilesAndPreservesOtherEntries() throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: false) + defer { try? FileManager.default.removeItem(at: root) } + + let prefix = ".spotiflac-ffmpeg-\(Bundle.main.bundleIdentifier ?? "spotiflac")-" + let ownOrphan = root.appendingPathComponent("\(prefix)\(UUID().uuidString).flac") + try Data("orphan".utf8).write(to: ownOrphan) + let foreignFile = root.appendingPathComponent(".spotiflac-ffmpeg-foreign-\(UUID().uuidString).flac") + try Data("foreign".utf8).write(to: foreignFile) + let malformedFile = root.appendingPathComponent("\(prefix)not-a-uuid.flac") + try Data("malformed".utf8).write(to: malformedFile) + let preservedDirectory = root.appendingPathComponent("\(prefix)\(UUID().uuidString).flac") + try FileManager.default.createDirectory(at: preservedDirectory, withIntermediateDirectories: false) + let directoryContent = preservedDirectory.appendingPathComponent("content.txt") + try Data("directory content".utf8).write(to: directoryContent) + let symlinkTarget = root.appendingPathComponent("symlink-target.txt") + try Data("symlink content".utf8).write(to: symlinkTarget) + let symlink = root.appendingPathComponent("\(prefix)\(UUID().uuidString).flac") + try FileManager.default.createSymbolicLink(at: symlink, withDestinationURL: symlinkTarget) + + let target = root.appendingPathComponent("output.flac") + let command = CoreFFmpegCommand( + command_id: "sweep", + arguments: ["convert", target.path], + output_path: target.path + ) + var staged: URL? + let result = executeCoreFFmpegCommand(command: command, cancelled: { false }) { arguments, _ in + XCTAssertFalse(FileManager.default.fileExists(atPath: ownOrphan.path)) + XCTAssertEqual(try? String(contentsOf: foreignFile, encoding: .utf8), "foreign") + XCTAssertEqual(try? String(contentsOf: malformedFile, encoding: .utf8), "malformed") + XCTAssertTrue(FileManager.default.fileExists(atPath: preservedDirectory.path)) + XCTAssertEqual(try? String(contentsOf: directoryContent, encoding: .utf8), "directory content") + XCTAssertTrue(isSymbolicLink(symlink)) + XCTAssertEqual(try? String(contentsOf: symlink, encoding: .utf8), "symlink content") + + staged = URL(fileURLWithPath: arguments.last!) + do { + try Data("published".utf8).write(to: staged!) + } catch { + XCTFail("staging write failed: \(error)") + return (false, "write failed") + } + return (true, "converted") + } + + XCTAssertTrue(result.0) + XCTAssertEqual(try String(contentsOf: target, encoding: .utf8), "published") + XCTAssertFalse(FileManager.default.fileExists(atPath: staged!.path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: ownOrphan.path)) + XCTAssertEqual(try String(contentsOf: foreignFile, encoding: .utf8), "foreign") + XCTAssertEqual(try String(contentsOf: malformedFile, encoding: .utf8), "malformed") + XCTAssertEqual(try String(contentsOf: directoryContent, encoding: .utf8), "directory content") + XCTAssertTrue(isSymbolicLink(symlink)) + XCTAssertEqual(try String(contentsOf: symlink, encoding: .utf8), "symlink content") + } + + func testFFmpegOverlappingExecutionsPreserveLiveStagingAndPublishBothOutputs() throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: false) + defer { try? FileManager.default.removeItem(at: root) } + + let firstTarget = root.appendingPathComponent("first.flac") + let secondTarget = root.appendingPathComponent("second.flac") + let firstCommand = CoreFFmpegCommand( + command_id: "first", + arguments: ["convert-first", firstTarget.path], + output_path: firstTarget.path + ) + let secondCommand = CoreFFmpegCommand( + command_id: "second", + arguments: ["convert-second", secondTarget.path], + output_path: secondTarget.path + ) + var firstStage: URL? + var secondStage: URL? + + let result = executeCoreFFmpegCommand(command: firstCommand, cancelled: { false }) { arguments, _ in + firstStage = URL(fileURLWithPath: arguments.last!) + do { + try Data("first staged".utf8).write(to: firstStage!) + } catch { + XCTFail("first staging write failed: \(error)") + return (false, "write failed") + } + + let nestedResult = executeCoreFFmpegCommand(command: secondCommand, cancelled: { false }) { nestedArguments, _ in + secondStage = URL(fileURLWithPath: nestedArguments.last!) + XCTAssertTrue(FileManager.default.fileExists(atPath: firstStage!.path)) + XCTAssertEqual(try? String(contentsOf: firstStage!, encoding: .utf8), "first staged") + do { + try Data("second published".utf8).write(to: secondStage!) + } catch { + XCTFail("second staging write failed: \(error)") + return (false, "write failed") + } + return (true, "second complete") + } + + XCTAssertTrue(nestedResult.0) + XCTAssertEqual(try? String(contentsOf: secondTarget, encoding: .utf8), "second published") + XCTAssertTrue(FileManager.default.fileExists(atPath: firstStage!.path)) + XCTAssertEqual(try? String(contentsOf: firstStage!, encoding: .utf8), "first staged") + return (true, "first complete") + } + + XCTAssertTrue(result.0) + XCTAssertEqual(try String(contentsOf: firstTarget, encoding: .utf8), "first staged") + XCTAssertEqual(try String(contentsOf: secondTarget, encoding: .utf8), "second published") + XCTAssertFalse(FileManager.default.fileExists(atPath: firstStage!.path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: secondStage!.path)) + } + + func testFFmpegCancellationAndInvalidArgumentsDoNotStopOtherCommands() { + let started = DispatchSemaphore(value: 0) + let release = DispatchSemaphore(value: 0) + let completed = DispatchSemaphore(value: 0) + let closed = expectation(description: "pump closed") + let lock = NSLock() + var active = Set(["empty", "a", "b"]) + var claimed = false + let execution = CoreFFmpegExecution(wait: { _ in + lock.lock() + defer { lock.unlock() } + if claimed { return [] } + claimed = true + return [CoreFFmpegCommand(command_id: "empty", arguments: []), + CoreFFmpegCommand(command_id: "a", arguments: ["a"]), + CoreFFmpegCommand(command_id: "b", arguments: ["b"])] + }, active: { id in + lock.lock() + defer { lock.unlock() } + return active.contains(id) + }, complete: { id, success, _, error in + if id == "empty" { XCTAssertEqual(error, "FFmpeg arguments are empty") } + if id == "a" { XCTAssertFalse(success) } + if id == "b" { + XCTAssertTrue(success) + completed.signal() + } + }, close: { closed.fulfill() }) + _ = execution.run(execute: { arguments, cancelled in + if arguments == ["a"] { + started.signal() + XCTAssertEqual(release.wait(timeout: .now() + 2), .success) + XCTAssertTrue(cancelled()) + return (false, "cancelled") + } + XCTAssertEqual(arguments, ["b"]) + XCTAssertFalse(cancelled()) + return (true, "finished") + }) { + XCTAssertEqual(started.wait(timeout: .now() + 2), .success) + lock.lock() + active.remove("a") + lock.unlock() + release.signal() + XCTAssertEqual(completed.wait(timeout: .now() + 2), .success) + return "finished" + } + wait(for: [closed], timeout: 2) + } + + func testFFmpegClosedRegistryReleasesHandle() { + let closed = DispatchSemaphore(value: 0) + let execution = CoreFFmpegExecution(wait: { _ in + throw NSError(domain: "closed owner", code: 1) + }, active: { _ in false }, complete: { _, _, _, _ in + XCTFail("Closed registry produced a command") + }, close: { closed.signal() }) + _ = execution.run { + XCTAssertEqual(closed.wait(timeout: .now() + 2), .success) + return "finished" + } + } + + func testProgressReconnectResetsCursorAfterOwnerShutdown() { + let first = expectation(description: "first owner delivered") + let replacement = expectation(description: "replacement owner delivered") + let lock = NSLock() + var connections = 0 + var closes = 0 + var cursors = [Int64]() + let stream = DownloadProgressSubscription(interval: 0.01, connect: { + lock.lock() + connections += 1 + let owner = connections + lock.unlock() + return CoreDownloadProgress(wait: { sequence, _ in + lock.lock() + cursors.append(sequence) + lock.unlock() + if owner == 1 && sequence == 0 { return "{\"seq\":99,\"reset\":true,\"items\":{}}" } + if owner == 1 { throw NSError(domain: "closed owner", code: 1) } + return "{\"seq\":1,\"reset\":true,\"items\":{}}" + }, close: { + lock.lock() + closes += 1 + lock.unlock() + }) + }) + stream.start { event in + let sequence = (event as? [String: Any])?["seq"] as? Int + if sequence == 99 { first.fulfill() } + if sequence == 1 { replacement.fulfill() } + } + wait(for: [first, replacement], timeout: 2) + stream.stop() + lock.lock() + XCTAssertEqual(connections, 2) + XCTAssertEqual(closes, 2) + XCTAssertEqual(Array(cursors.prefix(3)), [0, 99, 0]) + lock.unlock() + } + + func testProgressStopClosesBlockedOwnerSubscription() { + let started = expectation(description: "wait started") + let stopped = expectation(description: "wait stopped") + let released = DispatchSemaphore(value: 0) + let stream = DownloadProgressSubscription(connect: { + CoreDownloadProgress(wait: { _, _ in + started.fulfill() + XCTAssertEqual(released.wait(timeout: .now() + 2), .success) + stopped.fulfill() + return "{\"seq\":1,\"items\":{}}" + }, close: { released.signal() }) + }) + stream.start { _ in XCTFail("Stopped listener received an event") } + wait(for: [started], timeout: 2) + stream.stop() + wait(for: [stopped], timeout: 2) + } + func testProgressRestartDoesNotAcceptCancelledWaiterState() { let oldStarted = expectation(description: "old waiter started") let replacementDelivered = expectation(description: "replacement snapshot") diff --git a/lib/constants/app_info.dart b/lib/constants/app_info.dart index 9de823b5..b117e292 100644 --- a/lib/constants/app_info.dart +++ b/lib/constants/app_info.dart @@ -1,8 +1,8 @@ import 'package:flutter/foundation.dart'; class AppInfo { - static const String version = '4.9.6'; - static const String buildNumber = '144'; + static const String version = '5.0.0'; + static const String buildNumber = '147'; static const String fullVersion = '$version+$buildNumber'; static const String gitCommit = String.fromEnvironment('GIT_COMMIT'); diff --git a/lib/l10n/arb/app_en.arb b/lib/l10n/arb/app_en.arb index 720b1033..9967b4ad 100644 --- a/lib/l10n/arb/app_en.arb +++ b/lib/l10n/arb/app_en.arb @@ -4215,6 +4215,10 @@ "@downloadLocationSubtitle": { "description": "Subtitle shown in the download location picker sheet" }, + "storageAutomaticFolder": "Automatic folder", + "storageAutomaticDownloadFolder": "Uses Download/SpotiFLAC when writable, otherwise app storage", + "storageSafRecommended": "Choose folder (Recommended)", + "storageDownloadFolderHint": "In the Android folder picker, open Download and select SpotiFLAC. If it does not exist, create it using New folder, then tap Use this folder to grant access.", "storageModeAppFolder": "App Folder (Recommended)", "@storageModeAppFolder": { "description": "Storage mode option - app-managed folder" diff --git a/lib/l10n/arb/app_id.arb b/lib/l10n/arb/app_id.arb index c44ecd19..994aa226 100644 --- a/lib/l10n/arb/app_id.arb +++ b/lib/l10n/arb/app_id.arb @@ -5914,6 +5914,10 @@ "libraryFilterTitle": "Filters", "cleanupOrphanedDownloadsResult": "Removed {count} orphaned entries from history", "extensionHooksAvailable": "{count} hook tersedia", + "storageAutomaticFolder": "Folder otomatis", + "storageAutomaticDownloadFolder": "Menggunakan Download/SpotiFLAC jika dapat ditulis, atau penyimpanan aplikasi", + "storageSafRecommended": "Pilih folder (Disarankan)", + "storageDownloadFolderHint": "Di pemilih folder Android, buka Download dan pilih SpotiFLAC. Jika belum ada, buat melalui Folder baru, lalu ketuk Gunakan folder ini untuk memberikan akses.", "backupSettingsOnly": "Pengaturan saja", "backupSelectHistory": "Riwayat download", "backupSelectCollections": "Koleksi library", diff --git a/lib/main.dart b/lib/main.dart index 2e4d96f1..d8b6f61b 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -1,8 +1,12 @@ import 'dart:async'; +import 'dart:convert'; import 'dart:io'; +import 'package:crypto/crypto.dart'; import 'package:device_info_plus/device_info_plus.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:path_provider/path_provider.dart'; import 'package:shared_preferences/shared_preferences.dart'; import 'package:spotiflac_android/app.dart'; import 'package:spotiflac_android/models/settings.dart'; @@ -17,19 +21,23 @@ import 'package:spotiflac_android/services/platform_bridge.dart'; import 'package:spotiflac_android/services/share_intent_service.dart'; import 'package:spotiflac_android/services/cover_cache_manager.dart'; import 'package:spotiflac_android/services/app_state_database.dart'; -import 'package:spotiflac_android/services/extension_storage_service.dart'; import 'package:spotiflac_android/utils/local_library_scan_prefs.dart'; import 'package:spotiflac_android/utils/logger.dart'; import 'package:spotiflac_android/utils/extension_auth_launcher.dart'; final _log = AppLogger('Main'); +_StartupBenchmark? _startupBenchmark; void main() { + if (const bool.fromEnvironment('CORE_BACKEND_STARTUP_BENCHMARK')) { + _startupBenchmark = _StartupBenchmark(); + } // Catch uncaught Dart errors so a failing async path is logged, not fatal. // Native (Go) crashes still can't be caught here. runZonedGuarded( () async { WidgetsFlutterBinding.ensureInitialized(); + _startupBenchmark?.observeFirstFrame(); final previousOnError = FlutterError.onError; FlutterError.onError = (details) { @@ -66,7 +74,7 @@ void main() { ), initialThemeSettingsProvider.overrideWithValue(bootstrapTheme), ], - child: _EagerInitialization( + child: EagerInitialization( child: SpotiFLACApp( disableOverscrollEffects: runtimeProfile.disableOverscrollEffects, ), @@ -80,6 +88,79 @@ void main() { ); } +// Opt-in release measurement. The ordinary startup path owns initialization; +// reporting and validation happen after both measured boundaries are captured. +class _StartupBenchmark { + final _watch = Stopwatch()..start(); + late final Future<int> _firstFrame; + + void observeFirstFrame() { + _firstFrame = WidgetsBinding.instance.waitUntilFirstFrameRasterized.then( + (_) => _watch.elapsedMicroseconds, + ); + } + + void extensionsReady(ExtensionState state, AppSettings settings) { + final ready = _watch.elapsedMicroseconds; + unawaited(_report(state, settings, ready)); + } + + Future<void> _report( + ExtensionState state, + AppSettings settings, + int ready, + ) async { + try { + if (!kReleaseMode) throw StateError('Startup benchmark requires release'); + final firstFrame = await _firstFrame.timeout(const Duration(seconds: 30)); + if (!state.isInitialized || state.error != null) { + throw StateError('Extensions are not ready: ${state.error}'); + } + final backends = await PlatformBridge.getBackendImplementations(); + const backend = String.fromEnvironment('EXPECTED_CORE_BACKEND'); + if (!const ['go', 'rust'].contains(backend) || + backends.length != 4 || + backends.values.any((value) => value != backend)) { + throw StateError('Unexpected backend ownership: $backends'); + } + await _write({ + 'result': 'CORE_BACKEND_STARTUP_PASS backend=$backend', + 'startup': { + 'schema': 1, + 'backend': backend, + 'mode': 'release', + 'start_boundary': 'Dart main entry before bindings', + 'first_frame_boundary': 'first rasterized frame observed', + 'ready_boundary': 'production ensureInitialized completed', + 'cache': 'fresh process; warm filesystem', + 'first_frame_us': firstFrame, + 'extensions_ready_us': ready, + 'settings_sha256': sha256 + .convert(utf8.encode(jsonEncode(settings.toJson()))) + .toString(), + 'extensions': [ + for (final extension in state.extensions) + { + 'id': extension.id, + 'enabled': extension.enabled, + 'status': extension.status, + }, + ], + }, + }); + } catch (error) { + await _write({'result': 'CORE_BACKEND_STARTUP_FAIL $error'}); + } + } + + Future<void> _write(Map<String, Object?> result) async { + final support = await getApplicationSupportDirectory(); + await File( + '${support.path}/core-backend-probe-result.json', + ).writeAsString(jsonEncode({'pid': pid, ...result}), flush: true); + } +} + const _runtimeProfileTierKey = 'runtime_profile_tier_v1'; Future<void> _prepareAndroidInstallationState(SharedPreferences prefs) async { @@ -220,16 +301,16 @@ class _RuntimeProfile { }; } -class _EagerInitialization extends ConsumerStatefulWidget { - const _EagerInitialization({required this.child}); +class EagerInitialization extends ConsumerStatefulWidget { + const EagerInitialization({super.key, required this.child}); final Widget child; @override - ConsumerState<_EagerInitialization> createState() => + ConsumerState<EagerInitialization> createState() => _EagerInitializationState(); } -class _EagerInitializationState extends ConsumerState<_EagerInitialization> +class _EagerInitializationState extends ConsumerState<EagerInitialization> with WidgetsBindingObserver { ProviderSubscription<bool>? _localLibraryEnabledSub; Timer? _downloadHistoryWarmupTimer; @@ -293,8 +374,7 @@ class _EagerInitializationState extends ConsumerState<_EagerInitialization> ref.read(downloadQueueProvider.notifier).flushQueuePersistence(), ); } - // Backgrounded: return the Go heap's high-water mark to the OS so the - // process is a smaller kill target. + // Backgrounded: release idle native runtimes and connections. unawaited(PlatformBridge.releaseNativeMemory()); } } @@ -302,7 +382,7 @@ class _EagerInitializationState extends ConsumerState<_EagerInitialization> @override void didHaveMemoryPressure() { // OS memory pressure: drop decoded bitmaps (disk caches stay intact) and - // have the Go side release freed heap back to the OS. + // ask the native backend to release disposable runtimes and caches. final imageCache = PaintingBinding.instance.imageCache; imageCache.clear(); imageCache.clearLiveImages(); @@ -414,15 +494,11 @@ class _EagerInitializationState extends ConsumerState<_EagerInitialization> Future<void> _initializeExtensions() async { try { - final storage = await ExtensionStorageService.prepare(); - - await ref - .read(extensionProvider.notifier) - .initialize( - storage.extensionsDir, - storage.dataDir, - masterKey: storage.masterKey, - ); + await ref.read(extensionProvider.notifier).ensureInitialized(); + _startupBenchmark?.extensionsReady( + ref.read(extensionProvider), + ref.read(settingsProvider), + ); if (!mounted) return; NotificationService().verificationNotifications.setHandler(( target, diff --git a/lib/models/settings.dart b/lib/models/settings.dart index df0e1caa..1c4186cc 100644 --- a/lib/models/settings.dart +++ b/lib/models/settings.dart @@ -197,6 +197,15 @@ class AppSettings { this.playerMode = 'external', }); + @JsonKey(includeFromJson: false, includeToJson: false) + Map<String, dynamic> get lyricsFetchOptions => { + 'include_translation_netease': lyricsIncludeTranslationNetease, + 'include_romanization_netease': lyricsIncludeRomanizationNetease, + 'multi_person_word_by_word': lyricsMultiPersonWordByWord, + 'apple_elrc_word_sync': lyricsAppleElrcWordSync, + 'musixmatch_language': musixmatchLanguage, + }; + AppSettings copyWith({ String? defaultService, String? audioQuality, diff --git a/lib/providers/download_queue_provider.dart b/lib/providers/download_queue_provider.dart index 8e80250a..752e3dcd 100644 --- a/lib/providers/download_queue_provider.dart +++ b/lib/providers/download_queue_provider.dart @@ -295,7 +295,7 @@ final _multiUnderscoreRegex = RegExp(r'_+'); double _log10(num x) => log(x) / ln10; final _yearRegex = RegExp(r'^(\d{4})'); const _defaultOutputFolderName = 'SpotiFLAC'; -const _defaultAndroidMusicSubpath = 'Music/$_defaultOutputFolderName'; +const _defaultAndroidDownloadSubpath = 'Download/$_defaultOutputFolderName'; const _maxSafFilenameUtf8Bytes = 180; const _maxSafDirSegmentUtf8Bytes = 120; final _batchUniqueFilenameTokenPattern = RegExp( @@ -554,7 +554,10 @@ class DownloadQueueNotifier extends Notifier<DownloadQueueState> { ); _iosBackgroundExecutionExpired = true; if (state.isProcessing && !state.isPaused) { - pauseQueue(persistAcrossRestarts: false); + pauseQueue( + persistAcrossRestarts: false, + nativeCancelledItemIds: cancelledItemIds, + ); } if (requeueItemIds.isNotEmpty) { @@ -1275,9 +1278,13 @@ class DownloadQueueNotifier extends Notifier<DownloadQueueState> { _pausePendingItemIds.clear(); } - void pauseQueue({bool persistAcrossRestarts = true}) { + void pauseQueue({ + bool persistAcrossRestarts = true, + Set<String> nativeCancelledItemIds = const {}, + }) { if (state.isProcessing && !state.isPaused) { - if (_hasActiveAndroidNativeWorker) { + final nativeWorkerActive = _hasActiveAndroidNativeWorker; + if (nativeWorkerActive) { PlatformBridge.pauseNativeDownloadWorker().catchError((_) {}); } final activeIds = state.items @@ -1292,7 +1299,14 @@ class DownloadQueueNotifier extends Notifier<DownloadQueueState> { if (activeIds.isNotEmpty) { _pausePendingItemIds.addAll(activeIds); for (final id in activeIds) { - _requestNativeCancel(id); + if (nativeWorkerActive || nativeCancelledItemIds.contains(id)) { + // The native worker or iOS expiry already cancelled this attempt. + // A second cancel can arrive after it unwinds and leave a flag + // that aborts the resumed download before it starts. + _verificationWaitCoordinator.cancelItem(id); + } else { + _requestNativeCancel(id); + } _requeueItemForPause(id); } } @@ -1343,7 +1357,7 @@ class DownloadQueueNotifier extends Notifier<DownloadQueueState> { _log.i('Retrying item: ${item.track.name} (id: $id)'); // A cancel issued while the item never started leaves a pre-registered - // flag in the Go backend that the next attempt would consume and abort + // flag in the backend that the next attempt would consume and abort // instantly; the user asked for a retry, so drop it first. try { await PlatformBridge.resetDownloadCancel(id); @@ -1776,7 +1790,7 @@ class DownloadQueueNotifier extends Notifier<DownloadQueueState> { await _initOutputDir(); } - // iOS: Validate that outputDir is writable (not iCloud Drive which Go + // iOS: Validate that outputDir is writable (not iCloud Drive which native code // can't access), unless a bookmark makes this app-Documents path-shape // check irrelevant (see shouldValidateIosOutputDir). if (shouldValidateIosOutputDir( @@ -1793,7 +1807,7 @@ class DownloadQueueNotifier extends Notifier<DownloadQueueState> { _log.w( 'iOS: iCloud Drive path detected, falling back to app Documents folder', ); - _log.w('Go backend cannot write to iCloud Drive due to iOS sandboxing'); + _log.w('Native backend cannot access this iCloud Drive path'); final musicDir = await _ensureDefaultDocumentsOutputDir(); state = state.copyWith(outputDir: musicDir.path); ref.read(settingsProvider.notifier).setDownloadDirectory(musicDir.path); diff --git a/lib/providers/download_queue_provider_embedding.dart b/lib/providers/download_queue_provider_embedding.dart index 09770869..ad99b0f5 100644 --- a/lib/providers/download_queue_provider_embedding.dart +++ b/lib/providers/download_queue_provider_embedding.dart @@ -833,17 +833,17 @@ extension _DownloadQueueEmbedding on DownloadQueueNotifier { '${DateTime.now().millisecondsSinceEpoch}_${Random().nextInt(10000)}'; final coverPath = '${tempDir.path}/cover_embed_$uniqueId.jpg'; - // Go's cover pipeline: shared cache/singleflight, retries, timeouts. + // Mobile backends allocate temporary media inside their owned scope. final result = await PlatformBridge.downloadCoverToFile( coverUrl, - coverPath, + Platform.isAndroid || Platform.isIOS ? '' : coverPath, maxDimension: maxDimension, ); if (result['error'] != null) { _log.w('Failed to download cover: ${result['error']}'); return null; } - return coverPath; + return result['file_path'] as String? ?? coverPath; } catch (e) { _log.e('Failed to download cover for embedding: $e'); return null; diff --git a/lib/providers/download_queue_provider_native_worker.dart b/lib/providers/download_queue_provider_native_worker.dart index 27fd437b..05367cc2 100644 --- a/lib/providers/download_queue_provider_native_worker.dart +++ b/lib/providers/download_queue_provider_native_worker.dart @@ -1382,6 +1382,7 @@ extension _DownloadQueueNativeWorker on DownloadQueueNotifier { final postProcessedPath = await _runPostProcessingHooks( filePath, trackToDownload, + item.id, ); if (postProcessedPath != null && postProcessedPath.isNotEmpty) { filePath = postProcessedPath; diff --git a/lib/providers/download_queue_provider_paths.dart b/lib/providers/download_queue_provider_paths.dart index f0862f7c..90d254e7 100644 --- a/lib/providers/download_queue_provider_paths.dart +++ b/lib/providers/download_queue_provider_paths.dart @@ -6,8 +6,8 @@ extension _DownloadQueuePaths on DownloadQueueNotifier { return Directory('$documentsPath/$_defaultOutputFolderName'); } - Directory _defaultAndroidMusicOutputDir(String storageRootPath) { - return Directory('$storageRootPath/$_defaultAndroidMusicSubpath'); + Directory _defaultAndroidDownloadOutputDir(String storageRootPath) { + return Directory('$storageRootPath/$_defaultAndroidDownloadSubpath'); } Future<Directory> _ensureDefaultDocumentsOutputDir() async { @@ -19,17 +19,17 @@ extension _DownloadQueuePaths on DownloadQueueNotifier { return musicDir; } - Future<Directory?> _ensureDefaultAndroidMusicOutputDir() async { + Future<Directory?> _ensureDefaultAndroidDownloadOutputDir() async { final dir = await getExternalStorageDirectory(); if (dir == null) return null; - final musicDir = _defaultAndroidMusicOutputDir( + final downloadDir = _defaultAndroidDownloadOutputDir( dir.parent.parent.parent.parent.path, ); - if (!await musicDir.exists()) { - await musicDir.create(recursive: true); + if (!await downloadDir.exists()) { + await downloadDir.create(recursive: true); } - return musicDir; + return downloadDir; } Future<Directory?> _ensureAndroidAppSpecificOutputDir() async { @@ -78,7 +78,7 @@ extension _DownloadQueuePaths on DownloadQueueNotifier { Future<Directory> _findWritableAppFolder({String? failedOutputDir}) async { final candidates = <Future<Directory?> Function()>[ - if (Platform.isAndroid) _ensureDefaultAndroidMusicOutputDir, + if (Platform.isAndroid) _ensureDefaultAndroidDownloadOutputDir, if (Platform.isAndroid) _ensureAndroidAppSpecificOutputDir, () async => _ensureDefaultDocumentsOutputDir(), ]; @@ -141,11 +141,10 @@ extension _DownloadQueuePaths on DownloadQueueNotifier { _appFolderStorageFallback = fallback; try { return await fallback; - } catch (_) { + } finally { if (identical(_appFolderStorageFallback, fallback)) { _appFolderStorageFallback = null; } - rethrow; } } @@ -157,7 +156,7 @@ extension _DownloadQueuePaths on DownloadQueueNotifier { state = state.copyWith(outputDir: musicDir.path); } else { final musicDir = - await _ensureDefaultAndroidMusicOutputDir() ?? + await _ensureDefaultAndroidDownloadOutputDir() ?? await _ensureDefaultDocumentsOutputDir(); state = state.copyWith(outputDir: musicDir.path); } diff --git a/lib/providers/download_queue_provider_single_item.dart b/lib/providers/download_queue_provider_single_item.dart index cc5a84bb..0f845ddc 100644 --- a/lib/providers/download_queue_provider_single_item.dart +++ b/lib/providers/download_queue_provider_single_item.dart @@ -12,7 +12,11 @@ extension _SingleItemDownload on DownloadQueueNotifier { ); } - Future<String?> _runPostProcessingHooks(String filePath, Track track) async { + Future<String?> _runPostProcessingHooks( + String filePath, + Track track, + String itemId, + ) async { try { final settings = ref.read(settingsProvider); final extensionState = ref.read(extensionProvider); @@ -48,6 +52,7 @@ extension _SingleItemDownload on DownloadQueueNotifier { final result = await PlatformBridge.runPostProcessingV2( filePath, metadata: metadata, + itemId: itemId, ); if (result['success'] == true) { @@ -126,6 +131,7 @@ class _DownloadRun { /// probe doesn't have to copy the published file back out of SAF. Map<String, dynamic>? probedFinalMetadata; bool externalLrcWritten = false; + final Map<String, String> _directoryScopes = {}; Future<void> _run() async { final normalizedService = n._normalizeQueuedService(item.service); @@ -258,6 +264,14 @@ class _DownloadRun { } catch (e, stackTrace) { await _handleRunException(e, stackTrace); } finally { + for (final token in _directoryScopes.values) { + try { + await PlatformBridge.releaseDownloadDirectory(token); + } catch (e) { + _log.w('Failed to release output directory scope: $e'); + } + } + _directoryScopes.clear(); if (pausedDuringThisRun) { n._pausePendingItemIds.remove(item.id); } @@ -564,6 +578,10 @@ class _DownloadRun { if (!useSaf) { await n._ensureDirExists(outputDir, label: 'Output folder'); + if (!_directoryScopes.containsKey(outputDir)) { + _directoryScopes[outputDir] = + await PlatformBridge.acquireDownloadDirectory(outputDir); + } } _log.d('Output dir: $outputDir'); @@ -740,11 +758,28 @@ class _DownloadRun { final postProcessedPath = await n._runPostProcessingHooks( hookInput, trackToDownload, + item.id, ); if (postProcessedPath != null && postProcessedPath.isNotEmpty) { + if (deferredSafPublish && postProcessedPath != hookInput) { + final output = File(postProcessedPath); + if (!await output.exists() || await output.length() <= 0) { + throw StateError('Post-processing output missing or empty'); + } + await deleteFile(hookInput); + } filePath = postProcessedPath; result['file_path'] = postProcessedPath; } + if (await _shouldAbort( + 'during post-processing', + deleteFileOnAbort: wasExisting ? null : filePath, + )) { + if (!wasExisting && filePath != hookInput) { + await deleteFile(hookInput); + } + return false; + } } final autoConvertInput = filePath; @@ -800,7 +835,8 @@ class _DownloadRun { } final lrcTarget = filePath; - if (effectiveSafMode && lrcTarget != null && isContentUri(lrcTarget)) { + if (lrcTarget != null && + (!wasExisting || (effectiveSafMode && isContentUri(lrcTarget)))) { externalLrcWritten = await n._saveExternalLrc( result: result, settings: settings, @@ -808,7 +844,7 @@ class _DownloadRun { track: trackToDownload, service: item.service, filePath: lrcTarget, - storageMode: 'saf', + storageMode: effectiveSafMode ? 'saf' : 'app', downloadTreeUri: settings.downloadTreeUri, safRelativeDir: effectiveOutputDir, resolveBaseName: () async { @@ -958,9 +994,8 @@ class _DownloadRun { } else if (metadataEmbeddingEnabled && !isContentUriPath && isFlacFile && - !wasExisting && - decryptionDescriptor != null) { - await _embedLocalFlacAfterDecrypt(path); + !wasExisting) { + await _embedLocalFlac(path); } else if (metadataEmbeddingEnabled && !isContentUriPath && effectiveSafMode && @@ -1454,14 +1489,16 @@ class _DownloadRun { } } - Future<void> _embedLocalFlacAfterDecrypt(String currentFilePath) async { - _log.d( - 'Local FLAC after decrypt detected, embedding metadata and cover...', - ); + Future<void> _embedLocalFlac(String currentFilePath) async { + _log.d('Local FLAC detected, embedding metadata and cover...'); try { n.updateItemStatus(item.id, DownloadStatus.finalizing, progress: 0.99); - await _embedFinalMetadata(currentFilePath, format: 'flac'); + await _embedFinalMetadata( + currentFilePath, + format: 'flac', + writeExternalLrc: false, + ); _log.d('Local FLAC metadata embedding completed'); } catch (e) { _log.w('Local FLAC metadata embedding failed: $e'); @@ -1473,7 +1510,7 @@ class _DownloadRun { Future<String?> _embedFinalMetadata( String path, { required String format, - bool writeExternalLrc = true, + bool writeExternalLrc = false, }) async { final track = buildTrackForMetadataEmbedding( trackToDownload, diff --git a/lib/providers/extension_provider.dart b/lib/providers/extension_provider.dart index 5c222bfe..0d6ad0a5 100644 --- a/lib/providers/extension_provider.dart +++ b/lib/providers/extension_provider.dart @@ -5,7 +5,9 @@ import 'package:flutter/widgets.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:path_provider/path_provider.dart'; import 'package:shared_preferences/shared_preferences.dart'; +import 'package:spotiflac_android/models/settings.dart'; import 'package:spotiflac_android/services/platform_bridge.dart'; +import 'package:spotiflac_android/services/extension_storage_service.dart'; import 'package:spotiflac_android/utils/logger.dart'; import 'package:spotiflac_android/providers/settings_provider.dart'; @@ -26,12 +28,23 @@ class ExtensionNotifier extends Notifier<ExtensionState> { AppLifecycleListener? _appLifecycleListener; bool _cleanupInFlight = false; Completer<void>? _initializationCompleter; + Future<void>? _ensureInitializationFuture; + Future<void> _downloadDirectorySync = Future<void>.value(); final Map<String, DateTime> _healthExpiresAt = {}; final Map<String, Future<ExtensionHealthStatus?>> _healthInFlight = {}; final Map<String, int> _healthRequestSerial = {}; @override ExtensionState build() { + ref.listen<AppSettings>(settingsProvider, (previous, next) { + if (state.isInitialized && + (previous?.downloadDirectory != next.downloadDirectory || + previous?.downloadDirectoryBookmark != + next.downloadDirectoryBookmark || + previous?.storageMode != next.storageMode)) { + unawaited(_syncDownloadDirectory(next)); + } + }); _appLifecycleListener ??= AppLifecycleListener( onDetach: _scheduleLifecycleCleanup, ); @@ -45,6 +58,32 @@ class ExtensionNotifier extends Notifier<ExtensionState> { return const ExtensionState(); } + Future<void> _syncDownloadDirectory(AppSettings settings) { + _downloadDirectorySync = _downloadDirectorySync + .then((_) async { + if (!PlatformBridge.supportsCoreBackend) return; + final candidate = settings.downloadDirectory.trim(); + final path = + settings.storageMode == 'app' && + settings.downloadDirectoryBookmark.isEmpty && + candidate.startsWith('/') && + await Directory(candidate).exists() + ? candidate + : ''; + try { + await PlatformBridge.setDownloadDirectory(path); + } catch (_) { + // An invalid new selection must not leave the old permanent grant. + await PlatformBridge.setDownloadDirectory(''); + rethrow; + } + }) + .catchError((Object error) { + _log.w('Failed to sync output directory access: $error'); + }); + return _downloadDirectorySync; + } + void _scheduleLifecycleCleanup() { if (_cleanupInFlight) return; _cleanupInFlight = true; @@ -67,6 +106,34 @@ class ExtensionNotifier extends Notifier<ExtensionState> { } } + /// Startup and install/update actions share initialization, including the + /// private storage preparation that happens before the native call. + Future<void> ensureInitialized() { + if (state.isInitialized) return Future<void>.value(); + return _ensureInitializationFuture ??= _initializeFromStorage() + .whenComplete(() { + _ensureInitializationFuture = null; + }); + } + + Future<void> _initializeFromStorage() async { + final pending = _initializationCompleter?.future; + if (pending != null) { + await pending; + } else { + await ref.read(settingsProvider.notifier).ensureLoaded(); + final storage = await ExtensionStorageService.prepare(); + await initialize( + storage.extensionsDir, + storage.dataDir, + masterKey: storage.masterKey, + ); + } + if (!state.isInitialized) { + throw StateError(state.error ?? 'Extension system initialization failed'); + } + } + Future<void> initialize( String extensionsDir, String dataDir, { @@ -97,6 +164,21 @@ class ExtensionNotifier extends Notifier<ExtensionState> { } try { + final settings = ref.read(settingsProvider.notifier); + await settings.ensureLoaded(); + final snapshot = ref.read(settingsProvider); + final outputDirectory = snapshot.downloadDirectory.trim(); + final allowedDirectories = <String>[]; + // External scopes are acquired by the file/download operation. Missing + // output folders are resolved by the queue and must not prevent startup. + if (snapshot.storageMode == 'app' && + snapshot.downloadDirectoryBookmark.isEmpty && + outputDirectory.startsWith('/')) { + final directory = Directory(outputDirectory); + if (await directory.exists()) { + allowedDirectories.add(directory.path); + } + } if (Platform.isAndroid) { try { await PlatformBridge.prepareRuntimeState(dataDir); @@ -108,11 +190,17 @@ class ExtensionNotifier extends Notifier<ExtensionState> { extensionsDir, dataDir, masterKey: masterKey, + lyricsProviders: snapshot.lyricsProviders, + lyricsFetchOptions: snapshot.lyricsFetchOptions, + allowedDirectories: allowedDirectories, ); await loadExtensions(extensionsDir); + final loadError = state.error; + if (loadError != null) throw StateError(loadError); await loadProviderPriority(); await loadMetadataProviderPriority(); state = state.copyWith(isInitialized: true, isLoading: false); + await _syncDownloadDirectory(ref.read(settingsProvider)); _log.i('Extension system initialized'); } catch (e) { _log.e('Failed to initialize extension system: $e'); @@ -153,7 +241,7 @@ class ExtensionNotifier extends Notifier<ExtensionState> { final result = await PlatformBridge.loadExtensionsFromDir(dirPath); _log.d('Load extensions result: $result'); await refreshExtensions(); - state = state.copyWith(isLoading: false); + state = state.copyWith(isLoading: false, error: state.error); } catch (e) { _log.e('Failed to load extensions: $e'); state = state.copyWith(isLoading: false, error: e.toString()); @@ -306,6 +394,7 @@ class ExtensionNotifier extends Notifier<ExtensionState> { state = state.copyWith(isLoading: true, error: null); try { + await ensureInitialized(); final result = await PlatformBridge.loadExtensionFromPath(filePath); _log.i('Installed extension: ${result['name']}'); await refreshExtensions(); @@ -337,6 +426,17 @@ class ExtensionNotifier extends Notifier<ExtensionState> { var installed = 0; final failures = <String, String>{}; + try { + await ensureInitialized(); + } catch (e) { + state = state.copyWith(isLoading: false, error: e.toString()); + return ExtensionInstallBatchResult( + attempted: uniquePaths.length, + installed: 0, + failures: {for (final path in uniquePaths) path: e.toString()}, + ); + } + for (final path in uniquePaths) { try { final result = await PlatformBridge.loadExtensionFromPath(path); @@ -364,6 +464,7 @@ class ExtensionNotifier extends Notifier<ExtensionState> { Future<Map<String, dynamic>> checkExtensionUpgrade(String filePath) async { try { + await ensureInitialized(); return await PlatformBridge.checkExtensionUpgrade(filePath); } catch (e) { _log.e('Failed to check extension upgrade: $e'); @@ -375,6 +476,7 @@ class ExtensionNotifier extends Notifier<ExtensionState> { state = state.copyWith(isLoading: true, error: null); try { + await ensureInitialized(); final result = await PlatformBridge.upgradeExtension(filePath); _log.i( 'Upgraded extension: ${result['display_name']} to v${result['version']}', @@ -566,6 +668,8 @@ class ExtensionNotifier extends Notifier<ExtensionState> { return const ExtensionRestoreResult(); } + await ensureInitialized(); + final registryUrl = (data['registry_url'] as String?)?.trim() ?? ''; final itemsRaw = data['items']; final items = itemsRaw is List diff --git a/lib/providers/repo_provider.dart b/lib/providers/repo_provider.dart index 5ca2a461..7ad391d1 100644 --- a/lib/providers/repo_provider.dart +++ b/lib/providers/repo_provider.dart @@ -239,6 +239,7 @@ class RepoNotifier extends Notifier<RepoState> { ); try { + await ref.read(extensionProvider.notifier).ensureInitialized(); await PlatformBridge.initExtensionRepo(cacheDir); if (savedUrl.isNotEmpty) { diff --git a/lib/providers/settings_provider.dart b/lib/providers/settings_provider.dart index cf840212..6acaff66 100644 --- a/lib/providers/settings_provider.dart +++ b/lib/providers/settings_provider.dart @@ -116,18 +116,34 @@ class SettingsNotifier extends Notifier<AppSettings> { 2000, }; - final Future<SharedPreferences> _prefs = SharedPreferences.getInstance(); + Future<SharedPreferences> get _prefs => SharedPreferences.getInstance(); final FlutterSecureStorage _secureStorage = const FlutterSecureStorage(); bool _isSavingSettings = false; bool _saveQueued = false; String? _pendingSettingsJson; + Future<void>? _loadSettingsFuture; @override AppSettings build() { - unawaited(_loadSettings()); + unawaited( + ensureLoaded().catchError((Object error, StackTrace stack) { + _log.e('Failed to load settings', error, stack); + }), + ); return ref.read(initialSettingsProvider); } + /// Startup and extension actions share loading, including preference migrations. + Future<void> ensureLoaded() { + return _loadSettingsFuture ??= _loadSettings().catchError(( + Object error, + StackTrace stack, + ) { + _loadSettingsFuture = null; + Error.throwWithStackTrace(error, stack); + }); + } + Future<void> _loadSettings() async { final prefs = await _prefs; final rawSettings = prefs.getString(_settingsKey); @@ -197,9 +213,9 @@ class SettingsNotifier extends Notifier<AppSettings> { LogBuffer.loggingEnabled = state.enableLogging; - _syncLyricsSettingsToBackend(); - _syncNetworkCompatibilitySettingsToBackend(); - _syncExtensionFallbackSettingsToBackend(); + await syncLyricsSettingsToBackend(); + await _syncNetworkCompatibilitySettingsToBackend(); + await _syncExtensionFallbackSettingsToBackend(); } void _syncLyricsSettingsToBackend() { @@ -217,41 +233,34 @@ class SettingsNotifier extends Notifier<AppSettings> { } try { - await PlatformBridge.setLyricsFetchOptions({ - 'include_translation_netease': snapshot.lyricsIncludeTranslationNetease, - 'include_romanization_netease': - snapshot.lyricsIncludeRomanizationNetease, - 'multi_person_word_by_word': snapshot.lyricsMultiPersonWordByWord, - 'apple_elrc_word_sync': snapshot.lyricsAppleElrcWordSync, - 'musixmatch_language': snapshot.musixmatchLanguage, - }); + await PlatformBridge.setLyricsFetchOptions(snapshot.lyricsFetchOptions); } catch (e) { _log.w('Failed to sync lyrics fetch options to backend: $e'); } } - void _syncNetworkCompatibilitySettingsToBackend() { + Future<void> _syncNetworkCompatibilitySettingsToBackend() async { if (!PlatformBridge.supportsCoreBackend) return; final compatibilityMode = state.networkCompatibilityMode; - PlatformBridge.setNetworkCompatibilityOptions( + await PlatformBridge.setNetworkCompatibilityOptions( allowHttp: compatibilityMode, insecureTls: false, ).catchError((Object e) { _log.w('Failed to sync network compatibility options to backend: $e'); }); - PlatformBridge.setAllowPrivateNetwork(state.allowLocalNetwork).catchError(( - Object e, - ) { + await PlatformBridge.setAllowPrivateNetwork( + state.allowLocalNetwork, + ).catchError((Object e) { _log.w('Failed to sync allow local network option to backend: $e'); }); } - void _syncExtensionFallbackSettingsToBackend() { + Future<void> _syncExtensionFallbackSettingsToBackend() async { if (!PlatformBridge.supportsCoreBackend) return; - PlatformBridge.setDownloadFallbackExtensionIds( + await PlatformBridge.setDownloadFallbackExtensionIds( state.downloadFallbackExtensionIds, ).catchError((Object e) { _log.w('Failed to sync extension fallback settings to backend: $e'); @@ -348,8 +357,8 @@ class SettingsNotifier extends Notifier<AppSettings> { LogBuffer.loggingEnabled = state.enableLogging; _syncLyricsSettingsToBackend(); - _syncNetworkCompatibilitySettingsToBackend(); - _syncExtensionFallbackSettingsToBackend(); + await _syncNetworkCompatibilitySettingsToBackend(); + await _syncExtensionFallbackSettingsToBackend(); } Future<void> _normalizeIosDownloadDirectoryIfNeeded() async { @@ -727,7 +736,7 @@ class SettingsNotifier extends Notifier<AppSettings> { extensionIds == null && state.downloadFallbackExtensionIds != null, ); _saveSettings(); - _syncExtensionFallbackSettingsToBackend(); + unawaited(_syncExtensionFallbackSettingsToBackend()); } void setSeparateSingles(bool enabled) { @@ -810,13 +819,13 @@ class SettingsNotifier extends Notifier<AppSettings> { void setNetworkCompatibilityMode(bool enabled) { state = state.copyWith(networkCompatibilityMode: enabled); _saveSettings(); - _syncNetworkCompatibilitySettingsToBackend(); + unawaited(_syncNetworkCompatibilitySettingsToBackend()); } void setAllowLocalNetwork(bool enabled) { state = state.copyWith(allowLocalNetwork: enabled); _saveSettings(); - _syncNetworkCompatibilitySettingsToBackend(); + unawaited(_syncNetworkCompatibilitySettingsToBackend()); } void setSongLinkRegion(String region) { diff --git a/lib/screens/main_shell.dart b/lib/screens/main_shell.dart index ea307603..a80dd84e 100644 --- a/lib/screens/main_shell.dart +++ b/lib/screens/main_shell.dart @@ -186,7 +186,7 @@ class _MainShellState extends ConsumerState<MainShell> notifier.setDownloadDirectory(''); Navigator.of(dialogContext).pop(); }, - child: Text(context.l10n.storageModeAppFolder), + child: Text(context.l10n.storageAutomaticFolder), ), FilledButton( onPressed: isPickingFolder diff --git a/lib/screens/settings/files_settings_page.dart b/lib/screens/settings/files_settings_page.dart index f52138e3..aee04866 100644 --- a/lib/screens/settings/files_settings_page.dart +++ b/lib/screens/settings/files_settings_page.dart @@ -139,7 +139,7 @@ class _FilesSettingsPageState extends ConsumerState<FilesSettingsPage> { subtitle: settings.downloadDirectory.isEmpty ? (Platform.isIOS ? context.l10n.setupAppDocumentsFolder - : 'Music/SpotiFLAC') + : 'Download/SpotiFLAC') : settings.downloadDirectory, onTap: () => _pickDirectory(context, ref), showDivider: false, @@ -455,20 +455,24 @@ class _FilesSettingsPageState extends ConsumerState<FilesSettingsPage> { } Future<String> _getDefaultAndroidDirectory() async { - const directMusicPath = '/storage/emulated/0/Music/SpotiFLAC'; + const directDownloadPath = '/storage/emulated/0/Download/SpotiFLAC'; try { - final musicDir = Directory(directMusicPath); - if (!await musicDir.exists()) await musicDir.create(recursive: true); - return musicDir.path; + final downloadDir = Directory(directDownloadPath); + if (!await downloadDir.exists()) { + await downloadDir.create(recursive: true); + } + return downloadDir.path; } catch (_) {} try { final externalDir = await getExternalStorageDirectory(); if (externalDir != null) { - final musicDir = Directory( - '${externalDir.parent.parent.parent.parent.path}/Music/SpotiFLAC', + final downloadDir = Directory( + '${externalDir.parent.parent.parent.parent.path}/Download/SpotiFLAC', ); - if (!await musicDir.exists()) await musicDir.create(recursive: true); - return musicDir.path; + if (!await downloadDir.exists()) { + await downloadDir.create(recursive: true); + } + return downloadDir.path; } } catch (_) {} final appDir = await getApplicationDocumentsDirectory(); @@ -511,8 +515,8 @@ class _FilesSettingsPageState extends ConsumerState<FilesSettingsPage> { ), ListTile( leading: Icon(Icons.folder_special, color: colorScheme.primary), - title: Text(context.l10n.storageModeAppFolder), - subtitle: Text(context.l10n.storageModeAppFolderSubtitle), + title: Text(context.l10n.storageAutomaticFolder), + subtitle: Text(context.l10n.storageAutomaticDownloadFolder), trailing: !isSafMode ? const Icon(Icons.check) : null, onTap: () async { Navigator.pop(ctx); @@ -525,8 +529,8 @@ class _FilesSettingsPageState extends ConsumerState<FilesSettingsPage> { ), ListTile( leading: Icon(Icons.folder_open, color: colorScheme.primary), - title: Text(context.l10n.storageModeSaf), - subtitle: Text(context.l10n.storageModeSafSubtitle), + title: Text(context.l10n.storageSafRecommended), + subtitle: Text(context.l10n.storageDownloadFolderHint), trailing: isSafMode ? const Icon(Icons.check) : null, onTap: () async { Navigator.pop(ctx); diff --git a/lib/screens/settings/log_screen.dart b/lib/screens/settings/log_screen.dart index 4a675c92..d5471ef9 100644 --- a/lib/screens/settings/log_screen.dart +++ b/lib/screens/settings/log_screen.dart @@ -640,7 +640,7 @@ class _LogEntryTile extends StatelessWidget { borderRadius: BorderRadius.circular(4), ), child: Text( - context.l10n.actionGo, + 'Native', style: const TextStyle( fontSize: 9, fontWeight: FontWeight.bold, diff --git a/lib/screens/setup_screen.dart b/lib/screens/setup_screen.dart index 1636cad3..584d396e 100644 --- a/lib/screens/setup_screen.dart +++ b/lib/screens/setup_screen.dart @@ -425,14 +425,14 @@ class _SetupScreenState extends ConsumerState<SetupScreen> { Future<String> _getDefaultDirectory() async { if (Platform.isAndroid) { - final musicDir = Directory('/storage/emulated/0/Music/SpotiFLAC'); + final downloadDir = Directory('/storage/emulated/0/Download/SpotiFLAC'); try { - if (!await musicDir.exists()) { - await musicDir.create(recursive: true); + if (!await downloadDir.exists()) { + await downloadDir.create(recursive: true); } - return musicDir.path; + return downloadDir.path; } catch (e) { - debugPrint('Cannot create Music folder: $e'); + debugPrint('Cannot create Download folder: $e'); } } final appDir = await getApplicationDocumentsDirectory(); @@ -923,7 +923,9 @@ class _SetupScreenState extends ConsumerState<SetupScreen> { Widget _buildDirectoryStep(ColorScheme colorScheme) { return _StepLayout( title: context.l10n.setupFolderChoose, - description: context.l10n.setupFolderDescription, + description: Platform.isAndroid + ? context.l10n.storageDownloadFolderHint + : context.l10n.setupFolderDescription, icon: Icons.create_new_folder, child: Column( children: [ diff --git a/lib/services/cover_download_service.dart b/lib/services/cover_download_service.dart index 98ba2d42..ec9846ff 100644 --- a/lib/services/cover_download_service.dart +++ b/lib/services/cover_download_service.dart @@ -31,16 +31,17 @@ class CoverDownloadService { ? 'cover' : safeBaseName.trim(); final tempDir = await Directory.systemTemp.createTemp('save_cover_'); - final tempPath = p.join(tempDir.path, 'cover.image'); + var tempPath = p.join(tempDir.path, 'cover.image'); IosSecurityScopedAccess? iosBookmarkAccess; try { final download = await PlatformBridge.downloadCoverToFile( normalizedUrl, - tempPath, + Platform.isAndroid || Platform.isIOS ? '' : tempPath, ); final error = download['error']?.toString().trim() ?? ''; if (error.isNotEmpty) throw StateError(error); + tempPath = download['file_path'] as String? ?? tempPath; final tempFile = File(tempPath); if (!await tempFile.exists() || await tempFile.length() <= 0) { @@ -104,6 +105,8 @@ class CoverDownloadService { await PlatformBridge.stopAccessingIosBookmark(iosBookmarkAccess); } try { + final tempFile = File(tempPath); + if (await tempFile.exists()) await tempFile.delete(); if (await tempDir.exists()) await tempDir.delete(recursive: true); } catch (_) {} } diff --git a/lib/services/extension_storage_service.dart b/lib/services/extension_storage_service.dart index c1b33cc5..844046ea 100644 --- a/lib/services/extension_storage_service.dart +++ b/lib/services/extension_storage_service.dart @@ -27,7 +27,15 @@ class ExtensionStorageService { static Future<ExtensionStoragePaths>? _preparing; static Future<ExtensionStoragePaths> prepare() { - return _preparing ??= _prepare(); + return _preparing ??= _prepare().catchError(( + Object error, + StackTrace stack, + ) { + // A transient filesystem/keystore failure must not poison every retry + // until the process is restarted. + _preparing = null; + Error.throwWithStackTrace(error, stack); + }); } static Future<ExtensionStoragePaths> _prepare() async { diff --git a/lib/services/ffmpeg_service.dart b/lib/services/ffmpeg_service.dart index 2893f354..91ce9eed 100644 --- a/lib/services/ffmpeg_service.dart +++ b/lib/services/ffmpeg_service.dart @@ -2117,7 +2117,7 @@ class FFmpegService { return null; } - // Write tags + cover via the native ID3-chunk writer in the Go backend. + // Write tags + cover via the native ID3-chunk writer. final hasMetadata = metadata.values.any((v) => v.trim().isNotEmpty); final hasCover = coverPath != null && coverPath.trim().isNotEmpty; if (hasMetadata || hasCover) { diff --git a/lib/services/history_database.dart b/lib/services/history_database.dart index b019dc78..d938b812 100644 --- a/lib/services/history_database.dart +++ b/lib/services/history_database.dart @@ -5,6 +5,7 @@ import 'package:path_provider/path_provider.dart'; import 'package:shared_preferences/shared_preferences.dart'; import 'package:spotiflac_android/services/sqlite_helpers.dart' as sqlite; import 'package:spotiflac_android/utils/isrc_utils.dart' as isrc; +import 'package:spotiflac_android/utils/ios_container_paths.dart'; import 'package:spotiflac_android/utils/logger.dart'; import 'package:spotiflac_android/utils/path_match_keys.dart'; @@ -507,21 +508,13 @@ class HistoryDatabase { void _putPathKeysInBatch(Batch batch, String id, String? filePath) => sqlite.putPathKeysInBatch(batch, 'history_path_keys', id, filePath); - static final _iosContainerPattern = RegExp( - r'/var/mobile/Containers/Data/Application/[A-F0-9\-]+/', - caseSensitive: false, - ); - Future<void> _initContainerPath() async { if (!Platform.isIOS || _currentContainerPath != null) return; try { final docDir = await getApplicationDocumentsDirectory(); - final match = _iosContainerPattern.firstMatch(docDir.path); - if (match != null) { - _currentContainerPath = match.group(0); - _log.d('iOS container path: $_currentContainerPath'); - } + _currentContainerPath = docDir.parent.path; + _log.d('iOS container path: $_currentContainerPath'); } catch (e) { _log.w('Failed to get iOS container path: $e'); } @@ -531,18 +524,7 @@ class HistoryDatabase { if (filePath == null || filePath.isEmpty) return filePath ?? ''; if (!Platform.isIOS || _currentContainerPath == null) return filePath; - if (_iosContainerPattern.hasMatch(filePath)) { - final normalized = filePath.replaceFirst( - _iosContainerPattern, - _currentContainerPath!, - ); - if (normalized != filePath) { - _log.d('Normalized iOS path: $filePath -> $normalized'); - } - return normalized; - } - - return filePath; + return rebaseIosSandboxPath(filePath, '$_currentContainerPath/Documents'); } Future<bool> migrateIosContainerPaths() async { @@ -572,7 +554,7 @@ class HistoryDatabase { final id = row['id'] as String; final oldPath = row['file_path'] as String?; - if (oldPath != null && _iosContainerPattern.hasMatch(oldPath)) { + if (oldPath != null) { final newPath = _normalizeIosPath(oldPath); if (newPath != oldPath) { batch.update( diff --git a/lib/services/library_database.dart b/lib/services/library_database.dart index 2434a14e..e9f62ee0 100644 --- a/lib/services/library_database.dart +++ b/lib/services/library_database.dart @@ -6,6 +6,7 @@ import 'package:path_provider/path_provider.dart'; import 'package:spotiflac_android/utils/logger.dart'; import 'package:spotiflac_android/utils/audio_format_utils.dart'; import 'package:spotiflac_android/utils/file_access.dart'; +import 'package:spotiflac_android/utils/ios_container_paths.dart'; import 'package:spotiflac_android/services/history_database.dart'; import 'package:spotiflac_android/services/library_cleanup.dart'; import 'package:spotiflac_android/services/sqlite_helpers.dart' as sqlite; @@ -48,6 +49,7 @@ class LibraryDatabase { version: schemaVersion, onCreate: _createDB, onUpgrade: _upgradeDB, + onOpen: _migrateIosContainerPaths, ); // Library upserts use INSERT OR REPLACE. Recursive triggers ensure the // implicit delete also decrements materialized lookup ref-counts. @@ -62,6 +64,56 @@ class LibraryDatabase { bool get searchFtsAvailable => _searchFtsAvailable ?? false; + Future<void> _migrateIosContainerPaths(Database db) async { + if (!Platform.isIOS) return; + final documents = await getApplicationDocumentsDirectory(); + // Keep rows and their lookup keys consistent before any startup cleanup + // can mistake a relocated file for a deleted one. Bookmarks remain the + // authority for external sources. + await db.transaction((txn) async { + const localSources = "bookmark IS NULL OR bookmark = ''"; + final sources = await txn.query( + 'library_sources', + columns: ['id', 'path'], + where: localSources, + ); + final rows = await txn.query( + 'library', + columns: ['id', 'file_path', 'cover_path'], + where: + 'source_id IN (SELECT id FROM library_sources WHERE $localSources)', + ); + final batch = txn.batch(); + for (final row in rows) { + final updates = <String, Object?>{}; + for (final column in ['file_path', 'cover_path']) { + final previous = row[column] as String?; + if (previous == null) continue; + final current = rebaseIosSandboxPath(previous, documents.path); + if (current != previous) updates[column] = current; + } + if (updates.isEmpty) continue; + final id = row['id'] as String; + batch.update('library', updates, where: 'id = ?', whereArgs: [id]); + if (updates.containsKey('file_path')) { + _putPathKeysInBatch(batch, id, updates['file_path'] as String); + } + } + for (final source in sources) { + final previous = source['path'] as String; + final current = rebaseIosSandboxPath(previous, documents.path); + if (current == previous) continue; + batch.update( + 'library_sources', + {'path': current}, + where: 'id = ?', + whereArgs: [source['id']], + ); + } + await batch.commit(noResult: true); + }); + } + Future<void> _ensureHistoryAttached(Database db) async { if (_historyAttached) return; await HistoryDatabase.instance.database; diff --git a/lib/services/platform_bridge.dart b/lib/services/platform_bridge.dart index 236a46a7..0c48cf9b 100644 --- a/lib/services/platform_bridge.dart +++ b/lib/services/platform_bridge.dart @@ -727,6 +727,20 @@ class PlatformBridge { await _channel.invokeMethod('setDownloadDirectory', {'path': path}); } + /// Holds backend access through download and finalization, including folder changes. + static Future<String> acquireDownloadDirectory(String path) async { + if (!supportsCoreBackend) return ''; + return await _channel.invokeMethod<String>('acquireDownloadDirectory', { + 'path': path, + }) ?? + ''; + } + + static Future<void> releaseDownloadDirectory(String token) async { + if (!supportsCoreBackend || token.isEmpty) return; + await _channel.invokeMethod('releaseDownloadDirectory', {'token': token}); + } + static Future<void> setNetworkCompatibilityOptions({ required bool allowHttp, required bool insecureTls, @@ -741,6 +755,17 @@ class PlatformBridge { await _channel.invokeMethod('setAllowPrivateNetwork', {'allowed': allowed}); } + /// Reports domain ownership during native backend migration. + static Future<Map<String, String>> getBackendImplementations({ + String? filePath, + }) async { + final result = await _channel.invokeMapMethod<String, String>( + 'getBackendImplementations', + filePath == null ? null : {'file_path': filePath}, + ); + return result ?? const {}; + } + static Future<String> buildFilename( String template, Map<String, dynamic> metadata, @@ -1040,9 +1065,11 @@ class PlatformBridge { String audioPath, String outputPath, ) { - return _invokeMap('extractCoverToFile', { - 'audio_path': audioPath, - 'output_path': outputPath, + return _withMediaFileAccess([audioPath, outputPath], () { + return _invokeMap('extractCoverToFile', { + 'audio_path': audioPath, + 'output_path': outputPath, + }); }); } @@ -1100,7 +1127,12 @@ class PlatformBridge { static Future<Map<String, dynamic>> reEnrichFile( Map<String, dynamic> request, ) { - return _invokeMap('reEnrichFile', {'request_json': jsonEncode(request)}); + Future<Map<String, dynamic>> invoke() => + _invokeMap('reEnrichFile', {'request_json': jsonEncode(request)}); + if (request['preview_only'] == true) return invoke(); + return _withMediaFileAccess([ + request['file_path'] as String? ?? '', + ], invoke); } static Future<Map<String, dynamic>> readFileMetadata( @@ -1136,12 +1168,41 @@ class PlatformBridge { String filePath, Map<String, String> metadata, ) { - return _invokeMap('editFileMetadata', { - 'file_path': filePath, - 'metadata_json': jsonEncode(metadata), + return _withMediaFileAccess([filePath, metadata['cover_path'] ?? ''], () { + return _invokeMap('editFileMetadata', { + 'file_path': filePath, + 'metadata_json': jsonEncode(metadata), + }); }); } + static Future<T> _withMediaFileAccess<T>( + Iterable<String> paths, + Future<T> Function() operation, + ) async { + // A user-selected library file or cover can live outside the download + // folder. Hold only this operation's backend grants; SAF is staged natively. + final scopes = <String>[]; + try { + final directories = <String>{ + for (final path in paths) + if (path.startsWith('/')) File(path).parent.path, + }; + for (final directory in directories) { + scopes.add(await acquireDownloadDirectory(directory)); + } + return await operation(); + } finally { + for (final scope in scopes.reversed) { + try { + await releaseDownloadDirectory(scope); + } catch (e) { + _log.w('Failed to release media file access: $e'); + } + } + } + } + /// Writes ISRC and label into an M4A/MP4 file as iTunes freeform atoms. /// FFmpeg's MP4 muxer drops these keys, so they must be written natively /// after the FFmpeg metadata pass. [filePath] must be a local file path. @@ -1150,9 +1211,11 @@ class PlatformBridge { String filePath, Map<String, String> fields, ) { - return _invokeMap('writeM4AFreeformTags', { - 'file_path': filePath, - 'metadata_json': jsonEncode(fields), + return _withMediaFileAccess([filePath], () { + return _invokeMap('writeM4AFreeformTags', { + 'file_path': filePath, + 'metadata_json': jsonEncode(fields), + }); }); } @@ -1164,9 +1227,11 @@ class PlatformBridge { String filePath, String sourcePath, ) { - return _invokeMap('ensureAC4Config', { - 'file_path': filePath, - 'source_path': sourcePath, + return _withMediaFileAccess([filePath, sourcePath], () { + return _invokeMap('ensureAC4Config', { + 'file_path': filePath, + 'source_path': sourcePath, + }); }); } @@ -1179,24 +1244,28 @@ class PlatformBridge { Map<String, String> metadata, String coverPath, ) { - return _invokeMap('writeAC4Metadata', { - 'file_path': filePath, - 'metadata_json': jsonEncode(metadata), - 'cover_path': coverPath, + return _withMediaFileAccess([filePath, coverPath], () { + return _invokeMap('writeAC4Metadata', { + 'file_path': filePath, + 'metadata_json': jsonEncode(metadata), + 'cover_path': coverPath, + }); }); } /// Rewrites ARTIST/ALBUMARTIST Vorbis comments as multiple split entries - /// using the native Go FLAC writer, fixing FFmpeg's tag deduplication. + /// using the selected native FLAC writer, fixing FFmpeg's tag deduplication. static Future<Map<String, dynamic>> rewriteSplitArtistTags( String filePath, String artist, String albumArtist, ) { - return _invokeMap('rewriteSplitArtistTags', { - 'file_path': filePath, - 'artist': artist, - 'album_artist': albumArtist, + return _withMediaFileAccess([filePath], () { + return _invokeMap('rewriteSplitArtistTags', { + 'file_path': filePath, + 'artist': artist, + 'album_artist': albumArtist, + }); }); } @@ -1493,7 +1562,7 @@ class PlatformBridge { await _channel.invokeMethod('clearLogs'); } - /// Ask the Go backend to GC and return freed heap to the OS. Best-effort: + /// Ask the native backend to release unused memory. Best-effort: /// safe to call on memory pressure or when the app is backgrounded. static Future<void> releaseNativeMemory({bool underPressure = false}) async { try { @@ -1524,12 +1593,18 @@ class PlatformBridge { String extensionsDir, String dataDir, { required String masterKey, + required List<String> lyricsProviders, + required Map<String, dynamic> lyricsFetchOptions, + List<String> allowedDirectories = const [], }) async { _log.d('initExtensionSystem: $extensionsDir, $dataDir'); await _channel.invokeMethod('initExtensionSystem', { 'extensions_dir': extensionsDir, 'data_dir': dataDir, 'master_key': masterKey, + 'lyrics_providers_json': jsonEncode(lyricsProviders), + 'lyrics_options_json': jsonEncode(lyricsFetchOptions), + 'allowed_directories': allowedDirectories, }); } @@ -2513,17 +2588,34 @@ class PlatformBridge { static Future<Map<String, dynamic>> runPostProcessingV2( String filePath, { Map<String, dynamic>? metadata, + String? itemId, }) async { final input = <String, dynamic>{}; + if (itemId != null && itemId.isNotEmpty) input['item_id'] = itemId; if (filePath.startsWith('content://')) { input['uri'] = filePath; } else { input['path'] = filePath; } - return _invokeMap('runPostProcessingV2', { + final result = await _invokeMap('runPostProcessingV2', { 'input': jsonEncode(input), 'metadata': metadata != null ? jsonEncode(metadata) : '', }); + if (result['success'] != true) return result; + + // The Go/Rust result uses new_file_path; the queue consumes file_path. + // SAF publishes back to its document URI after processing a temporary file. + final uri = result['new_file_uri']; + final path = result['new_file_path']; + final legacyPath = result['file_path']; + if (filePath.startsWith('content://') && uri is String && uri.isNotEmpty) { + result['file_path'] = uri; + } else if (path is String && path.isNotEmpty) { + result['file_path'] = path; + } else if (legacyPath is! String || legacyPath.isEmpty) { + result['file_path'] = filePath; + } + return result; } static Future<void> initExtensionRepo(String cacheDir) async { diff --git a/lib/services/sqlite_helpers.dart b/lib/services/sqlite_helpers.dart index 0d27cdd7..edbba964 100644 --- a/lib/services/sqlite_helpers.dart +++ b/lib/services/sqlite_helpers.dart @@ -50,6 +50,7 @@ Future<Database> openAppDatabase( required Future<void> Function(Database db, int version) onCreate, required Future<void> Function(Database db, int oldVersion, int newVersion) onUpgrade, + Future<void> Function(Database db)? onOpen, bool foreignKeys = false, bool incrementalAutoVacuum = true, }) async { @@ -94,6 +95,7 @@ Future<Database> openAppDatabase( }, onCreate: onCreate, onUpgrade: onUpgrade, + onOpen: onOpen, ); } diff --git a/lib/utils/ffmpeg_reenrich.dart b/lib/utils/ffmpeg_reenrich.dart index dfa1257a..7b5a69cf 100644 --- a/lib/utils/ffmpeg_reenrich.dart +++ b/lib/utils/ffmpeg_reenrich.dart @@ -49,10 +49,12 @@ Future<bool> applyFfmpegReEnrichResult({ try { if (!_hasValue(effectiveCoverPath)) { + Directory? extractionDirectory; try { final tempDir = await Directory.systemTemp.createTemp( 'reenrich_cover_', ); + extractionDirectory = tempDir; final coverOutput = '${tempDir.path}${Platform.pathSeparator}cover.jpg'; final extracted = await PlatformBridge.extractCoverToFile( ffmpegTarget, @@ -61,12 +63,15 @@ Future<bool> applyFfmpegReEnrichResult({ if (extracted['error'] == null) { effectiveCoverPath = coverOutput; extractedCoverPath = coverOutput; - } else { + } + } catch (_) { + } finally { + if (extractedCoverPath == null && extractionDirectory != null) { try { - await tempDir.delete(recursive: true); + await extractionDirectory.delete(recursive: true); } catch (_) {} } - } catch (_) {} + } } final metadata = (result['metadata'] as Map<String, dynamic>?)?.map( diff --git a/lib/utils/file_access.dart b/lib/utils/file_access.dart index 07e658fd..97f063da 100644 --- a/lib/utils/file_access.dart +++ b/lib/utils/file_access.dart @@ -4,6 +4,7 @@ import 'package:open_filex/open_filex.dart'; import 'package:path_provider/path_provider.dart'; import 'package:spotiflac_android/services/music_player_service.dart'; import 'package:spotiflac_android/services/platform_bridge.dart'; +import 'package:spotiflac_android/utils/ios_container_paths.dart'; import 'package:spotiflac_android/utils/mime_utils.dart'; /// Whether the queue should run its iOS output-directory path-shape check @@ -65,7 +66,7 @@ String _joinRecoveredIosPath(String documentsPath, String suffix) { /// Checks if a path is a valid writable directory on iOS. /// Returns false if: /// - The path is the app container root (not writable) -/// - The path is an iCloud Drive path (not accessible by Go backend) +/// - The path is an iCloud Drive path (not accessible by the backend) /// - The path is outside the app sandbox bool isValidIosWritablePath(String path) { if (!Platform.isIOS) return true; @@ -109,8 +110,8 @@ Future<String> validateOrFixIosPath( }) async { if (!Platform.isIOS) return path; - final trimmed = path.trim(); final docDir = await getApplicationDocumentsDirectory(); + final trimmed = rebaseIosSandboxPath(path.trim(), docDir.path); final nestedLegacyMatch = _iosNestedLegacyDocumentsPattern.firstMatch( trimmed, diff --git a/lib/utils/ios_container_paths.dart b/lib/utils/ios_container_paths.dart new file mode 100644 index 00000000..81edb8bc --- /dev/null +++ b/lib/utils/ios_container_paths.dart @@ -0,0 +1,32 @@ +final _iosSandboxRoot = RegExp( + r'^(?:/private)?/var/mobile/Containers/Data/Application/[A-F0-9\-]+(?=/)' + r'|^/[^\n]+/Library/Developer/CoreSimulator/Devices/[A-F0-9\-]+' + r'/data/Containers/Data/Application/[A-F0-9\-]+(?=/)', + caseSensitive: false, +); + +/// Restores an app-owned path after iOS relocates the app's data container. +/// Bookmarked external folders must be resolved through their bookmark instead. +String rebaseIosSandboxPath(String path, String documentsPath) { + final current = _iosSandboxRoot.firstMatch(documentsPath); + final previous = _iosSandboxRoot.firstMatch(path); + if (current == null || + previous == null || + documentsPath.substring(current.end) != '/Documents') { + return path; + } + + final suffix = path.substring(previous.end); + const appDirectories = [ + '/Documents', + '/Library/Application Support', + '/Library/Caches', + ]; + if (!appDirectories.any( + (directory) => suffix == directory || suffix.startsWith('$directory/'), + ) || + suffix.split('/').contains('..')) { + return path; + } + return '${documentsPath.substring(0, current.end)}$suffix'; +} diff --git a/lib/utils/logger.dart b/lib/utils/logger.dart index fd2d0b87..880c6092 100644 --- a/lib/utils/logger.dart +++ b/lib/utils/logger.dart @@ -106,8 +106,8 @@ class LogEntry { @override String toString() { final errorPart = error != null ? ' | $error' : ''; - final goPart = isFromGo ? ' [Go]' : ''; - return '[$formattedTime] [$level]$goPart [$tag] $message$errorPart'; + final nativePart = isFromGo ? ' [Native]' : ''; + return '[$formattedTime] [$level]$nativePart [$tag] $message$errorPart'; } } @@ -209,7 +209,7 @@ class LogBuffer extends ChangeNotifier { } final timestamp = logMap['timestamp'] as String? ?? ''; - final tag = logMap['tag'] as String? ?? 'Go'; + final tag = logMap['tag'] as String? ?? 'Native'; final message = logMap['message'] as String? ?? ''; DateTime parsedTime = DateTime.now(); @@ -245,7 +245,7 @@ class LogBuffer extends ChangeNotifier { _lastGoLogIndex = nextIndex; } catch (e) { if (kDebugMode) { - debugPrint('Failed to fetch Go logs: $e'); + debugPrint('Failed to fetch native backend logs: $e'); } } } @@ -353,7 +353,7 @@ class LogBuffer extends ChangeNotifier { buffer.writeln('Warnings: $warnCount'); buffer.writeln('Info: $infoCount'); buffer.writeln('Debug: $debugCount'); - buffer.writeln('From Go Backend: $goCount'); + buffer.writeln('From Native Backend: $goCount'); buffer.writeln(); buffer.writeln('=' * 60); diff --git a/lib/utils/lyrics_metadata_helper.dart b/lib/utils/lyrics_metadata_helper.dart index 9aa23abd..243f7bb4 100644 --- a/lib/utils/lyrics_metadata_helper.dart +++ b/lib/utils/lyrics_metadata_helper.dart @@ -83,7 +83,7 @@ String _sidecarLrcPath(String path) { return '$path.lrc'; } -/// Writes a ".lrc" sidecar next to a re-enriched audio file when the Go backend +/// Writes a ".lrc" sidecar next to a re-enriched audio file when the backend /// result requests it (`write_external_lrc`), honoring the user's lyrics mode. /// /// This handles the filesystem case only. SAF (`content://`) files are written diff --git a/pubspec.yaml b/pubspec.yaml index 60f9f10b..06005f88 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,7 +1,7 @@ name: spotiflac_android description: Extension-driven music metadata and downloads for mobile devices publish_to: "none" -version: 4.9.6+144 +version: 5.0.0+147 environment: sdk: ^3.10.0 diff --git a/rust_backend/Cargo.lock b/rust_backend/Cargo.lock new file mode 100644 index 00000000..80f58907 --- /dev/null +++ b/rust_backend/Cargo.lock @@ -0,0 +1,3434 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", + "zeroize", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "askama" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75363874b771be265f4ffe307ca705ef6f3baa19011c149da8674a87f1b75c4" +dependencies = [ + "askama_derive", + "itoa", + "percent-encoding", + "serde", + "serde_json", +] + +[[package]] +name = "askama_derive" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "129397200fe83088e8a68407a8e2b1f826cf0086b21ccdb866a722c8bcd3a94f" +dependencies = [ + "askama_parser", + "basic-toml", + "memchr", + "proc-macro2", + "quote", + "rustc-hash", + "serde", + "serde_derive", + "syn 2.0.119", +] + +[[package]] +name = "askama_parser" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ab5630b3d5eaf232620167977f95eb51f3432fc76852328774afbd242d4358" +dependencies = [ + "memchr", + "serde", + "serde_derive", + "winnow 0.7.15", +] + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-compression" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blowfish" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7" +dependencies = [ + "byteorder", + "cipher", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb1307f12aa967b5a58416e87b3653360e0fd614a016b6e970db08fecbb1b80d" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cap-primitives" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", +] + +[[package]] +name = "cap-std" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes 3.0.1", + "rustix", +] + +[[package]] +name = "cargo-platform" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cc" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +dependencies = [ + "find-msvc-tools", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" + +[[package]] +name = "convert_case" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "time", + "version_check", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "cssparser" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9cdaae01d5ed7882b04d795e7f752f46ff52d2fa3b50a20d28c464510bba98" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a" +dependencies = [ + "quote", + "syn 3.0.5", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b04dc5a38e4f151a79d9f2451ae6037fb6eaf5cba34771f44781f80e508498e3" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs-err" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +dependencies = [ + "autocfg", +] + +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core 0.10.1", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "goblin" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b363a30c165f666402fe6a3024d3bec7ebc898f96a4a23bd1c99f8dbf3f4f47" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hickory-proto" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand", + "ring", + "thiserror", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "html5ever" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8" +dependencies = [ + "log", + "markup5ever", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.52.0", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "markup5ever" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "relative-path" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +dependencies = [ + "serde", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rquickjs" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d96fb23e8ff51c8d4772ea84e44b5238543b7d81e9cc23c0219511a8f16482" +dependencies = [ + "rquickjs-core", + "rquickjs-macro", +] + +[[package]] +name = "rquickjs-core" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c6dbfedfbf458dc119c21ccd032c7527d185210b11a1ae796cd04106e6db280" +dependencies = [ + "hashbrown", + "relative-path", + "rquickjs-sys", +] + +[[package]] +name = "rquickjs-macro" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4446670c6ae57191ac325a4c9675fee5b31172533ad68753f7afa83759f082fd" +dependencies = [ + "convert_case", + "fnv", + "ident_case", + "indexmap", + "proc-macro-crate", + "proc-macro2", + "quote", + "rquickjs-core", + "syn 2.0.119", +] + +[[package]] +name = "rquickjs-sys" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d0aaff245bed1c6f3c39e477fb6b98d710d9d298bfec7c8dfc589bed0e5cef" +dependencies = [ + "bindgen", + "cc", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", +] + +[[package]] +name = "rustls" +version = "0.23.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scraper" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd0be4d296f048bfb06dd01bbc80ef789ddd2e55583e8d2e6b804942abfabc2" +dependencies = [ + "cssparser", + "ego-tree", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "scroll" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1783eabc414609e28a5ba76aee5ddd52199f7107a0b24c2e9746a1ecc34a683d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8adfa1c298912827b8a28b223b3b874357397ae706e6190acd9bf28cee99114d" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" + +[[package]] +name = "smawk" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spotiflac-bindgen" +version = "0.1.0" +dependencies = [ + "uniffi", +] + +[[package]] +name = "spotiflac-core" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "chrono", + "image", + "regex", + "rustix", + "serde", + "serde_json", + "tz-rs", + "unicode-general-category", + "unicode-normalization", +] + +[[package]] +name = "spotiflac-extensions" +version = "0.1.0" +dependencies = [ + "aes", + "aes-gcm", + "base64 0.22.1", + "blowfish", + "cap-std", + "chrono", + "hmac", + "httpdate", + "md-5", + "rcgen", + "regex", + "rquickjs", + "rustix", + "rustls", + "serde", + "serde_json", + "sha1", + "sha2", + "spotiflac-core", + "spotiflac-network", + "spotiflac-providers", + "tempfile", + "thiserror", + "time", + "tokio", + "zeroize", + "zip", +] + +[[package]] +name = "spotiflac-mobile" +version = "0.1.0" +dependencies = [ + "rustix", + "serde_json", + "spotiflac-core", + "spotiflac-extensions", + "thiserror", + "uniffi", + "zeroize", +] + +[[package]] +name = "spotiflac-network" +version = "0.1.0" +dependencies = [ + "async-compression", + "bytes", + "cookie", + "futures-util", + "hickory-proto", + "http", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-platform-verifier", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tower-service", + "url", +] + +[[package]] +name = "spotiflac-providers" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "html5ever", + "httpdate", + "regex", + "rustix", + "scraper", + "serde", + "serde_json", + "spotiflac-core", + "spotiflac-network", + "tempfile", + "thiserror", + "url", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "tendril" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08" +dependencies = [ + "new_debug_unreachable", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "smawk", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "tz-rs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc6c929ffa10fb34f4a3c7e9a73620a83ef2e85e47f9ec3381b8289e6762f42" + +[[package]] +name = "unicode-general-category" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2281c8c1d221438e373249e065ca4989c4c36952c211ff21a0ee91c44a3869e7" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "uniffi" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eefd5468602930da46b1f49d3448c6dfc2e81295f93120f23f8174fd70267f" +dependencies = [ + "anyhow", + "camino", + "cargo_metadata", + "clap", + "uniffi_bindgen", + "uniffi_core", + "uniffi_macros", + "uniffi_pipeline", +] + +[[package]] +name = "uniffi_bindgen" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a0c9b375d32e1365cdb2bdd7cb495eecf6fac851ddbad077412b4ee1888514" +dependencies = [ + "anyhow", + "askama", + "camino", + "cargo_metadata", + "fs-err", + "glob", + "goblin", + "heck", + "indexmap", + "once_cell", + "serde", + "tempfile", + "textwrap", + "toml", + "uniffi_internal_macros", + "uniffi_meta", + "uniffi_pipeline", + "uniffi_udl", +] + +[[package]] +name = "uniffi_core" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec017b112701681f6fbbe5d92014b5c468eb0b177a94389de03ceec40665095" +dependencies = [ + "anyhow", + "bytes", + "once_cell", + "static_assertions", +] + +[[package]] +name = "uniffi_internal_macros" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4641669b48fefbc5e80ff08c5004d9c7617fb91232131a6734ab6712779cb04c" +dependencies = [ + "anyhow", + "indexmap", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "uniffi_macros" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb8617ee814de22caf7417bf514715ba0b3f46bd9d5a5d794413fd8282cb737" +dependencies = [ + "camino", + "fs-err", + "once_cell", + "proc-macro2", + "quote", + "serde", + "syn 2.0.119", + "toml", + "uniffi_meta", +] + +[[package]] +name = "uniffi_meta" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58d5b94fc92803d21b2928bd15c6f06e57609b95caf98ea561c99cda1b6d2a25" +dependencies = [ + "anyhow", + "siphasher", + "uniffi_internal_macros", + "uniffi_pipeline", +] + +[[package]] +name = "uniffi_pipeline" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "032739b3ec725576914c15899dedaf080163ced86b6934566c20ec2b20ce90ca" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "tempfile", + "uniffi_internal_macros", +] + +[[package]] +name = "uniffi_udl" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a1d0a0252ce1af9e8ce78ba67ac0d8937fb2bedaf10cbddd43d3614d06ec6" +dependencies = [ + "anyhow", + "textwrap", + "uniffi_meta", + "weedle2", +] + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.5", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web_atoms" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba8b815c1b593dc0baf78dd0f4fc8fdb2de53198fb1163738093e9a311c33fb3" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weedle2" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998d2c24ec099a87daf9467808859f9d82b61f1d9c9701251aea037f514eae0e" +dependencies = [ + "nom", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags", + "windows-sys 0.52.0", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec", + "time", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/rust_backend/Cargo.toml b/rust_backend/Cargo.toml new file mode 100644 index 00000000..56cbf632 --- /dev/null +++ b/rust_backend/Cargo.toml @@ -0,0 +1,70 @@ +[workspace] +members = ["crates/core", "crates/network", "crates/providers", "crates/extensions", "crates/mobile", "crates/bindgen"] +resolver = "3" + +[workspace.package] +version = "0.1.0" +edition = "2024" +rust-version = "1.98.1" +license = "MIT" +publish = false + +[workspace.dependencies] +image = { version = "=0.25.10", default-features = false, features = ["jpeg", "png", "gif", "webp"] } +spotiflac-core = { path = "crates/core" } +spotiflac-extensions = { path = "crates/extensions" } +spotiflac-network = { path = "crates/network" } +spotiflac-providers = { path = "crates/providers" } +rquickjs = { version = "=0.13.0", features = ["bindgen"] } +base64 = "0.22" +sha2 = "0.10" +sha1 = "0.10" +md-5 = "0.10" +hmac = "0.12" +aes-gcm = "=0.10.3" +aes = { version = "=0.8.4", features = ["zeroize"] } +blowfish = { version = "=0.9.1", features = ["zeroize"] } +zeroize = "1.8" +tempfile = "3.27" +zip = { version = "=8.6.0", default-features = false, features = ["deflate-flate2-zlib-rs"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["float_roundtrip", "raw_value"] } +thiserror = "2.0" +uniffi = { version = "=0.31.2", default-features = false } +tokio = { version = "=1.53.1", features = ["rt-multi-thread", "net", "time", "sync", "io-util", "macros"] } +hyper = { version = "=1.11.1", features = ["client", "http1", "http2"] } +hyper-util = { version = "=0.1.20", features = ["client-legacy", "http1", "http2", "tokio"] } +hyper-rustls = { version = "=0.27.9", default-features = false, features = ["http1", "http2", "ring", "tls12"] } +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +rustls-native-certs = "=0.8.4" +rustls-platform-verifier = "=0.7.0" +rustls-pemfile = "2.2" +http = "1.4" +http-body-util = "0.1" +bytes = "1.11" +futures-util = "0.3" +tokio-util = { version = "0.7", features = ["io"] } +async-compression = { version = "0.4", features = ["tokio", "gzip"] } +tower-service = "0.3" +hickory-proto = { version = "=0.26.2", default-features = false, features = ["std"] } +cookie = "0.18" +url = "2.5" +regex = "=1.13.1" +scraper = { version = "=0.27.0", default-features = false } +html5ever = "=0.39.0" +# Match the Unicode 15 tables used by the Go backend's title/artist matching. +unicode-normalization = "=0.1.22" +unicode-general-category = "=0.6.0" +time = { version = "=0.3.55", features = ["parsing"] } +chrono = { version = "=0.4.45", default-features = false, features = ["clock"] } +tz-rs = "=0.7.3" +httpdate = "=1.0.3" +cap-std = "=4.0.3" +rustix = { version = "=1.1.4", features = ["fs"] } + +[workspace.lints.rust] +unsafe_code = "deny" + +[profile.release] +lto = "thin" +codegen-units = 1 diff --git a/rust_backend/NOTICE b/rust_backend/NOTICE new file mode 100644 index 00000000..d3c7eddc --- /dev/null +++ b/rust_backend/NOTICE @@ -0,0 +1,40 @@ +# Third-party notices + +The English title-casing control flow in +`crates/core/src/metadata/musicbrainz/casing.rs` follows +`golang.org/x/text/cases`. Its frozen Unicode 15 data is generated using the +public Go APIs. No Go runtime is linked into +the Rust backend. Retain this notice in source and binary distributions. +The artwork resampling in `crates/core/src/cover.rs` also follows the +ApproxBiLinear pixel-center and premultiplied-alpha calculations from +`golang.org/x/image/draw`, under the same Go Authors BSD license below. + +Copyright 2009 The Go Authors. +Copyright 2014 The Go Authors. All rights reserved. +Copyright 2015 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. +* Neither the name of Google LLC nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/rust_backend/SpotiFLACBackend.podspec b/rust_backend/SpotiFLACBackend.podspec new file mode 100644 index 00000000..3c264865 --- /dev/null +++ b/rust_backend/SpotiFLACBackend.podspec @@ -0,0 +1,16 @@ +Pod::Spec.new do |spec| + spec.name = 'SpotiFLACBackend' + spec.version = '0.1.0' + spec.summary = 'Rust backend bindings for SpotiFLAC Mobile' + spec.homepage = 'https://github.com/spotiflacapp/SpotiFLAC-Mobile' + spec.license = { :type => 'MIT', :file => 'LICENSE' } + spec.author = { 'SpotiFLAC Mobile' => 'noreply@spotiflac.local' } + spec.source = { :path => '.' } + spec.ios.deployment_target = '16.0' + spec.swift_version = '5.0' + spec.static_framework = true + # build_rust_backend.sh stages this spec beside the generated iOS artifacts. + spec.source_files = 'SpotiFLACBackend.swift' + spec.vendored_frameworks = 'SpotiFLACBackendFFI.xcframework' + spec.frameworks = 'Security', 'CoreFoundation' +end diff --git a/rust_backend/crates/bindgen/Cargo.toml b/rust_backend/crates/bindgen/Cargo.toml new file mode 100644 index 00000000..51b948e4 --- /dev/null +++ b/rust_backend/crates/bindgen/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "spotiflac-bindgen" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +uniffi = { workspace = true, features = ["cli", "cargo-metadata"] } + +[lints] +workspace = true diff --git a/rust_backend/crates/bindgen/src/main.rs b/rust_backend/crates/bindgen/src/main.rs new file mode 100644 index 00000000..a01b5470 --- /dev/null +++ b/rust_backend/crates/bindgen/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + uniffi::uniffi_bindgen_main(); +} diff --git a/rust_backend/crates/core/Cargo.toml b/rust_backend/crates/core/Cargo.toml new file mode 100644 index 00000000..6d38a356 --- /dev/null +++ b/rust_backend/crates/core/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "spotiflac-core" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +base64.workspace = true +image.workspace = true +serde.workspace = true +serde_json.workspace = true +regex.workspace = true +unicode-normalization.workspace = true +unicode-general-category.workspace = true +chrono.workspace = true +rustix.workspace = true + +[target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies] +tz-rs.workspace = true + +[lints] +workspace = true diff --git a/rust_backend/crates/core/src/app_version.rs b/rust_backend/crates/core/src/app_version.rs new file mode 100644 index 00000000..d970ae4b --- /dev/null +++ b/rust_backend/crates/core/src/app_version.rs @@ -0,0 +1,106 @@ +//! One application version shared by native owners, VMs and HTTP clients. + +use std::sync::{Arc, RwLock}; + +#[derive(Debug, Default)] +struct State { + version: String, + closed: bool, +} + +#[derive(Clone, Debug, Default)] +pub struct AppVersion(Arc<RwLock<State>>); + +#[derive(Clone, Copy, Debug)] +pub struct AppVersionClosed; + +impl std::fmt::Display for AppVersionClosed { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("application version state closed") + } +} + +impl std::error::Error for AppVersionClosed {} + +impl AppVersion { + pub fn get(&self) -> String { + self.0 + .read() + .expect("application version lock") + .version + .clone() + } + + pub fn user_agent(&self) -> String { + let version = self.get(); + if version.is_empty() { + "SpotiFLAC-Mobile".into() + } else { + format!("SpotiFLAC-Mobile/{version}") + } + } + + pub fn set(&self, version: &str) -> Result<(), AppVersionClosed> { + let mut state = self.0.write().expect("application version lock"); + if state.closed { + return Err(AppVersionClosed); + } + state.version = version.trim().into(); + Ok(()) + } + + /// Reads keep the last snapshot for calls already unwinding. Owners reject + /// new calls separately; no retained handle can change a closed version. + pub fn close(&self) { + self.0.write().expect("application version lock").closed = true; + } +} + +impl From<&str> for AppVersion { + fn from(version: &str) -> Self { + Self(Arc::new(RwLock::new(State { + version: version.trim().into(), + closed: false, + }))) + } +} + +impl From<String> for AppVersion { + fn from(version: String) -> Self { + Self::from(version.as_str()) + } +} + +#[cfg(test)] +mod tests { + use super::AppVersion; + + #[test] + fn shared_updates_are_atomic_trimmed_and_cannot_outlive_the_owner() { + let version = AppVersion::from("\u{0085} 1.0 \u{2003}"); + assert_eq!(version.get(), "1.0"); + let retained = version.clone(); + std::thread::scope(|scope| { + for value in ["", "first version", "second version"] { + let version = &version; + scope.spawn(move || { + for _ in 0..1000 { + version.set(value).unwrap(); + assert!(matches!( + version.user_agent().as_str(), + "SpotiFLAC-Mobile" + | "SpotiFLAC-Mobile/first version" + | "SpotiFLAC-Mobile/second version" + )); + } + }); + } + }); + version.set("").unwrap(); + assert_eq!(retained.user_agent(), "SpotiFLAC-Mobile"); + version.close(); + assert!(retained.set("new").is_err()); + assert_eq!(retained.get(), ""); + assert_eq!(AppVersion::from("separate owner").get(), "separate owner"); + } +} diff --git a/rust_backend/crates/core/src/cancellation.rs b/rust_backend/crates/core/src/cancellation.rs new file mode 100644 index 00000000..8138116a --- /dev/null +++ b/rust_backend/crates/core/src/cancellation.rs @@ -0,0 +1,395 @@ +//! Request cancellation with Go-compatible sentinels and reference counting. +//! +//! Each registry owns one domain. Leases release their reference on drop; an +//! explicit release also wakes a pending wait. Shutdown closes the instance and +//! wakes all leases, including untracked requests with an empty ID. + +use std::collections::BTreeMap; +use std::fmt; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex, Weak}; +use std::task::Waker; +use std::thread::Thread; +use std::time::{Duration, Instant}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CancellationDomain { + Download, + ExtensionRequest, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CancellationError { + DownloadCancelled, + ExtensionRequestCancelled, + RegistryClosed, + LeaseReleased, +} + +impl fmt::Display for CancellationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::DownloadCancelled => "download cancelled", + Self::ExtensionRequestCancelled => "extension request cancelled", + Self::RegistryClosed => "cancellation registry closed", + Self::LeaseReleased => "request lease released", + }) + } +} + +impl std::error::Error for CancellationError {} + +#[derive(Default)] +struct Entry { + cancelled: Arc<AtomicBool>, + references: usize, +} + +#[derive(Default)] +struct State { + closed: bool, + entries: BTreeMap<String, Entry>, +} + +impl State { + fn check_open(&self) -> Result<(), CancellationError> { + if self.closed { + Err(CancellationError::RegistryClosed) + } else { + Ok(()) + } + } +} + +#[derive(Default)] +struct Shared { + state: Mutex<State>, + changed: Condvar, + waker: Option<Waker>, + threads: Mutex<Vec<Weak<Thread>>>, +} + +impl Shared { + fn notify(&self) { + self.changed.notify_all(); + self.threads + .lock() + .expect("cancellation thread lock") + .retain(|thread| { + if let Some(thread) = thread.upgrade() { + thread.unpark(); + true + } else { + false + } + }); + if let Some(waker) = &self.waker { + waker.wake_by_ref(); + } + } +} + +pub struct CancellationRegistry { + shared: Arc<Shared>, + domain: CancellationDomain, +} + +impl CancellationRegistry { + pub fn new(domain: CancellationDomain) -> Self { + Self { + shared: Arc::new(Shared::default()), + domain, + } + } + + /// Wake an external executor after cancellation, release or shutdown. + /// The executor must recheck its own request; other IDs may still be active. + pub fn with_waker(domain: CancellationDomain, waker: Waker) -> Self { + Self { + shared: Arc::new(Shared { + waker: Some(waker), + ..Shared::default() + }), + domain, + } + } + + pub fn acquire(&self, id: &str) -> Result<RequestLease, CancellationError> { + let mut state = self.shared.state.lock().expect("cancellation state lock"); + state.check_open()?; + let cancelled = if id.is_empty() { + Arc::new(AtomicBool::new(false)) + } else { + let entry = state.entries.entry(id.to_owned()).or_default(); + entry.references += 1; + Arc::clone(&entry.cancelled) + }; + Ok(RequestLease { + shared: Arc::clone(&self.shared), + id: id.to_owned(), + domain: self.domain, + cancelled, + released: AtomicBool::new(false), + }) + } + + pub fn cancel(&self, id: &str) -> Result<(), CancellationError> { + let mut state = self.shared.state.lock().expect("cancellation state lock"); + state.check_open()?; + if !id.is_empty() { + state + .entries + .entry(id.to_owned()) + .or_default() + .cancelled + .store(true, Ordering::Release); + drop(state); + self.shared.notify(); + } + Ok(()) + } + + /// Cancel active work only. Idle sentinels remain available to reset/retry. + /// IDs are sorted for deterministic native results; Go's ordering is unspecified. + pub fn cancel_active(&self) -> Result<Vec<String>, CancellationError> { + let state = self.shared.state.lock().expect("cancellation state lock"); + state.check_open()?; + let mut ids = Vec::new(); + for (id, entry) in &state.entries { + if entry.references > 0 { + entry.cancelled.store(true, Ordering::Release); + ids.push(id.clone()); + } + } + drop(state); + self.shared.notify(); + Ok(ids) + } + + pub fn is_cancelled(&self, id: &str) -> Result<bool, CancellationError> { + let state = self.shared.state.lock().expect("cancellation state lock"); + state.check_open()?; + Ok(state + .entries + .get(id) + .is_some_and(|entry| entry.cancelled.load(Ordering::Acquire))) + } + + pub fn reset_if_idle(&self, id: &str) -> Result<(), CancellationError> { + let mut state = self.shared.state.lock().expect("cancellation state lock"); + state.check_open()?; + if state + .entries + .get(id) + .is_some_and(|entry| entry.references == 0) + { + state.entries.remove(id); + } + Ok(()) + } + + /// Terminal and idempotent. A new registry is required for subsequent work. + pub fn shutdown(&self) { + let mut state = self.shared.state.lock().expect("cancellation state lock"); + state.closed = true; + state.entries.clear(); + drop(state); + self.shared.notify(); + } +} + +impl Drop for CancellationRegistry { + fn drop(&mut self) { + self.shutdown(); + } +} + +pub struct RequestLease { + shared: Arc<Shared>, + id: String, + domain: CancellationDomain, + cancelled: Arc<AtomicBool>, + released: AtomicBool, +} + +impl RequestLease { + /// Notify a parked native worker on cancellation/release/shutdown. The + /// caller retains the thread while waiting and rechecks after subscribing. + pub fn observe_thread(&self, thread: &Arc<Thread>) { + let mut threads = self + .shared + .threads + .lock() + .expect("cancellation thread lock"); + threads.retain(|thread| thread.strong_count() > 0); + threads.push(Arc::downgrade(thread)); + } + + pub fn domain(&self) -> CancellationDomain { + self.domain + } + + /// Query the active request identity with Go's whitespace normalization. + /// The lease itself still retains its original, exact cancellation ID. + pub fn active_request_cancelled(&self) -> bool { + if self.domain != CancellationDomain::ExtensionRequest || self.id.trim().is_empty() { + return false; + } + let state = self.shared.state.lock().expect("cancellation state lock"); + state + .entries + .get(self.id.trim()) + .is_some_and(|entry| entry.cancelled.load(Ordering::Acquire)) + } + + fn check_live(&self, state: &State) -> Result<(), CancellationError> { + state.check_open()?; + if self.released.load(Ordering::Acquire) { + return Err(CancellationError::LeaseReleased); + } + Ok(()) + } + + pub fn is_cancelled(&self) -> Result<bool, CancellationError> { + let state = self.shared.state.lock().expect("cancellation state lock"); + self.check_live(&state)?; + Ok(self.cancelled.load(Ordering::Acquire)) + } + + /// Preserve the exact existing download/extension cancellation messages. + pub fn check_active(&self) -> Result<(), CancellationError> { + if self.is_cancelled()? { + return Err(match self.domain { + CancellationDomain::Download => CancellationError::DownloadCancelled, + CancellationDomain::ExtensionRequest => { + CancellationError::ExtensionRequestCancelled + } + }); + } + Ok(()) + } + + /// Block without polling. True means cancelled, false means heartbeat expiry. + /// Bounds match the existing progress waiter: default 15 s, maximum 60 s. + /// Call from a worker thread; cancellation/release/shutdown may run elsewhere. + pub fn wait_cancelled(&self, timeout_ms: i64) -> Result<bool, CancellationError> { + let timeout = wait_timeout(timeout_ms); + let started = Instant::now(); + let mut state = self.shared.state.lock().expect("cancellation state lock"); + loop { + self.check_live(&state)?; + if self.cancelled.load(Ordering::Acquire) { + return Ok(true); + } + let remaining = timeout.saturating_sub(started.elapsed()); + if remaining.is_zero() { + return Ok(false); + } + // Every predicate change takes this same lock, avoiding lost wakeups + // between checking the predicate and entering Condvar::wait_timeout. + (state, _) = self + .shared + .changed + .wait_timeout(state, remaining) + .expect("cancellation wait lock"); + } + } + + /// Release this acquisition once; other acquisitions for the same ID remain. + /// Holding or dropping this old lease cannot remove a later retry's entry. + pub fn release(&self) { + let mut state = self.shared.state.lock().expect("cancellation state lock"); + if self.released.swap(true, Ordering::AcqRel) { + return; + } + if let Some(entry) = state.entries.get_mut(&self.id) { + entry.references -= 1; + if entry.references == 0 { + state.entries.remove(&self.id); + } + } + drop(state); + self.shared.notify(); + } +} + +impl Drop for RequestLease { + fn drop(&mut self) { + self.release(); + } +} + +fn wait_timeout(timeout_ms: i64) -> Duration { + let bounded_ms = if timeout_ms <= 0 { + 15_000 + } else { + timeout_ms.min(60_000) + }; + Duration::from_millis(bounded_ms as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Weak; + use std::sync::atomic::AtomicUsize; + use std::task::Wake; + + #[test] + fn external_waker_observes_changes_without_holding_the_registry_lock() { + #[derive(Default)] + struct Observer { + shared: Mutex<Weak<Shared>>, + calls: AtomicUsize, + } + impl Wake for Observer { + fn wake(self: Arc<Self>) { + if let Some(shared) = self.shared.lock().unwrap().upgrade() { + // Executors may synchronously inspect cancellation on wake. + let _state = shared.state.try_lock().expect("wake held registry lock"); + self.calls.fetch_add(1, Ordering::Relaxed); + } + } + } + let observer = Arc::new(Observer::default()); + let registry = CancellationRegistry::with_waker( + CancellationDomain::Download, + Waker::from(Arc::clone(&observer)), + ); + *observer.shared.lock().unwrap() = Arc::downgrade(®istry.shared); + let first = registry.acquire("first").unwrap(); + let second = registry.acquire("second").unwrap(); + let anonymous = registry.acquire("").unwrap(); + registry.cancel("first").unwrap(); + assert!(first.is_cancelled().unwrap()); + assert!(!second.is_cancelled().unwrap()); + assert_eq!(observer.calls.load(Ordering::Relaxed), 1); + registry.cancel("").unwrap(); + assert_eq!(observer.calls.load(Ordering::Relaxed), 1); + assert_eq!(registry.cancel_active().unwrap(), ["first", "second"]); + assert!(second.is_cancelled().unwrap()); + assert!(!anonymous.is_cancelled().unwrap()); + assert_eq!(observer.calls.load(Ordering::Relaxed), 2); + second.release(); + second.release(); + assert_eq!(second.check_active(), Err(CancellationError::LeaseReleased)); + assert_eq!(observer.calls.load(Ordering::Relaxed), 3); + registry.shutdown(); + assert_eq!( + anonymous.check_active(), + Err(CancellationError::RegistryClosed) + ); + assert_eq!(observer.calls.load(Ordering::Relaxed), 4); + } + + #[test] + fn timeout_bounds_do_not_overflow() { + for input in [i64::MIN, -1, 0] { + assert_eq!(wait_timeout(input), Duration::from_secs(15)); + } + assert_eq!(wait_timeout(1), Duration::from_millis(1)); + for input in [60_000, 60_001, i64::MAX] { + assert_eq!(wait_timeout(input), Duration::from_secs(60)); + } + } +} diff --git a/rust_backend/crates/core/src/clock.rs b/rust_backend/crates/core/src/clock.rs new file mode 100644 index 00000000..04dfc52a --- /dev/null +++ b/rust_backend/crates/core/src/clock.rs @@ -0,0 +1,117 @@ +use chrono::{DateTime, Datelike, Timelike, Utc}; +use serde::Serialize; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LocalTime { + year: i32, + month: u32, + day: u32, + hour: u32, + minute: u32, + second: u32, + weekday: u32, + offset_minutes: i32, + timezone: String, + timestamp: i64, +} + +pub fn local_time() -> LocalTime { + local_time_at(Utc::now().timestamp()).expect("current timestamp fits calendar range") +} + +pub fn local_time_at(timestamp: i64) -> Result<LocalTime, String> { + let (offset, _, timezone) = zone_at(timestamp); + let local = DateTime::from_timestamp( + timestamp + .checked_add(i64::from(offset)) + .ok_or("timestamp overflow")?, + 0, + ) + .ok_or("timestamp outside calendar range")?; + Ok(LocalTime { + year: local.year(), + month: local.month(), + day: local.day(), + hour: local.hour(), + minute: local.minute(), + second: local.second(), + weekday: local.weekday().num_days_from_sunday(), + offset_minutes: -(offset / 60), + timezone, + timestamp, + }) +} + +pub(crate) fn zone_at(timestamp: i64) -> (i32, String, String) { + // Go 1.26's Android and iOS initLocal use UTC. Host Unix builds load TZ or + // /etc/localtime once; retain that distinction in the compatibility SDK. + #[cfg(any(target_os = "android", target_os = "ios"))] + { + let _ = timestamp; + (0, "UTC".into(), "UTC".into()) + } + #[cfg(not(any(target_os = "android", target_os = "ios")))] + { + static LOCAL: std::sync::LazyLock<(tz::TimeZone, String)> = + std::sync::LazyLock::new(load_zone); + let (zone, name) = &*LOCAL; + match zone.find_local_time_type(timestamp) { + Ok(kind) => ( + kind.ut_offset(), + kind.time_zone_designation().into(), + name.clone(), + ), + Err(_) => (0, "UTC".into(), "UTC".into()), + } + } +} + +#[cfg(not(any(target_os = "android", target_os = "ios")))] +fn load_zone() -> (tz::TimeZone, String) { + use std::io::Read; + let load = |path: &std::path::Path| { + let file = std::fs::File::open(path).ok()?; + let mut data = Vec::new(); + file.take(10 * 1024 * 1024 + 1) + .read_to_end(&mut data) + .ok()?; + if data.len() > 10 * 1024 * 1024 { + return None; + } + tz::TimeZone::from_tz_data(&data).ok() + }; + let configured = std::env::var("TZ").ok(); + if configured.is_none() { + if let Some(zone) = load(std::path::Path::new("/etc/localtime")) { + return (zone, "Local".into()); + } + } else if let Some(raw) = configured { + let name = raw.strip_prefix(':').unwrap_or(&raw); + if name.starts_with('/') { + if let Some(zone) = load(std::path::Path::new(name)) { + return ( + zone, + if name == "/etc/localtime" { + "Local" + } else { + name + } + .into(), + ); + } + } else if !name.is_empty() && name != "UTC" && !name.contains("..") { + for directory in [ + "/usr/share/zoneinfo", + "/usr/share/lib/zoneinfo", + "/usr/lib/locale/TZ", + "/etc/zoneinfo", + ] { + if let Some(zone) = load(&std::path::Path::new(directory).join(name)) { + return (zone, name.into()); + } + } + } + } + (tz::TimeZone::utc(), "UTC".into()) +} diff --git a/rust_backend/crates/core/src/cover.rs b/rust_backend/crates/core/src/cover.rs new file mode 100644 index 00000000..46a8d63b --- /dev/null +++ b/rust_backend/crates/core/src/cover.rs @@ -0,0 +1,242 @@ +//! Provider artwork sizing. Resolution selection remains extension-owned. + +use image::{DynamicImage, ImageFormat, ImageReader}; +use std::borrow::Cow; +use std::io::Cursor; + +pub const MAX_DOWNLOAD_BYTES: usize = 24 << 20; +pub const LIBRARY_MAX_DIMENSION: i64 = 800; +const MAX_DECODE_PIXELS: u64 = 16_000_000; + +fn reader(data: &[u8]) -> Result<ImageReader<Cursor<&[u8]>>, String> { + let reader = ImageReader::new(Cursor::new(data)) + .with_guessed_format() + .map_err(|error| error.to_string())?; + match reader.format() { + Some(ImageFormat::Jpeg | ImageFormat::Png | ImageFormat::Gif | ImageFormat::WebP) => { + Ok(reader) + } + _ => Err("unknown image format".into()), + } +} + +pub fn dimensions(data: &[u8]) -> (u32, u32) { + reader(data) + .and_then(|reader| reader.into_dimensions().map_err(|error| error.to_string())) + .unwrap_or_default() +} + +/// Keep original bytes unless shrinking is required. Returned PNG pixels retain +/// alpha; other decoded formats are encoded as JPEG at the Go quality setting. +pub fn resize<'a>( + data: &'a [u8], + max_dimension: i64, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Cow<'a, [u8]>, String> { + check()?; + if data.is_empty() || max_dimension <= 0 { + return Ok(Cow::Borrowed(data)); + } + let header = reader(data).map_err(|error| format!("decode artwork dimensions: {error}"))?; + let png = header.format() == Some(ImageFormat::Png); + let (width, height) = header + .into_dimensions() + .map_err(|error| format!("decode artwork dimensions: {error}"))?; + if width == 0 || height == 0 { + return Err(format!("invalid artwork dimensions {width}x{height}")); + } + if i64::from(width.max(height)) <= max_dimension { + return Ok(Cow::Borrowed(data)); + } + if u64::from(width) * u64::from(height) > MAX_DECODE_PIXELS { + return Err(format!( + "artwork dimensions {width}x{height} exceed safe decode limit" + )); + } + check()?; + let source = reader(data)? + .decode() + .map_err(|error| format!("decode artwork: {error}"))?; + check()?; + let limit = max_dimension as u32; // Positive and less than an input u32 dimension. + let (dw, dh) = if width >= height { + ( + limit, + ((u64::from(height) * u64::from(limit) + u64::from(width) / 2) / u64::from(width)) + .max(1) as u32, + ) + } else { + ( + ((u64::from(width) * u64::from(limit) + u64::from(height) / 2) / u64::from(height)) + .max(1) as u32, + limit, + ) + }; + let source = Pixels::new(source); + let mut pixels = vec![0; dw as usize * dh as usize * 4]; + // Match x/image/draw ApproxBiLinear: sample four neighbors at pixel centers + // in premultiplied 16-bit space, then truncate into a premultiplied RGBA8 + // destination. See NOTICE. + for y in 0..dh { + let (y0, y1, fy) = neighbors(y, height, dh); + for x in 0..dw { + if x % 1024 == 0 { + check()?; + } + let (x0, x1, fx) = neighbors(x, width, dw); + let samples = [ + source.at(x0, y0), + source.at(x1, y0), + source.at(x0, y1), + source.at(x1, y1), + ]; + let offset = (y as usize * dw as usize + x as usize) * 4; + for channel in 0..4 { + let top = (1.0 - fx) * f64::from(samples[0][channel]) + + fx * f64::from(samples[1][channel]); + let bottom = (1.0 - fx) * f64::from(samples[2][channel]) + + fx * f64::from(samples[3][channel]); + pixels[offset + channel] = (((1.0 - fy) * top + fy * bottom) as u32 >> 8) as u8; + } + } + } + check()?; + let mut encoded = Vec::new(); + if png { + for pixel in pixels.as_chunks_mut::<4>().0 { + let alpha = u32::from(pixel[3]); + for channel in &mut pixel[..3] { + *channel = ((u32::from(*channel) * 0xffff) + .checked_div(alpha) + .unwrap_or(0) + >> 8) + .min(255) as u8; + } + } + let image = image::RgbaImage::from_raw(dw, dh, pixels).expect("cover pixel dimensions"); + image + .write_to(&mut Cursor::new(&mut encoded), ImageFormat::Png) + .map_err(|error| format!("encode resized PNG artwork: {error}"))?; + } else { + let rgb: Vec<_> = pixels + .as_chunks::<4>() + .0 + .iter() + .flat_map(|pixel| pixel[..3].iter().copied()) + .collect(); + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut encoded, 88) + .encode(&rgb, dw, dh, image::ExtendedColorType::Rgb8) + .map_err(|error| format!("encode resized JPEG artwork: {error}"))?; + } + check()?; + Ok(Cow::Owned(encoded)) +} + +fn neighbors(destination: u32, source_size: u32, destination_size: u32) -> (u32, u32, f64) { + let source = (f64::from(destination) + 0.5) + * (f64::from(source_size) / f64::from(destination_size)) + - 0.5; + if source < 0.0 { + return (0, 0, 0.0); + } + let first = source as u32; + if first >= source_size - 1 { + return (source_size - 1, source_size - 1, 0.0); + } + (first, first + 1, source - f64::from(first)) +} + +enum Pixels { + Rgb(image::RgbImage), + Eight(image::RgbaImage), + Sixteen(image::ImageBuffer<image::Rgba<u16>, Vec<u16>>), +} + +impl Pixels { + fn new(image: DynamicImage) -> Self { + if let DynamicImage::ImageRgb8(image) = image { + return Self::Rgb(image); + } + match image.color() { + image::ColorType::L16 + | image::ColorType::La16 + | image::ColorType::Rgb16 + | image::ColorType::Rgba16 => Self::Sixteen(image.into_rgba16()), + _ => Self::Eight(image.into_rgba8()), + } + } + + fn at(&self, x: u32, y: u32) -> [u32; 4] { + let mut pixel = match self { + Self::Rgb(image) => { + let [r, g, b] = image + .get_pixel(x, y) + .0 + .map(|value| u32::from(value) * 0x101); + return [r, g, b, 0xffff]; + } + Self::Eight(image) => image + .get_pixel(x, y) + .0 + .map(|value| u32::from(value) * 0x101), + Self::Sixteen(image) => image.get_pixel(x, y).0.map(u32::from), + }; + for channel in 0..3 { + pixel[channel] = pixel[channel] * pixel[3] / 0xffff; + } + pixel + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rgb_and_opaque_rgba_produce_identical_resized_pngs() { + for (width, height) in [(13, 7), (7, 13), (1, 13), (13, 1)] { + let rgb = image::RgbImage::from_fn(width, height, |x, y| { + image::Rgb([(x * 37 + y * 11) as u8, (x * 3 + y * 71) as u8, 255]) + }); + let rgb = DynamicImage::ImageRgb8(rgb); + let rgba = DynamicImage::ImageRgba8(rgb.to_rgba8()); + let encode = |image: &DynamicImage| { + let mut output = Cursor::new(Vec::new()); + image.write_to(&mut output, ImageFormat::Png).unwrap(); + output.into_inner() + }; + let (rgb, rgba) = (encode(&rgb), encode(&rgba)); + for dimension in [1, 2, 3, 6] { + assert_eq!( + resize(&rgb, dimension, &|| Ok(())).unwrap(), + resize(&rgba, dimension, &|| Ok(())).unwrap(), + "{width}x{height} to {dimension}" + ); + } + } + } + + #[test] + fn webp_is_preserved_when_small_and_resized_to_jpeg_with_alpha_composited() { + let image = DynamicImage::ImageRgba8(image::RgbaImage::from_pixel( + 6, + 3, + image::Rgba([20, 40, 60, 128]), + )); + let mut encoded = Cursor::new(Vec::new()); + image.write_to(&mut encoded, ImageFormat::WebP).unwrap(); + let data = encoded.into_inner(); + let unchanged = resize(&data, 6, &|| Ok(())).unwrap(); + assert!(matches!(unchanged, Cow::Borrowed(_))); + assert_eq!(unchanged.as_ref(), data); + let resized = resize(&data, 3, &|| Ok(())).unwrap(); + assert_eq!(image::guess_format(&resized).unwrap(), ImageFormat::Jpeg); + let image = image::load_from_memory(&resized).unwrap().into_rgb8(); + assert_eq!(image.dimensions(), (3, 2)); + for pixel in image.pixels() { + for (actual, expected) in pixel.0.into_iter().zip([10, 20, 30]) { + assert!(actual.abs_diff(expected) <= 3); + } + } + } +} diff --git a/rust_backend/crates/core/src/cue.rs b/rust_backend/crates/core/src/cue.rs new file mode 100644 index 00000000..9cbc4510 --- /dev/null +++ b/rust_backend/crates/core/src/cue.rs @@ -0,0 +1,380 @@ +use serde::Serialize; +use std::io::Read; + +const MAX_SCANNER_BUFFER: usize = 64 * 1024; +const READ_BUFFER_SIZE: usize = 8 * 1024; + +#[derive(Clone, Debug, Serialize)] +pub struct CueSheet { + pub performer: String, + pub title: String, + pub file_name: String, + pub file_type: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub genre: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub date: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub comment: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub composer: String, + pub tracks: Vec<CueTrack>, +} + +#[derive(Clone, Debug, Serialize)] +pub struct CueTrack { + pub number: i64, + pub title: String, + pub performer: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub isrc: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub composer: String, + pub start_time: f64, + pub pre_gap: f64, +} + +/// Parse a CUE sheet from a bounded stream. +pub fn parse( + reader: &mut impl Read, + check: &dyn Fn() -> Result<(), String>, +) -> Result<CueSheet, String> { + check()?; + let mut lines = Lines { + reader, + check, + buffer: [0; READ_BUFFER_SIZE], + start: 0, + end: 0, + eof: false, + }; + let mut raw_line = Vec::new(); + let mut sheet = CueSheet { + performer: String::new(), + title: String::new(), + file_name: String::new(), + file_type: String::new(), + genre: String::new(), + date: String::new(), + comment: String::new(), + composer: String::new(), + tracks: Vec::new(), + }; + let mut current_track: Option<CueTrack> = None; + + while lines.next(&mut raw_line)? { + let mut line = crate::text::utf8(&raw_line); + line = line.trim().to_owned(); + if line.is_empty() { + continue; + } + + if let Some(without_bom) = line.strip_prefix('\u{feff}') { + line = without_bom.trim().to_owned(); + if line.is_empty() { + continue; + } + } + + if let Some((key, value)) = parse_rem(&line) { + let value = unquote(value); + match key.to_uppercase().as_str() { + "GENRE" => sheet.genre = value, + "DATE" => sheet.date = value, + "COMMENT" => sheet.comment = value, + "COMPOSER" => { + if let Some(track) = current_track.as_mut() { + track.composer = value; + } else { + sheet.composer = value; + } + } + _ => {} + } + continue; + } + + if let Some(rest) = command_rest(&line, "PERFORMER ") { + let value = unquote(rest); + if let Some(track) = current_track.as_mut() { + track.performer = value; + } else { + sheet.performer = value; + } + continue; + } + + if let Some(rest) = command_rest(&line, "TITLE ") { + let value = unquote(rest); + if let Some(track) = current_track.as_mut() { + track.title = value; + } else { + sheet.title = value; + } + continue; + } + + if let Some(rest) = command_rest(&line, "FILE ") { + let (file_name, file_type) = parse_file_line(rest); + sheet.file_name = file_name; + sheet.file_type = file_type; + continue; + } + + if command_rest(&line, "TRACK ").is_some() { + if let Some(track) = current_track.take() { + sheet.tracks.push(track); + } + let number = line + .split_whitespace() + .nth(1) + .and_then(|value| value.parse::<i64>().ok()) + .unwrap_or(0); + current_track = Some(CueTrack { + number, + title: String::new(), + performer: String::new(), + isrc: String::new(), + composer: String::new(), + start_time: 0.0, + pre_gap: -1.0, + }); + continue; + } + + if command_rest(&line, "INDEX ").is_some() && current_track.is_some() { + let parts: Vec<_> = line.split_whitespace().collect(); + if parts.len() >= 3 { + let index = parts[1].parse::<i64>().unwrap_or(0); + let seconds = timestamp(parts[2]); + if let Some(track) = current_track.as_mut() { + match index { + 0 => track.pre_gap = seconds, + 1 => track.start_time = seconds, + _ => {} + } + } + } + continue; + } + + if let Some(rest) = command_rest(&line, "ISRC ") { + if let Some(track) = current_track.as_mut() { + track.isrc = rest.trim().to_owned(); + } + continue; + } + + if let Some(rest) = command_rest(&line, "SONGWRITER ") { + let value = unquote(rest); + if let Some(track) = current_track.as_mut() { + track.composer = value; + } else { + sheet.composer = value; + } + } + } + + if let Some(track) = current_track { + sheet.tracks.push(track); + } + if sheet.tracks.is_empty() { + return Err("no tracks found in cue file".into()); + } + Ok(sheet) +} + +struct Lines<'a, R> { + reader: &'a mut R, + check: &'a dyn Fn() -> Result<(), String>, + buffer: [u8; READ_BUFFER_SIZE], + start: usize, + end: usize, + eof: bool, +} + +impl<R: Read> Lines<'_, R> { + fn next(&mut self, line: &mut Vec<u8>) -> Result<bool, String> { + line.clear(); + let mut raw_length = 0; + loop { + (self.check)()?; + if self.start == self.end { + if self.eof { + if line.is_empty() { + return Ok(false); + } + strip_carriage_return(line); + return Ok(true); + } + let count = self + .reader + .read(&mut self.buffer) + .map_err(|error| format!("error reading cue file: {error}"))?; + if count == 0 { + self.eof = true; + continue; + } + self.start = 0; + self.end = count; + } + + let remaining = &self.buffer[self.start..self.end]; + if let Some(offset) = remaining.iter().position(|byte| *byte == b'\n') { + if raw_length + offset + 1 > MAX_SCANNER_BUFFER { + return Err("error reading cue file: bufio.Scanner: token too long".into()); + } + line.extend_from_slice(&remaining[..offset]); + self.start += offset + 1; + strip_carriage_return(line); + return Ok(true); + } + + raw_length += remaining.len(); + if raw_length >= MAX_SCANNER_BUFFER { + return Err("error reading cue file: bufio.Scanner: token too long".into()); + } + line.extend_from_slice(remaining); + self.start = self.end; + } + } +} + +fn strip_carriage_return(line: &mut Vec<u8>) { + if line.last() == Some(&b'\r') { + line.pop(); + } +} + +fn command_rest<'a>(line: &'a str, command: &str) -> Option<&'a str> { + let prefix = line.get(..command.len())?; + if prefix.eq_ignore_ascii_case(command) { + line.get(command.len()..) + } else { + None + } +} + +fn parse_rem(line: &str) -> Option<(&str, &str)> { + if !line.starts_with("REM ") { + return None; + } + let bytes = line.as_bytes(); + let mut key_start = 3; + while key_start < bytes.len() && rem_space(bytes[key_start]) { + key_start += 1; + } + if key_start == bytes.len() { + return None; + } + let mut key_end = key_start; + while key_end < bytes.len() && !rem_space(bytes[key_end]) { + key_end += 1; + } + if key_end == bytes.len() { + return None; + } + let mut value_start = key_end; + while value_start < bytes.len() && rem_space(bytes[value_start]) { + value_start += 1; + } + if value_start == bytes.len() { + return None; + } + Some((&line[key_start..key_end], &line[value_start..])) +} + +fn rem_space(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\r' | b'\n' | b'\x0c') +} + +fn timestamp(value: &str) -> f64 { + let parts: Vec<_> = value.split(':').collect(); + if parts.len() != 3 { + return 0.0; + } + let minutes = parts[0].parse::<i64>().unwrap_or(0); + let seconds = parts[1].parse::<i64>().unwrap_or(0); + let frames = parts[2].parse::<i64>().unwrap_or(0); + minutes as f64 * 60.0 + seconds as f64 + frames as f64 / 75.0 +} + +fn unquote(value: &str) -> String { + let value = value.trim(); + let Some(start) = value.find('"') else { + return value.to_owned(); + }; + let Some(end) = value[start + 1..].find('"') else { + return value.to_owned(); + }; + value[start + 1..start + 1 + end].to_owned() +} + +fn parse_file_line(rest: &str) -> (String, String) { + let rest = rest.trim(); + if let Some(end) = rest.strip_prefix('"').and_then(|value| value.find('"')) { + let file_name = rest[1..end + 1].to_owned(); + let file_type = rest[end + 2..].trim().to_owned(); + return (file_name, file_type); + } + if rest.starts_with('"') { + return (rest.to_owned(), String::new()); + } + + let parts: Vec<_> = rest.split_whitespace().collect(); + match parts.len() { + 0 => (String::new(), String::new()), + 1 => (parts[0].to_owned(), String::new()), + _ => ( + parts[..parts.len() - 1].join(" "), + parts[parts.len() - 1].to_owned(), + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + use std::io::Cursor; + + #[test] + fn parse_cue_semantics_and_bounds() { + let input = "\u{feff}PERFORMER \"Album Artist\"\nREM COMPOSER \"Album Writer\"\nTITLE \"Album\"\nFILE \"album.flac\" FLAC\nTRACK 01 AUDIO\nTITLE \"Song\"\nPERFORMER \"Track Artist\"\nREM COMPOSER \"Track Writer\"\nINDEX 01 01:02:37\nTRACK 02 AUDIO\nTITLE \"Second\"\nINDEX 00 03:00:00\n"; + let sheet = parse(&mut Cursor::new(input.as_bytes()), &|| Ok(())).unwrap(); + assert_eq!(sheet.performer, "Album Artist"); + assert_eq!(sheet.composer, "Album Writer"); + assert_eq!(sheet.file_name, "album.flac"); + assert_eq!(sheet.tracks.len(), 2); + assert_eq!(sheet.tracks[0].composer, "Track Writer"); + assert!((sheet.tracks[0].start_time - (62.0 + 37.0 / 75.0)).abs() < f64::EPSILON); + assert_eq!(sheet.tracks[0].pre_gap, -1.0); + assert_eq!(sheet.tracks[1].pre_gap, 180.0); + + let mut malformed = Cursor::new(b"TITLE \"no tracks\"\n".to_vec()); + assert_eq!( + parse(&mut malformed, &|| Ok(())).unwrap_err(), + "no tracks found in cue file" + ); + + let mut oversized = Cursor::new(vec![b'x'; MAX_SCANNER_BUFFER]); + assert!( + parse(&mut oversized, &|| Ok(())) + .unwrap_err() + .contains("token too long") + ); + + let checks = Cell::new(0); + let mut cancelled = Cursor::new(b"TRACK 01 AUDIO\n".to_vec()); + let error = parse(&mut cancelled, &|| { + checks.set(checks.get() + 1); + if checks.get() > 1 { + Err("cancelled".into()) + } else { + Ok(()) + } + }) + .unwrap_err(); + assert_eq!(error, "cancelled"); + } +} diff --git a/rust_backend/crates/core/src/downloads.rs b/rust_backend/crates/core/src/downloads.rs new file mode 100644 index 00000000..c13df218 --- /dev/null +++ b/rust_backend/crates/core/src/downloads.rs @@ -0,0 +1,155 @@ +//! The native download domain owns cancellation and progress together. + +use crate::cancellation::{ + CancellationDomain, CancellationError, CancellationRegistry, RequestLease, +}; +use crate::progress::ProgressRegistry; +use std::sync::Arc; + +/// Application download input, shared by the strategy and extension facades. +/// Keep Go's null/duplicate/case-folded JSON handling in the existing decoder. +#[derive(Clone, Default, serde::Serialize)] +pub struct DownloadRequest { + pub contract_version: i64, + pub isrc: String, + pub service: String, + pub download_provider: String, + pub provider_track_id: String, + pub spotify_id: String, + pub track_name: String, + pub artist_name: String, + pub album_name: String, + pub album_artist: String, + pub cover_url: String, + pub cover_max_dimension: i64, + pub output_dir: String, + pub album_folder_template: String, + pub output_path: String, + pub output_fd: i64, + pub output_ext: String, + pub filename_format: String, + pub quality: String, + pub embed_metadata: bool, + pub artist_tag_mode: String, + pub embed_lyrics: bool, + pub embed_replaygain: bool, + pub post_processing_enabled: bool, + pub track_number: i64, + pub playlist_position: i64, + pub disc_number: i64, + pub total_tracks: i64, + pub total_discs: i64, + pub release_date: String, + pub item_id: String, + pub duration_ms: i64, + pub source: String, + pub genre: String, + pub label: String, + pub copyright: String, + pub composer: String, + pub comment: String, + pub explicit: bool, + pub album_type: String, + pub upc: String, + pub tidal_id: String, + pub qobuz_id: String, + pub deezer_id: String, + pub lyrics_mode: String, + pub use_extensions: bool, + pub use_fallback: bool, + pub requires_container_conversion: bool, + pub allow_quality_variant: bool, + pub quality_variant: String, + pub songlink_region: String, + pub network_concurrency_limit: i64, +} + +crate::lyrics::json::go_deserialize!(DownloadRequest { + "contract_version" => contract_version, "isrc" => isrc, "service" => service, + "download_provider" => download_provider, "provider_track_id" => provider_track_id, + "spotify_id" => spotify_id, "track_name" => track_name, "artist_name" => artist_name, + "album_name" => album_name, "album_artist" => album_artist, "cover_url" => cover_url, + "cover_max_dimension" => cover_max_dimension, "output_dir" => output_dir, + "album_folder_template" => album_folder_template, "output_path" => output_path, + "output_fd" => output_fd, "output_ext" => output_ext, "filename_format" => filename_format, + "quality" => quality, "embed_metadata" => embed_metadata, "artist_tag_mode" => artist_tag_mode, + "embed_lyrics" => embed_lyrics, "embed_replaygain" => embed_replaygain, + "post_processing_enabled" => post_processing_enabled, "track_number" => track_number, + "playlist_position" => playlist_position, "disc_number" => disc_number, + "total_tracks" => total_tracks, "total_discs" => total_discs, "release_date" => release_date, + "item_id" => item_id, "duration_ms" => duration_ms, "source" => source, "genre" => genre, + "label" => label, "copyright" => copyright, "composer" => composer, "comment" => comment, + "explicit" => explicit, "album_type" => album_type, "upc" => upc, "tidal_id" => tidal_id, + "qobuz_id" => qobuz_id, "deezer_id" => deezer_id, "lyrics_mode" => lyrics_mode, + "use_extensions" => use_extensions, "use_fallback" => use_fallback, + "requires_container_conversion" => requires_container_conversion, + "allow_quality_variant" => allow_quality_variant, "quality_variant" => quality_variant, + "songlink_region" => songlink_region, "network_concurrency_limit" => network_concurrency_limit, +}); + +impl DownloadRequest { + pub fn parse(raw: &str) -> Result<Self, serde_json::Error> { + serde_json::from_str(&crate::text::json_surrogates(raw)) + } + + pub fn normalize(&mut self) { + for value in [ + &mut self.track_name, + &mut self.artist_name, + &mut self.album_name, + &mut self.album_artist, + &mut self.output_dir, + &mut self.output_path, + &mut self.output_ext, + ] { + *value = value.trim().to_owned(); + } + } +} + +pub struct DownloadState { + pub progress: Arc<ProgressRegistry>, + pub cancellation: Arc<CancellationRegistry>, +} + +impl Default for DownloadState { + fn default() -> Self { + Self { + progress: Arc::new(ProgressRegistry::new()), + cancellation: Arc::new(CancellationRegistry::new(CancellationDomain::Download)), + } + } +} + +impl DownloadState { + pub fn acquire(&self, id: &str) -> Result<RequestLease, CancellationError> { + self.cancellation.acquire(id) + } + + pub fn cancel(&self, id: &str) -> Result<(), CancellationError> { + if !id.is_empty() { + self.cancellation.cancel(id)?; + let _ = self.progress.remove(id); + } + Ok(()) + } + + pub fn cancel_active(&self) -> Result<Vec<String>, CancellationError> { + let ids = self.cancellation.cancel_active()?; + for id in &ids { + let _ = self.progress.remove(id); + } + Ok(ids) + } + + pub fn shutdown(&self) { + self.cancellation.shutdown(); + self.progress.shutdown(); + } +} + +impl Drop for DownloadState { + fn drop(&mut self) { + self.shutdown(); + } +} diff --git a/rust_backend/crates/core/src/filename.rs b/rust_backend/crates/core/src/filename.rs new file mode 100644 index 00000000..6256bd71 --- /dev/null +++ b/rust_backend/crates/core/src/filename.rs @@ -0,0 +1,99 @@ +//! Filename rules shared with the existing Go backend. + +mod api; +mod date; +mod template; +pub use api::build_filename_json; +pub use template::{build_filename, build_filename_checked}; + +pub const MAX_SANITIZED_FILENAME_BYTES: usize = 200; + +/// Normalize a UTF-8 filename with the existing `SanitizeFilename` contract. +/// +/// The limit applies to UTF-8 bytes, not characters. This does not normalize +/// Unicode, remove format characters, or preserve an extension separately. +pub fn sanitize_filename(filename: &str) -> String { + let mut filtered = String::with_capacity(filename.len()); + for character in filename.chars() { + if matches!( + character, + '<' | '>' | ':' | '"' | '/' | '\\' | '|' | '?' | '*' + ) || character <= '\u{1f}' + { + filtered.push(' '); + } else if !character.is_control() { + filtered.push(character); + } + } + + let trimmed = filtered.trim().trim_matches(['.', ' ']); + let mut normalized = String::with_capacity(trimmed.len()); + for word in trimmed.split_whitespace() { + if !normalized.is_empty() { + normalized.push(' '); + } + for character in word.chars() { + if character != '_' || !normalized.ends_with('_') { + normalized.push(character); + } + } + } + + let mut sanitized = normalized.trim_matches(['_', ' ']); + if sanitized.len() > MAX_SANITIZED_FILENAME_BYTES { + let mut end = MAX_SANITIZED_FILENAME_BYTES; + while !sanitized.is_char_boundary(end) { + end -= 1; + } + sanitized = sanitized[..end] + .trim_matches(['.', ' ']) + .trim() + .trim_matches(['_', ' ']); + } + + if sanitized.is_empty() { + "Unknown".to_owned() + } else { + sanitized.to_owned() + } +} + +/// Sanitize a filename while preserving a token that was present in the raw +/// filename, such as a quality variant marker. +pub fn sanitize_filename_preserving_token(filename: &str, token: &str) -> String { + let sanitized = sanitize_filename(filename); + let token = token.trim(); + if token.is_empty() || !filename.contains(token) || sanitized.contains(token) { + return sanitized; + } + + let safe_token = sanitize_filename(token); + let suffix = format!(" - {safe_token}"); + let prefix_limit = MAX_SANITIZED_FILENAME_BYTES.saturating_sub(suffix.len()); + if prefix_limit == 0 { + return truncate_utf8_bytes(&safe_token, MAX_SANITIZED_FILENAME_BYTES).to_owned(); + } + + let raw_prefix = filename.replace(token, ""); + let raw_prefix = raw_prefix.trim_matches([' ', '_', '-']); + let prefix = sanitize_filename(raw_prefix); + let prefix = truncate_utf8_bytes(&prefix, prefix_limit) + .trim_matches(['.', ' ', '_', '-']) + .trim(); + if prefix.is_empty() || prefix == "Unknown" { + return safe_token; + } + format!("{prefix}{suffix}") +} + +fn truncate_utf8_bytes(value: &str, max_bytes: usize) -> &str { + if value.len() <= max_bytes { + return value; + } + + let mut end = max_bytes; + while !value.is_char_boundary(end) { + end -= 1; + } + &value[..end] +} diff --git a/rust_backend/crates/core/src/filename/api.rs b/rust_backend/crates/core/src/filename/api.rs new file mode 100644 index 00000000..d74b7cf4 --- /dev/null +++ b/rust_backend/crates/core/src/filename/api.rs @@ -0,0 +1,23 @@ +use serde_json::{Map, Number, Value}; + +const MAX_INPUT_BYTES: usize = 8 * 1024 * 1024; + +/// Native BuildFilename accepts a JSON object (or null). Go's JSON decoder +/// exports every number as float64, even when the token looks like an integer. +pub fn build_filename_json(template: &str, metadata_json: &str) -> Result<String, String> { + if template.len() > MAX_INPUT_BYTES || metadata_json.len() > MAX_INPUT_BYTES { + return Err("filename input exceeds 8 MiB limit".into()); + } + let mut metadata = serde_json::from_str::<Option<Map<String, Value>>>( + &crate::text::json_surrogates(metadata_json), + ) + .map_err(|error| error.to_string())? + .unwrap_or_default(); + for value in metadata.values_mut() { + if let Value::Number(number) = value { + *number = Number::from_f64(number.as_f64().ok_or("invalid metadata number")?) + .ok_or("invalid metadata number")?; + } + } + super::build_filename_checked(template, &metadata, MAX_INPUT_BYTES, &|| Ok(())) +} diff --git a/rust_backend/crates/core/src/filename/date.rs b/rust_backend/crates/core/src/filename/date.rs new file mode 100644 index 00000000..5d8b8788 --- /dev/null +++ b/rust_backend/crates/core/src/filename/date.rs @@ -0,0 +1,334 @@ +use chrono::{Datelike, NaiveDate, NaiveDateTime, Timelike}; +use regex::Regex; +use std::sync::LazyLock; + +static YEAR: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[0-9]{4}").unwrap()); +static SIMPLE: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"^([0-9]{4})(?:([-/.])([0-9]{2})(?:([-/.])([0-9]{2}))?)?$").unwrap() +}); +static RFC3339: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{1,2}):([0-9]{2}):([0-9]{2})(?:[.,]([0-9]+))?(Z|([-+])([0-9]{2}):([0-9]{2}))$").unwrap() +}); + +struct Date { + value: NaiveDateTime, + offset: i32, + zone: String, +} + +pub(super) fn format( + raw: &str, + pattern: &str, + limit: usize, + check: &dyn Fn() -> Result<(), String>, +) -> Result<String, String> { + let Some(date) = parse(raw) else { + return Ok(String::new()); + }; + let mut layout = String::new(); + let mut chars = pattern.chars(); + let mut next_check = 0; + while let Some(character) = chars.next() { + if layout.len() >= next_check { + check()?; + next_check = layout.len().saturating_add(1024); + } + if layout.len() > limit { + return Err("filename result exceeds output limit".into()); + } + if character != '%' { + layout.push(character); + continue; + } + match chars.next() { + Some('Y') => layout.push_str("2006"), + Some('y') => layout.push_str("06"), + Some('m') => layout.push_str("01"), + Some('d') => layout.push_str("02"), + Some('b') => layout.push_str("Jan"), + Some('B') => layout.push_str("January"), + Some('%') => layout.push('%'), + Some(other) => { + layout.push('%'); + layout.push(other); + } + None => layout.push('%'), + } + } + render(&date, &layout, limit, check) +} + +fn parse(raw: &str) -> Option<Date> { + let raw = raw.trim(); + if raw.is_empty() { + return None; + } + if let Some(value) = rfc3339(raw) { + return Some(value); + } + if let Some(value) = simple(raw) { + return Some(value); + } + if let Some(prefix) = raw.get(..10) + && prefix.as_bytes().get(4) == Some(&b'-') + && let Some(value) = simple(prefix) + { + return Some(value); + } + let year = YEAR.find(raw)?.as_str().parse::<i32>().ok()?; + if year <= 0 { + return None; + } + utc(year, 1, 1) +} + +fn rfc3339(raw: &str) -> Option<Date> { + let captures = RFC3339.captures(raw)?; + let number = |index| captures.get(index)?.as_str().parse::<u32>().ok(); + let fraction = captures.get(7).map_or("", |value| value.as_str()); + let mut nanos = 0; + for index in 0..9 { + nanos = nanos * 10 + + fraction + .as_bytes() + .get(index) + .map_or(0, |byte| u32::from(*byte - b'0')); + } + let value = NaiveDate::from_ymd_opt(number(1)? as i32, number(2)?, number(3)?)? + .and_hms_nano_opt(number(4)?, number(5)?, number(6)?, nanos)?; + if &captures[8] == "Z" { + return Some(Date { + value, + offset: 0, + zone: "UTC".into(), + }); + } + let (hours, minutes) = (number(10)?, number(11)?); + // Go's non-strict RFC3339 path accepts inclusive 24-hour/60-minute offsets. + if hours > 24 || minutes > 60 { + return None; + } + let offset = (hours * 3600 + minutes * 60) as i32 * if &captures[9] == "-" { -1 } else { 1 }; + let (local_offset, name, _) = + crate::clock::zone_at(value.and_utc().timestamp() - i64::from(offset)); + Some(Date { + value, + offset, + zone: if offset == local_offset { + name + } else { + String::new() + }, + }) +} + +fn simple(raw: &str) -> Option<Date> { + let captures = SIMPLE.captures(raw)?; + if captures.get(4).is_some() && captures.get(2)?.as_str() != captures.get(4)?.as_str() { + return None; + } + utc( + captures[1].parse().ok()?, + captures + .get(3) + .map_or(Some(1), |value| value.as_str().parse().ok())?, + captures + .get(5) + .map_or(Some(1), |value| value.as_str().parse().ok())?, + ) +} + +fn utc(year: i32, month: u32, day: u32) -> Option<Date> { + Some(Date { + value: NaiveDate::from_ymd_opt(year, month, day)?.and_hms_opt(0, 0, 0)?, + offset: 0, + zone: "UTC".into(), + }) +} + +fn render( + date: &Date, + mut layout: &str, + limit: usize, + check: &dyn Fn() -> Result<(), String>, +) -> Result<String, String> { + let value = date.value; + let mut output = String::new(); + while !layout.is_empty() { + check()?; + if output.len() > limit { + return Err("filename result exceeds output limit".into()); + } + let token = token(layout); + let Some(token) = token else { + let character = layout.chars().next().unwrap(); + output.push(character); + layout = &layout[character.len_utf8()..]; + continue; + }; + let text = match token { + "January" => MONTHS[value.month0() as usize].into(), + "Jan" => MONTHS[value.month0() as usize][..3].into(), + "Monday" => DAYS[value.weekday().num_days_from_sunday() as usize].into(), + "Mon" => DAYS[value.weekday().num_days_from_sunday() as usize][..3].into(), + "2006" => format!("{:04}", value.year()), + "06" => format!("{:02}", value.year().abs() % 100), + "1" => value.month().to_string(), + "01" => format!("{:02}", value.month()), + "2" => value.day().to_string(), + "02" => format!("{:02}", value.day()), + "_2" => format!("{:2}", value.day()), + "002" => format!("{:03}", value.ordinal()), + "__2" => format!("{:3}", value.ordinal()), + "15" => format!("{:02}", value.hour()), + "3" => value.hour12().1.to_string(), + "03" => format!("{:02}", value.hour12().1), + "4" => value.minute().to_string(), + "04" => format!("{:02}", value.minute()), + "5" => value.second().to_string(), + "05" => format!("{:02}", value.second()), + "PM" => if value.hour() >= 12 { "PM" } else { "AM" }.into(), + "pm" => if value.hour() >= 12 { "pm" } else { "am" }.into(), + "MST" => { + if date.zone.is_empty() { + timezone(date.offset, "-0700") + } else { + date.zone.clone() + } + } + other if other.starts_with(['.', ',']) => { + let length = (other.len() - 1).min(9); + let digits = format!("{:09}", value.nanosecond()); + let digits = &digits[..length]; + let digits = if other.as_bytes()[1] == b'9' { + digits.trim_end_matches('0') + } else { + digits + }; + if digits.is_empty() { + String::new() + } else { + format!("{}{digits}", &other[..1]) + } + } + other => timezone(date.offset, other), + }; + output.push_str(&text); + layout = &layout[token.len()..]; + } + if output.len() > limit { + return Err("filename result exceeds output limit".into()); + } + Ok(output) +} + +fn token(layout: &str) -> Option<&str> { + if layout.starts_with("_2006") { + return None; + } + for token in [ + "January", + "Monday", + "Jan", + "Mon", + "MST", + "002", + "01", + "02", + "03", + "04", + "05", + "06", + "15", + "1", + "2006", + "2", + "_2", + "__2", + "3", + "4", + "5", + "PM", + "pm", + "-070000", + "-07:00:00", + "-0700", + "-07:00", + "-07", + "Z070000", + "Z07:00:00", + "Z0700", + "Z07:00", + "Z07", + ] { + if layout.starts_with(token) + && !(matches!(token, "Jan" | "Mon") + && layout + .as_bytes() + .get(token.len()) + .is_some_and(u8::is_ascii_lowercase)) + { + return Some(&layout[..token.len()]); + } + } + let bytes = layout.as_bytes(); + if matches!(bytes[0], b'.' | b',') + && bytes + .get(1) + .is_some_and(|value| matches!(value, b'0' | b'9')) + { + let length = 1 + bytes[1..] + .iter() + .take_while(|value| **value == bytes[1]) + .count(); + if !bytes.get(length).is_some_and(u8::is_ascii_digit) { + return Some(&layout[..length]); + } + } + None +} + +fn timezone(offset: i32, token: &str) -> String { + if offset == 0 && token.starts_with('Z') { + return "Z".into(); + } + let minutes = offset.abs() / 60; + let mut result = format!("{}{:02}", if offset < 0 { '-' } else { '+' }, minutes / 60); + if token.len() > 3 { + if token.contains(':') { + result.push(':'); + } + result.push_str(&format!("{:02}", minutes % 60)); + } + if token.len() >= 7 { + if token.contains(':') { + result.push(':'); + } + result.push_str(&format!("{:02}", offset.abs() % 60)); + } + result +} + +const MONTHS: [&str; 12] = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +]; +const DAYS: [&str; 7] = [ + "Sunday", + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", +]; diff --git a/rust_backend/crates/core/src/filename/template.rs b/rust_backend/crates/core/src/filename/template.rs new file mode 100644 index 00000000..b10aee36 --- /dev/null +++ b/rust_backend/crates/core/src/filename/template.rs @@ -0,0 +1,208 @@ +use super::date; +use regex::{Captures, Regex}; +use serde_json::{Map, Value}; +use std::sync::LazyLock; + +static NUMBERS: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"\{(track|disc|playlist_position|playlistPosition|position):([0-9]+)\}").unwrap() +}); +static DATES: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\{date:([^{}]+)\}").unwrap()); +static EMPTY_GROUP: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"\[[\t\n\f\r ]*\]|\([\t\n\f\r ]*\)").unwrap()); +static DANGLING: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"[\t\n\f\r ]*[-_|][\t\n\f\r ]*([\]\)])").unwrap()); +static REPEATED: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"[\t\n\f\r ]*[-–—_|][\t\n\f\r ]*(?:[-–—_|][\t\n\f\r ]*)+").unwrap() +}); + +/// Expand the legacy filename template without sanitizing its result. Path +/// publication applies sanitization separately, as in the existing backend. +pub fn build_filename(template: &str, metadata: &Map<String, Value>) -> String { + build_filename_checked(template, metadata, usize::MAX, &|| Ok(())) + .expect("unlimited filename expansion") +} + +pub fn build_filename_checked( + template: &str, + metadata: &Map<String, Value>, + limit: usize, + check: &dyn Fn() -> Result<(), String>, +) -> Result<String, String> { + let budget = Budget { limit, check }; + check()?; + let template = if template.is_empty() { + "{artist} - {title}" + } else { + template + }; + let numbers = budget.replace(template, &NUMBERS, |capture: &Captures<'_>| { + let key = match &capture[1] { + "playlistPosition" | "position" => "playlist_position", + key => key, + }; + Ok(capture[2] + .parse::<isize>() + .map(|width| padded(integer(metadata, key), width)) + .unwrap_or_default()) + })?; + let date = ["date", "release_date", "year"] + .into_iter() + .map(|key| string(metadata, key)) + .find(|value| !value.is_empty()) + .unwrap_or_default(); + let mut result = budget.replace(&numbers, &DATES, |capture| { + date::format(&date, &capture[1], limit, check) + })?; + let mut year = string(metadata, "year"); + if year.is_empty() { + year = crate::text::utf8(&date.as_bytes()[..date.len().min(4)]); + } + let track = integer(metadata, "track"); + let disc = integer(metadata, "disc"); + let position = integer(metadata, "playlist_position"); + let mut placeholders = vec![ + ("title", string(metadata, "title")), + ("artist", string(metadata, "artist")), + ("album", string(metadata, "album")), + ("track", padded(track, 2)), + ("track_raw", padded(track, 1)), + ("disc", padded(disc, 1)), + ("disc_raw", padded(disc, 1)), + ("playlist_position", padded(position, 2)), + ("playlist position", padded(position, 2)), + ("playlistPosition", padded(position, 2)), + ("position", padded(position, 2)), + ("playlist_position_raw", padded(position, 1)), + ("year", year), + ("date", date), + ("quality", string(metadata, "quality")), + ("quality_variant", string(metadata, "quality_variant")), + ("isrc", string(metadata, "isrc")), + ("provider", string(metadata, "provider")), + ("platform", string(metadata, "provider")), + ("provider_id", string(metadata, "provider_id")), + ("id", string(metadata, "provider_id")), + ]; + // Go's map iteration permits several outcomes when a metadata value itself + // contains another placeholder. Use a stable order from that allowed set. + placeholders.sort_unstable_by_key(|(key, _)| *key); + let mut cleanup = false; + for (key, value) in placeholders { + let placeholder = format!("{{{key}}}"); + if value.is_empty() + && matches!(key, "isrc" | "provider" | "platform" | "provider_id" | "id") + && result.contains(&placeholder) + { + cleanup = true; + } + result = budget.literal(&result, &placeholder, &value)?; + } + if cleanup { + let groups = budget.replace(&result, &EMPTY_GROUP, |_| Ok(String::new()))?; + let dangling = budget.replace(&groups, &DANGLING, |capture| Ok(capture[1].into()))?; + let repeated = budget.replace(&dangling, &REPEATED, |_| Ok(" - ".into()))?; + result = repeated + .split_whitespace() + .collect::<Vec<_>>() + .join(" ") + .trim_matches([' ', '-', '–', '—', '_', '|']) + .into(); + } + check()?; + Ok(result) +} + +struct Budget<'a> { + limit: usize, + check: &'a dyn Fn() -> Result<(), String>, +} + +impl Budget<'_> { + fn push(&self, output: &mut String, value: &str) -> Result<(), String> { + (self.check)()?; + if value.len() > self.limit.saturating_sub(output.len()) { + return Err("filename result exceeds output limit".into()); + } + output.push_str(value); + Ok(()) + } + + fn replace( + &self, + input: &str, + pattern: &Regex, + replacement: impl Fn(&Captures<'_>) -> Result<String, String>, + ) -> Result<String, String> { + let mut output = String::new(); + let mut offset = 0; + for capture in pattern.captures_iter(input) { + let matched = capture.get(0).unwrap(); + self.push(&mut output, &input[offset..matched.start()])?; + self.push(&mut output, &replacement(&capture)?)?; + offset = matched.end(); + } + self.push(&mut output, &input[offset..])?; + Ok(output) + } + + fn literal(&self, input: &str, pattern: &str, value: &str) -> Result<String, String> { + let mut output = String::new(); + let mut offset = 0; + for (start, _) in input.match_indices(pattern) { + self.push(&mut output, &input[offset..start])?; + self.push(&mut output, value)?; + offset = start + pattern.len(); + } + self.push(&mut output, &input[offset..])?; + Ok(output) + } +} + +fn string(metadata: &Map<String, Value>, key: &str) -> String { + match metadata.get(key) { + Some(Value::String(value)) => value.trim().into(), + Some(Value::Number(value)) => value + .as_i64() + .map(|value| value.to_string()) + .unwrap_or_else(|| (value.as_f64().unwrap_or(0.0) as isize).to_string()), + _ => String::new(), + } +} + +fn integer(metadata: &Map<String, Value>, key: &str) -> isize { + let aliases: &[&str] = match key { + "track" => &["track_number"], + "disc" => &["disc_number"], + "playlist_position" => &["playlistPosition", "playlist position", "position"], + _ => &[], + }; + std::iter::once(key) + .chain(aliases.iter().copied()) + .find_map(|key| match metadata.get(key) { + Some(Value::String(value)) => value.trim().parse::<isize>().ok(), + Some(Value::Number(value)) => Some( + value + .as_i64() + .map(|value| value as isize) + .unwrap_or_else(|| value.as_f64().unwrap_or(0.0) as isize), + ), + _ => None, + }) + .unwrap_or(0) +} + +fn padded(number: isize, width: isize) -> String { + if number <= 0 || width <= 0 { + return String::new(); + } + // Go's fmt accepts widths through one million; Rust's formatter has a lower limit. + if width > 1_000_000 { + return format!("%!(BADWIDTH){number}"); + } + let digits = number.to_string(); + let width = width as usize; + let mut output = String::with_capacity(width.max(digits.len())); + output.extend(std::iter::repeat_n('0', width.saturating_sub(digits.len()))); + output.push_str(&digits); + output +} diff --git a/rust_backend/crates/core/src/isrc/cache.rs b/rust_backend/crates/core/src/isrc/cache.rs new file mode 100644 index 00000000..f6bd71eb --- /dev/null +++ b/rust_backend/crates/core/src/isrc/cache.rs @@ -0,0 +1,385 @@ +use crate::matching::uppercase; +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, TryLockError, Weak}; +use std::time::{Duration, Instant}; + +type Check<'a> = &'a (dyn Fn() -> Result<(), String> + Sync); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FileStamp { + pub path: String, + pub size: u64, + pub modified_ns: i128, + pub directory: bool, +} + +/// The native owner supplies filesystem access. A cache hit never grants access +/// to a path: adapters must validate directory authority before each operation. +pub trait IndexFiles: Sync { + /// Return entries in lexical depth-first walk order, preserving directory + /// boundaries rather than sorting the complete path strings afterward. + fn list(&self, directory: &str, check: Check<'_>) -> Result<Vec<FileStamp>, String>; + fn read(&self, path: &str, check: Check<'_>) -> Result<String, String>; + fn stat(&self, path: &str) -> Result<Option<FileStamp>, String>; +} + +#[derive(Clone)] +struct FileEntry { + stamp: FileStamp, + isrc: String, +} + +struct Index { + entries: BTreeMap<String, String>, + files: BTreeMap<String, FileEntry>, + built_at: Instant, +} + +pub struct IndexCache { + indexes: Mutex<BTreeMap<String, Arc<Mutex<Index>>>>, + builders: Mutex<BTreeMap<String, Weak<Mutex<()>>>>, + ttl: Duration, +} + +impl Default for IndexCache { + fn default() -> Self { + Self::with_ttl(Duration::from_secs(5 * 60)) + } +} + +impl IndexCache { + pub fn with_ttl(ttl: Duration) -> Self { + Self { + indexes: Mutex::default(), + builders: Mutex::default(), + ttl, + } + } + + fn cached(&self, directory: &str) -> Option<Arc<Mutex<Index>>> { + self.indexes + .lock() + .expect("ISRC cache lock") + .get(directory) + .cloned() + } + + fn fresh(&self, directory: &str) -> Option<Arc<Mutex<Index>>> { + self.cached(directory) + .filter(|index| index.lock().expect("ISRC index lock").built_at.elapsed() < self.ttl) + } + + fn index( + &self, + directory: &str, + files: &dyn IndexFiles, + force: bool, + check: Check<'_>, + ) -> Result<Arc<Mutex<Index>>, String> { + check()?; + if !force && let Some(index) = self.fresh(directory) { + return Ok(index); + } + let builder = { + let mut builders = self.builders.lock().expect("ISRC builders lock"); + let entry = builders.entry(directory.into()).or_default(); + entry.upgrade().unwrap_or_else(|| { + let lock = Arc::new(Mutex::new(())); + *entry = Arc::downgrade(&lock); + lock + }) + }; + let _building = wait_for_builder(&builder, check)?; + if !force && let Some(index) = self.fresh(directory) { + return Ok(index); + } + let previous = self + .cached(directory) + .map(|index| index.lock().expect("ISRC index lock").files.clone()) + .unwrap_or_default(); + let mut index = Index { + entries: BTreeMap::new(), + files: BTreeMap::new(), + built_at: Instant::now(), + }; + let mut changed = Vec::new(); + if !directory.is_empty() { + let listed = files.list(directory, check)?; + for stamp in listed { + check()?; + if stamp.directory || !super::supported(&stamp.path) { + continue; + } + if let Some(entry) = previous.get(&stamp.path) + && entry.stamp == stamp + { + if !entry.isrc.is_empty() { + index.entries.insert(entry.isrc.clone(), stamp.path.clone()); + } + index.files.insert(stamp.path.clone(), entry.clone()); + } else { + changed.push(stamp); + } + } + } + let next = AtomicUsize::new(0); + let values = Mutex::new(vec![String::new(); changed.len()]); + std::thread::scope(|scope| { + let handles = (0..changed.len().min(4)) + .map(|_| { + scope.spawn(|| { + loop { + check()?; + let position = next.fetch_add(1, Ordering::Relaxed); + let Some(stamp) = changed.get(position) else { + break; + }; + let isrc = uppercase(&files.read(&stamp.path, check)?); + values.lock().expect("ISRC parse results lock")[position] = isrc; + } + Ok::<_, String>(()) + }) + }) + .collect::<Vec<_>>(); + for handle in handles { + handle + .join() + .map_err(|_| "ISRC reader worker panicked".to_owned())??; + } + Ok::<_, String>(()) + })?; + // Changed files are applied after reused entries, in walk order, just + // as Go does after joining its four parsing workers. + for (stamp, isrc) in changed + .into_iter() + .zip(values.into_inner().expect("ISRC parse results lock")) + { + check()?; + if !isrc.is_empty() { + index.entries.insert(isrc.clone(), stamp.path.clone()); + } + index + .files + .insert(stamp.path.clone(), FileEntry { stamp, isrc }); + } + check()?; + let index = Arc::new(Mutex::new(index)); + if !directory.is_empty() { + self.indexes + .lock() + .expect("ISRC cache lock") + .insert(directory.into(), Arc::clone(&index)); + } + Ok(index) + } + + pub fn check( + &self, + directory: &str, + isrc: &str, + files: &dyn IndexFiles, + check: Check<'_>, + ) -> Result<String, String> { + check()?; + if directory.is_empty() || isrc.is_empty() { + return Ok(String::new()); + } + let index = self.index(directory, files, false, check)?; + let key = uppercase(isrc); + let path = index + .lock() + .expect("ISRC index lock") + .entries + .get(&key) + .cloned(); + let Some(path) = path else { + return Ok(String::new()); + }; + check()?; + let exists = files + .stat(&path)? + .is_some_and(|stamp| !stamp.directory && stamp.size > 0); + check()?; + if exists { + Ok(path) + } else { + index.lock().expect("ISRC index lock").entries.remove(&key); + Ok(String::new()) + } + } + + pub fn add( + &self, + directory: &str, + isrc: &str, + path: &str, + files: &dyn IndexFiles, + check: Check<'_>, + ) -> Result<(), String> { + check()?; + if directory.is_empty() || isrc.is_empty() || path.is_empty() { + return Ok(()); + } + if let Some(index) = self.cached(directory) { + let stamp = files.stat(path)?; + check()?; + let mut index = index.lock().expect("ISRC index lock"); + let isrc = uppercase(isrc); + index.entries.insert(isrc.clone(), path.into()); + if let Some(stamp) = stamp { + index.files.insert(path.into(), FileEntry { stamp, isrc }); + } + index.built_at = Instant::now(); + } + Ok(()) + } + + pub fn invalidate(&self, directory: &str) { + self.indexes + .lock() + .expect("ISRC cache lock") + .remove(directory); + } + + pub fn clear(&self) { + self.indexes.lock().expect("ISRC cache lock").clear(); + } + + pub fn prebuild( + &self, + directory: &str, + files: &dyn IndexFiles, + check: Check<'_>, + ) -> Result<(), String> { + if directory.is_empty() { + return Err("output directory is required".into()); + } + self.index(directory, files, true, check).map(|_| ()) + } + + pub fn check_batch( + &self, + directory: &str, + tracks: &[TrackQuery], + files: &dyn IndexFiles, + check: Check<'_>, + ) -> Result<Vec<TrackExistence>, String> { + let index = self.index(directory, files, false, check)?; + let index = index.lock().expect("ISRC index lock"); + tracks + .iter() + .map(|track| { + check()?; + // Go's batch API deliberately trusts the cache without stat calls. + let path = index + .entries + .get(&uppercase(&track.isrc)) + .filter(|_| !track.isrc.is_empty()); + Ok(TrackExistence { + isrc: track.isrc.clone(), + exists: path.is_some(), + file_path: path.cloned().unwrap_or_default(), + track_name: track.track_name.clone(), + artist_name: track.artist_name.clone(), + }) + }) + .collect() + } +} + +fn wait_for_builder<'a>( + builder: &'a Mutex<()>, + check: Check<'_>, +) -> Result<MutexGuard<'a, ()>, String> { + loop { + check()?; + match builder.try_lock() { + Ok(guard) => return Ok(guard), + Err(TryLockError::Poisoned(error)) => return Ok(error.into_inner()), + Err(TryLockError::WouldBlock) => std::thread::sleep(Duration::from_millis(5)), + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct TrackQuery { + pub isrc: String, + pub track_name: String, + pub artist_name: String, +} + +impl<'de> Deserialize<'de> for TrackQuery { + fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = TrackQuery; + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a track object") + } + fn visit_map<M: serde::de::MapAccess<'de>>( + self, + mut map: M, + ) -> Result<TrackQuery, M::Error> { + let mut track = TrackQuery::default(); + while let Some(key) = map.next_key::<String>()? { + let field = if equal_field(&key, "isrc") { + Some(&mut track.isrc) + } else if equal_field(&key, "track_name") { + Some(&mut track.track_name) + } else if equal_field(&key, "artist_name") { + Some(&mut track.artist_name) + } else { + None + }; + if let Some(field) = field { + if let Some(value) = map.next_value::<Option<String>>()? { + *field = value; + } + } else { + map.next_value::<serde::de::IgnoredAny>()?; + } + } + Ok(track) + } + } + deserializer.deserialize_map(Visitor) + } +} + +pub fn parse_tracks(json: &str) -> Result<Vec<TrackQuery>, String> { + // Go matches the snake_case JSON tags, folds field-name case, and ignores + // null string values without overwriting an earlier duplicate field. + serde_json::from_str::<Option<Vec<Option<TrackQuery>>>>(&crate::text::json_surrogates(json)) + .map(|tracks| { + tracks + .unwrap_or_default() + .into_iter() + .map(Option::unwrap_or_default) + .collect() + }) + .map_err(|error| format!("failed to parse tracks JSON: {error}")) +} + +fn equal_field(value: &str, target: &str) -> bool { + value.eq_ignore_ascii_case(target) + || (value.chars().count() == target.len() + && value.chars().zip(target.chars()).all(|(actual, expected)| { + actual.eq_ignore_ascii_case(&expected) + || (actual == 'ſ' && expected == 's') + || (actual == 'K' && expected == 'k') + })) +} + +#[derive(Debug, Serialize)] +pub struct TrackExistence { + pub isrc: String, + pub exists: bool, + #[serde(skip_serializing_if = "String::is_empty")] + pub file_path: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub track_name: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub artist_name: String, +} diff --git a/rust_backend/crates/core/src/isrc/mod.rs b/rust_backend/crates/core/src/isrc/mod.rs new file mode 100644 index 00000000..d63281ba --- /dev/null +++ b/rust_backend/crates/core/src/isrc/mod.rs @@ -0,0 +1,117 @@ +//! Duplicate detection retains Go's cache and format-specific ISRC contracts. + +mod cache; +mod native_files; +pub use cache::{FileStamp, IndexCache, IndexFiles, TrackExistence, TrackQuery, parse_tracks}; +pub use native_files::NativeFiles; + +use std::io::{Read, Seek, SeekFrom}; + +pub fn supported(path: &str) -> bool { + std::path::Path::new(path) + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + matches!( + extension.to_ascii_lowercase().as_str(), + "flac" | "mp3" | "m4a" | "ogg" | "opus" + ) + }) +} + +pub fn read_isrc( + reader: &mut (impl Read + Seek), + format: &str, + check: &dyn Fn() -> Result<(), String>, +) -> Result<String, String> { + let interrupted = std::cell::RefCell::new(None); + let checked = || { + let result = check(); + if let Err(error) = &result { + *interrupted.borrow_mut() = Some(error.clone()); + } + result + }; + checked()?; + let value = if format == "flac" { + read_flac(reader, &checked).unwrap_or_default() + } else if matches!(format, "mp3" | "m4a" | "ogg" | "opus") { + crate::tags::read_audio_tags(reader, format, &checked) + .map(|metadata| metadata.isrc.trim().to_owned()) + .unwrap_or_default() + } else { + String::new() + }; + if let Some(error) = interrupted.into_inner() { + return Err(error); + } + check()?; + Ok(value) +} + +fn read_flac( + reader: &mut (impl Read + Seek), + check: &dyn Fn() -> Result<(), String>, +) -> Result<String, String> { + reader.seek(SeekFrom::Start(0)).map_err(|e| e.to_string())?; + let mut header = [0; 4]; + reader.read_exact(&mut header).map_err(|e| e.to_string())?; + if &header != b"fLaC" { + return Ok(String::new()); + } + loop { + check()?; + reader.read_exact(&mut header).map_err(|e| e.to_string())?; + let length = u32::from_be_bytes([0, header[1], header[2], header[3]]); + if header[0] & 0x7f == 4 { + let mut payload = vec![0; length as usize]; + for chunk in payload.chunks_mut(64 * 1024) { + check()?; + reader.read_exact(chunk).map_err(|e| e.to_string())?; + } + return Ok(comment_isrc(&payload, check)); + } + if header[0] & 0x80 != 0 { + return Ok(String::new()); + } + reader + .seek(SeekFrom::Current(i64::from(length))) + .map_err(|e| e.to_string())?; + } +} + +fn comment_isrc(mut payload: &[u8], check: &dyn Fn() -> Result<(), String>) -> String { + fn integer(payload: &mut &[u8]) -> Option<usize> { + let (value, rest) = payload.split_at_checked(4)?; + *payload = rest; + Some(u32::from_le_bytes(value.try_into().ok()?) as usize) + } + let mut parse = || { + let vendor = integer(&mut payload)?; + payload = payload.get(vendor..)?; + let count = integer(&mut payload)?; + for _ in 0..count { + check().ok()?; + let length = integer(&mut payload)?; + let (comment, rest) = payload.split_at_checked(length)?; + payload = rest; + let Some(equal) = comment.iter().position(|byte| *byte == b'=') else { + continue; + }; + let key = std::str::from_utf8(&comment[..equal]).ok(); + if key.is_some_and(isrc_key) { + return Some(crate::text::utf8(&comment[equal + 1..]).trim().to_owned()); + } + } + None + }; + parse().unwrap_or_default() +} + +fn isrc_key(key: &str) -> bool { + // EqualFold accepts long s but not dotless i; uppercasing alone differs. + key.chars().count() == 4 + && key.chars().zip("ISRC".chars()).all(|(actual, expected)| { + actual.eq_ignore_ascii_case(&expected) || (actual == 'ſ' && expected == 'S') + }) +} diff --git a/rust_backend/crates/core/src/isrc/native_files.rs b/rust_backend/crates/core/src/isrc/native_files.rs new file mode 100644 index 00000000..44b25c69 --- /dev/null +++ b/rust_backend/crates/core/src/isrc/native_files.rs @@ -0,0 +1,110 @@ +use super::{FileStamp, IndexFiles}; +use std::fs::{self, OpenOptions}; +use std::path::{Component, Path, PathBuf}; +use std::time::UNIX_EPOCH; + +/// Ambient filesystem access for the trusted native application. JavaScript +/// hosts must use their scoped adapter instead. +pub struct NativeFiles; + +impl IndexFiles for NativeFiles { + fn list( + &self, + directory: &str, + check: &(dyn Fn() -> Result<(), String> + Sync), + ) -> Result<Vec<FileStamp>, String> { + let mut pending = vec![PathBuf::from(directory)]; + let mut result = Vec::new(); + while let Some(path) = pending.pop() { + check()?; + let Ok(metadata) = fs::symlink_metadata(&path) else { + continue; + }; + if !metadata.is_dir() { + result.push(stamp(&path, &metadata)); + continue; + } + let Ok(entries) = fs::read_dir(&path) else { + continue; + }; + let mut names = Vec::new(); + for entry in entries { + check()?; + if let Ok(entry) = entry { + names.push(entry.file_name()); + } + } + names.sort(); + pending.extend(names.into_iter().rev().map(|name| clean(&path.join(name)))); + } + check()?; + Ok(result) + } + + fn read( + &self, + path: &str, + check: &(dyn Fn() -> Result<(), String> + Sync), + ) -> Result<String, String> { + check()?; + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(rustix::fs::OFlags::NONBLOCK.bits() as i32); + } + let Ok(mut file) = options.open(path) else { + return Ok(String::new()); + }; + if !file.metadata().is_ok_and(|metadata| metadata.is_file()) { + return Ok(String::new()); + } + let format = Path::new(path) + .extension() + .and_then(|value| value.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + super::read_isrc(&mut file, &format, check) + } + + fn stat(&self, path: &str) -> Result<Option<FileStamp>, String> { + Ok(fs::metadata(path) + .ok() + .map(|metadata| stamp(Path::new(path), &metadata))) + } +} + +fn stamp(path: &Path, metadata: &fs::Metadata) -> FileStamp { + let modified_ns = metadata + .modified() + .map(|time| match time.duration_since(UNIX_EPOCH) { + Ok(duration) => duration.as_nanos() as i128, + Err(error) => -(error.duration().as_nanos() as i128), + }) + .unwrap_or_default(); + FileStamp { + path: path.to_string_lossy().into_owned(), + size: metadata.len(), + modified_ns, + directory: metadata.is_dir(), + } +} + +fn clean(path: &Path) -> PathBuf { + let mut output = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir if output.file_name().is_some_and(|name| name != "..") => { + output.pop(); + } + Component::ParentDir if output.has_root() => {} + other => output.push(other.as_os_str()), + } + } + if output.as_os_str().is_empty() { + output.push("."); + } + output +} diff --git a/rust_backend/crates/core/src/lib.rs b/rust_backend/crates/core/src/lib.rs new file mode 100644 index 00000000..4cf4d16b --- /dev/null +++ b/rust_backend/crates/core/src/lib.rs @@ -0,0 +1,19 @@ +//! Shared backend logic, independent of mobile bindings and UI runtimes. + +pub mod app_version; +pub mod cancellation; +pub mod clock; +pub mod cover; +pub mod cue; +pub mod downloads; +pub mod filename; +pub mod isrc; +pub mod lyrics; +pub mod matching; +pub mod media; +pub mod metadata; +pub mod progress; +pub mod resolver; +pub mod tags; +mod text; +pub use text::json_surrogates as normalize_json_surrogates; diff --git a/rust_backend/crates/core/src/lyrics.rs b/rust_backend/crates/core/src/lyrics.rs new file mode 100644 index 00000000..dadb78f1 --- /dev/null +++ b/rust_backend/crates/core/src/lyrics.rs @@ -0,0 +1,110 @@ +//! Lyrics contracts shared by built-in providers, extension hosts and exports. + +pub mod config; +pub mod errors; +pub mod file; +pub(crate) mod json; +pub mod lrc; +pub mod lrclib; +pub mod matching; +pub mod models; +pub mod payloads; + +use serde::{Deserialize, Serialize}; + +/// HTTP providers use Go's streaming JSON decoder: decode the first value, +/// replacing invalid UTF-8 and unpaired UTF-16 escapes before typed decoding. +pub fn decode_response<T: serde::de::DeserializeOwned>( + bytes: &[u8], +) -> Result<T, serde_json::Error> { + let text = crate::text::utf8(bytes); + let normalized = crate::text::json_surrogates(&text); + // Go scans a complete JSON value before decoding typed fields. Otherwise a + // wrong root type could hide truncated JSON and change provider cooldowns. + let raw = <&serde_json::value::RawValue>::deserialize( + &mut serde_json::Deserializer::from_str(&normalized), + )?; + serde_json::from_str(raw.get()) +} + +/// Complete-document counterpart to the streaming HTTP decoder. +pub fn decode_document<T: serde::de::DeserializeOwned>( + bytes: &[u8], +) -> Result<T, serde_json::Error> { + let text = crate::text::utf8(bytes); + let normalized = crate::text::json_surrogates(&text); + let raw: &serde_json::value::RawValue = serde_json::from_str(&normalized)?; + serde_json::from_str(raw.get()) +} + +pub fn text_from_bytes(bytes: &[u8]) -> String { + crate::text::utf8(bytes) +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LyricsLine { + pub start_time_ms: i64, + pub words: String, + pub end_time_ms: i64, +} + +json::go_deserialize!(LyricsLine { + "starttimems" => start_time_ms, + "words" => words, + "endtimems" => end_time_ms, +}); + +#[derive(Clone, Debug, Default, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LyricsResponse { + // Go distinguishes an uninitialized slice (null) from an explicit []. + pub lines: Option<Vec<LyricsLine>>, + pub sync_type: String, + pub instrumental: bool, + pub plain_lyrics: String, + pub provider: String, + pub source: String, +} + +json::go_deserialize!(LyricsResponse { + "lines" => lines, + "synctype" => sync_type, + "instrumental" => instrumental, + "plainlyrics" => plain_lyrics, + "provider" => provider, + "source" => source, +}); + +impl LyricsResponse { + pub fn lines(&self) -> &[LyricsLine] { + self.lines.as_deref().unwrap_or_default() + } + + pub fn has_usable_text(&self) -> bool { + self.instrumental + || !self.plain_lyrics.trim().is_empty() + || self + .lines() + .iter() + .any(|line| !line.words.trim().is_empty()) + } + + pub fn from_text(text: &str, provider: &str, source: &str) -> Self { + let mut result = Self { + provider: provider.into(), + source: source.into(), + ..Self::default() + }; + if let Some(lines) = lrc::parse_synced(text) { + result.plain_lyrics = lrc::plain_from_timed_lines(&lines); + result.lines = Some(lines); + result.sync_type = "LINE_SYNCED".into(); + } else if let Some(lines) = lrc::plain_text_lines(text) { + result.lines = Some(lines); + result.sync_type = "UNSYNCED".into(); + result.plain_lyrics = text.into(); + } + result + } +} diff --git a/rust_backend/crates/core/src/lyrics/config.rs b/rust_backend/crates/core/src/lyrics/config.rs new file mode 100644 index 00000000..e1c547bd --- /dev/null +++ b/rust_backend/crates/core/src/lyrics/config.rs @@ -0,0 +1,202 @@ +use crate::matching::lowercase; +use serde::Serialize; +use std::collections::HashSet; + +pub const DEFAULT_PROVIDERS: &[&str] = &["lrclib", "apple_music"]; +pub const BUILTIN_PROVIDERS: &[&str] = &[ + "lrclib", + "netease", + "musixmatch", + "apple_music", + "qqmusic", + "spotify", + "deezer", + "youtube", + "kugou", + "genius", + "lyricsplus", +]; + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct FetchOptions { + pub include_translation_netease: bool, + pub include_romanization_netease: bool, + pub multi_person_word_by_word: bool, + pub apple_elrc_word_sync: bool, + #[serde(skip_serializing_if = "String::is_empty")] + pub musixmatch_language: String, +} + +super::json::go_deserialize!(FetchOptions { + "include_translation_netease" => include_translation_netease, + "include_romanization_netease" => include_romanization_netease, + "multi_person_word_by_word" => multi_person_word_by_word, + "apple_elrc_word_sync" => apple_elrc_word_sync, + "musixmatch_language" => musixmatch_language, +}); + +impl Default for FetchOptions { + fn default() -> Self { + Self { + include_translation_netease: false, + include_romanization_netease: false, + multi_person_word_by_word: true, + apple_elrc_word_sync: false, + musixmatch_language: String::new(), + } + } +} + +impl FetchOptions { + /// Go unmarshals partial settings into the current options. Commit only a + /// fully valid document, including duplicate fields and null scalar values. + pub fn update_json(&mut self, raw: &str) -> Result<(), serde_json::Error> { + use super::json::Update; + if raw.trim().is_empty() { + return Ok(()); + } + let document = super::decode_document::<Box<serde_json::value::RawValue>>(raw.as_bytes())?; + let mut next = self.clone(); + next.update(&mut serde_json::Deserializer::from_str(document.get()))?; + *self = next; + Ok(()) + } + + pub fn normalize(&mut self) { + self.musixmatch_language = lowercase(self.musixmatch_language.trim()) + .chars() + .filter(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || matches!(ch, '-' | '_')) + .take(16) + .collect(); + } +} + +pub fn decode_providers(raw: &str) -> Result<Vec<String>, serde_json::Error> { + Ok( + super::decode_document::<Option<Vec<Option<String>>>>(raw.as_bytes())? + .unwrap_or_default() + .into_iter() + .map(Option::unwrap_or_default) + .collect(), + ) +} + +#[derive(Serialize)] +pub struct AvailableProvider { + pub id: &'static str, + pub name: &'static str, + pub has_proxy_dependency: bool, + pub description: &'static str, +} + +pub fn available_providers() -> Vec<AvailableProvider> { + [ + ( + "lrclib", + "LRCLIB", + false, + "Open-source synced lyrics database", + ), + ("netease", "Netease", true, "NetEase Cloud Music lyrics"), + ("musixmatch", "Musixmatch", true, "Musixmatch lyrics"), + ( + "apple_music", + "Apple Music", + true, + "Apple Music synced lyrics", + ), + ( + "qqmusic", + "QQ Music", + false, + "Direct QQ Music line-synced lyrics", + ), + ("spotify", "Spotify", true, "Spotify synced lyrics"), + ("deezer", "Deezer", true, "Deezer lyrics"), + ("youtube", "YouTube", true, "YouTube lyrics"), + ("kugou", "Kugou", false, "Direct Kugou synced lyrics"), + ("genius", "Genius", false, "Direct Genius lyrics"), + ( + "lyricsplus", + "LyricsPlus", + true, + "Word-by-word karaoke lyrics (Apple/Musixmatch/Spotify/QQ)", + ), + ] + .into_iter() + .map( + |(id, name, has_proxy_dependency, description)| AvailableProvider { + id, + name, + has_proxy_dependency, + description, + }, + ) + .collect() +} + +/// Empty/invalid selections use defaults when read, matching Go's getter. +pub fn provider_order(providers: &[String]) -> Vec<String> { + let mut result = normalize_provider_order(providers); + if result.is_empty() { + result = DEFAULT_PROVIDERS + .iter() + .map(|name| (*name).into()) + .collect(); + } + result +} + +pub fn normalize_provider_order(providers: &[String]) -> Vec<String> { + let mut seen = HashSet::new(); + providers + .iter() + .map(|name| lowercase(name.trim())) + .filter(|name| { + BUILTIN_PROVIDERS.contains(&name.as_str()) + || name + .strip_prefix("extension:") + .is_some_and(|id| !id.trim().is_empty()) + }) + .filter(|name| seen.insert(name.clone())) + .collect() +} + +pub fn resolve_order(configured: &[String], extensions: &[String]) -> Vec<String> { + let available: HashSet<_> = extensions + .iter() + .map(|id| format!("extension:{}", lowercase(id.trim()))) + .collect(); + configured + .iter() + .filter(|name| BUILTIN_PROVIDERS.contains(&name.as_str()) || available.contains(*name)) + .cloned() + .collect() +} + +pub fn cache_key( + spotify_id: &str, + track: &str, + artist: &str, + duration: f64, + providers: &[String], + extensions: &[String], + options: &FetchOptions, +) -> String { + let mut extensions: Vec<_> = extensions.iter().map(|id| lowercase(id.trim())).collect(); + extensions.sort(); + format!( + "{}|{}|{}|{:.0}|{}|{}|{}|{}|{}|{}|{}", + spotify_id.trim(), + lowercase(artist.trim()), + lowercase(track.trim()), + (duration / 10.0).round() * 10.0, + providers.join(","), + extensions.join(","), + options.include_translation_netease, + options.include_romanization_netease, + options.multi_person_word_by_word, + options.apple_elrc_word_sync, + options.musixmatch_language + ) +} diff --git a/rust_backend/crates/core/src/lyrics/errors.rs b/rust_backend/crates/core/src/lyrics/errors.rs new file mode 100644 index 00000000..8cd8f3ab --- /dev/null +++ b/rust_backend/crates/core/src/lyrics/errors.rs @@ -0,0 +1,102 @@ +use crate::matching::lowercase; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ErrorKind { + NotFound, + Unavailable, + Other, +} + +pub fn http_status(status: u16) -> ErrorKind { + if status == 429 || status >= 500 { + ErrorKind::Unavailable + } else { + ErrorKind::Other + } +} + +pub fn payload_not_found(message: &str) -> bool { + let message = lowercase(message); + [ + "lyrics not found", + "no lyrics found", + "no songs found", + "not found", + ] + .iter() + .any(|signal| message.contains(signal)) +} + +pub fn classify_payload(status: u16, message: &str) -> ErrorKind { + if payload_not_found(message) { + return ErrorKind::NotFound; + } + let message = lowercase(message); + if [ + "rate limit", + "too many requests", + "operation too frequent", + "操作频繁", + "missing required parameters", + ] + .iter() + .any(|signal| message.contains(signal)) + { + return ErrorKind::Unavailable; + } + http_status(status) +} + +pub fn detect_payload(raw: &str) -> Option<String> { + let raw = raw.trim(); + if !raw.starts_with('{') { + return None; + } + let value: Value = serde_json::from_str(&crate::text::json_surrogates(raw)).ok()?; + let payload = value.as_object()?; + if [ + "lyrics", + "lyric", + "lrc", + "content", + "lines", + "syncedLyrics", + "unsyncedLyrics", + ] + .iter() + .any(|key| payload.contains_key(*key)) + { + return None; + } + for key in ["message", "error", "detail", "reason"] { + if let Some(message) = value[key] + .as_str() + .map(str::trim) + .filter(|text| !text.is_empty()) + { + return Some(message.into()); + } + } + if value["success"] == false || value["isError"] == true { + return Some("request unsuccessful".into()); + } + if let Some(code) = value["code"].as_f64() + && code != 0.0 + && code != 200.0 + { + for key in ["message", "msg"] { + if let Some(message) = value[key] + .as_str() + .map(str::trim) + .filter(|text| !text.is_empty()) + { + return Some(message.into()); + } + } + return Some(format!("unexpected response code {code:.0}")); + } + None +} diff --git a/rust_backend/crates/core/src/lyrics/file.rs b/rust_backend/crates/core/src/lyrics/file.rs new file mode 100644 index 00000000..e5228003 --- /dev/null +++ b/rust_backend/crates/core/src/lyrics/file.rs @@ -0,0 +1,81 @@ +//! Existing-file lyrics with caller-owned file access and cancellation. + +use super::text_from_bytes; +use std::io::{Read, Seek}; + +const MAX_SIDECAR_BYTES: usize = 8 * 1024 * 1024; + +pub fn extract<R: Read + Seek>( + path: &str, + open: &impl Fn(&str) -> Result<R, String>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<String, String> { + check()?; + // Match filepath.Ext, including dotfiles and trailing dots on Unix. + let suffix = path + .rfind('.') + .filter(|index| !path[*index..].contains('/')); + let format = suffix.map_or_else(String::new, |index| path[index + 1..].to_ascii_lowercase()); + if matches!( + format.as_str(), + "flac" | "m4a" | "mp4" | "aac" | "mp3" | "opus" | "ogg" | "wav" | "aiff" | "aif" | "aifc" + ) && let Ok(metadata) = + open(path).and_then(|mut file| crate::tags::read_audio_tags(&mut file, &format, check)) + { + check()?; + if !metadata.lyrics.trim().is_empty() { + return Ok(metadata.lyrics); + } + if !matches!(format.as_str(), "flac" | "m4a" | "mp4" | "aac") + && looks_like_lyrics(&metadata.comment) + { + return Ok(metadata.comment); + } + } + check()?; + sidecar(path, open, check) +} + +pub fn sidecar<R: Read>( + path: &str, + open: &impl Fn(&str) -> Result<R, String>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<String, String> { + check()?; + let suffix = path + .rfind('.') + .filter(|index| !path[*index..].contains('/')); + let base = &path[..suffix.unwrap_or(path.len())]; + if base.trim().is_empty() { + return Err("no lyrics found in file".into()); + } + let mut file = open(&format!("{base}.lrc"))?; + let mut bytes = Vec::new(); + let mut buffer = [0; 16 * 1024]; + loop { + check()?; + let count = file.read(&mut buffer).map_err(|error| error.to_string())?; + if count == 0 { + break; + } + if bytes.len() + count > MAX_SIDECAR_BYTES { + return Err("lyrics sidecar exceeds size limit".into()); + } + bytes.extend_from_slice(&buffer[..count]); + } + check()?; + let text = text_from_bytes(&bytes).trim().to_owned(); + if text.is_empty() { + Err("no lyrics found in file".into()) + } else { + Ok(text) + } +} + +fn looks_like_lyrics(text: &str) -> bool { + let text = text.trim(); + let lower = text.to_ascii_lowercase(); + lower.contains("[ar:") + || lower.contains("[ti:") + || (text.contains('\n') && text.contains('[') && text.contains(']')) +} diff --git a/rust_backend/crates/core/src/lyrics/json.rs b/rust_backend/crates/core/src/lyrics/json.rs new file mode 100644 index 00000000..220f467d --- /dev/null +++ b/rust_backend/crates/core/src/lyrics/json.rs @@ -0,0 +1,163 @@ +//! Go's typed provider JSON accepts case-insensitive fields and null scalars. + +use serde::de::{DeserializeOwned, DeserializeSeed, Visitor}; +use serde::{Deserialize, Deserializer}; +use std::borrow::Cow; + +pub(super) fn decode<T: DeserializeOwned>(raw: &str) -> Result<T, serde_json::Error> { + serde_json::from_str(&crate::text::json_surrogates(raw)) +} + +pub(crate) fn field_name(name: &str) -> String { + name.chars() + .map(|ch| match ch { + 'ſ' => 's', + 'K' => 'k', + _ => ch.to_ascii_lowercase(), + }) + .collect() +} + +struct KeyVisitor; + +impl<'de> Visitor<'de> for KeyVisitor { + type Value = Cow<'de, str>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a string key") + } + + fn visit_borrowed_str<E: serde::de::Error>(self, value: &'de str) -> Result<Self::Value, E> { + Ok(Cow::Borrowed(value)) + } + + fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> { + Ok(Cow::Owned(value.into())) + } + + fn visit_string<E: serde::de::Error>(self, value: String) -> Result<Self::Value, E> { + Ok(Cow::Owned(value)) + } +} + +pub(crate) struct KeySeed; + +impl<'de> DeserializeSeed<'de> for KeySeed { + type Value = Cow<'de, str>; + + fn deserialize<D: Deserializer<'de>>(self, decoder: D) -> Result<Self::Value, D::Error> { + decoder.deserialize_str(KeyVisitor) + } +} + +pub(crate) trait Update<'de> { + fn update<D: Deserializer<'de>>(&mut self, decoder: D) -> Result<(), D::Error>; +} + +macro_rules! scalar { + ($($type:ty),*) => {$( + impl<'de> Update<'de> for $type { + fn update<D: Deserializer<'de>>(&mut self, decoder: D) -> Result<(), D::Error> { + // A later null does not overwrite an already decoded scalar. + if let Some(value) = Option::<Self>::deserialize(decoder)? { *self = value; } + Ok(()) + } + } + )*}; +} +scalar!(String, bool, i64, isize, f64); + +impl<'de, T: Deserialize<'de>> Update<'de> for Option<T> { + fn update<D: Deserializer<'de>>(&mut self, decoder: D) -> Result<(), D::Error> { + *self = Self::deserialize(decoder)?; + Ok(()) + } +} + +pub(crate) struct Field<'a, T>(pub &'a mut T); + +impl<'de, T: Update<'de>> DeserializeSeed<'de> for Field<'_, T> { + type Value = (); + fn deserialize<D: Deserializer<'de>>(self, decoder: D) -> Result<(), D::Error> { + self.0.update(decoder) + } +} + +macro_rules! go_deserialize { + ($name:ident { $($key:literal => $field:ident,)* }) => { + impl<'de> serde::Deserialize<'de> for $name { + fn deserialize<D: serde::Deserializer<'de>>(decoder: D) -> Result<Self, D::Error> { + let mut result = Self::default(); + $crate::lyrics::json::Update::update(&mut result, decoder)?; + Ok(result) + } + } + impl<'de> $crate::lyrics::json::Update<'de> for $name { + fn update<D: serde::Deserializer<'de>>(&mut self, decoder: D) -> Result<(), D::Error> { + struct Visitor<'a>(&'a mut $name); + impl<'de> serde::de::Visitor<'de> for Visitor<'_> { + type Value = (); + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a lyrics object or null") + } + fn visit_unit<E: serde::de::Error>(self) -> Result<(), E> { + Ok(()) + } + fn visit_map<M: serde::de::MapAccess<'de>>(self, mut map: M) -> Result<(), M::Error> { + while let Some(key) = map.next_key_seed($crate::lyrics::json::KeySeed)? { + let raw_key = key.as_ref(); + let folded; + let lookup = match raw_key { + $($key => raw_key,)* + _ => { + folded = $crate::lyrics::json::field_name(raw_key); + folded.as_str() + } + }; + match lookup { + $($key => map.next_value_seed($crate::lyrics::json::Field(&mut self.0.$field))?,)* + _ => { map.next_value::<serde::de::IgnoredAny>()?; } + } + } + Ok(()) + } + } + decoder.deserialize_any(Visitor(self)) + } + } + }; +} +pub(crate) use go_deserialize; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn key_seed_borrows_canonical_keys_and_owns_escaped_keys() { + let mut decoder = serde_json::Deserializer::from_str(r#""source""#); + assert!(matches!( + KeySeed.deserialize(&mut decoder).unwrap(), + Cow::Borrowed("source") + )); + + let mut decoder = serde_json::Deserializer::from_str(r#""so\u0075rce""#); + assert!(matches!( + KeySeed.deserialize(&mut decoder).unwrap(), + Cow::Owned(value) if value == "source" + )); + } + + #[test] + fn field_folding_preserves_special_letters_and_null_duplicates() { + assert_eq!(field_name("ſOURCE"), "source"); + assert_eq!(field_name("K"), "k"); + + let response: crate::lyrics::LyricsResponse = serde_json::from_str( + r#"{"plainlyrics":"kept","PLAINLYRICS":null,"ſource":"old","SOURCE":"new"}"#, + ) + .unwrap(); + assert_eq!(response.plain_lyrics, "kept"); + assert_eq!(response.source, "new"); + } +} diff --git a/rust_backend/crates/core/src/lyrics/lrc.rs b/rust_backend/crates/core/src/lyrics/lrc.rs new file mode 100644 index 00000000..0fc59c68 --- /dev/null +++ b/rust_backend/crates/core/src/lyrics/lrc.rs @@ -0,0 +1,189 @@ +use super::{LyricsLine, LyricsResponse}; +use crate::matching::lowercase; +use regex::Regex; +use std::sync::LazyLock; + +static TIMED_LINE: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"\[([0-9]{2}):([0-9]{2})\.([0-9]{2,3})\](.*)").unwrap()); +static METADATA: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"(?i)^\[[a-z][a-z0-9_]*:.*\]$").unwrap()); +static BACKGROUND: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)^\[bg:(.*)\]$").unwrap()); +static LEADING_TIME: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"^\[[0-9]{1,3}:[0-9]{1,2}(?:[.:][0-9]{1,3})?\]").unwrap()); +static INLINE_TIME: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"<[0-9]{1,3}:[0-9]{1,2}(?:[.:][0-9]{1,3})?>").unwrap()); +static INSTRUMENTAL: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"(?i)^\[instrumental:true\]$").unwrap()); + +pub fn is_instrumental_marker(raw: &str) -> bool { + INSTRUMENTAL.is_match(raw.trim()) +} + +/// Headers, empty timestamps and empty vocal markers are not embedded lyrics. +pub fn has_usable_content(raw: &str) -> bool { + if is_instrumental_marker(raw) { + return true; + } + for line in raw.split('\n') { + let mut cleaned = line.trim(); + if cleaned.is_empty() { + continue; + } + if let Some(background) = BACKGROUND.captures(cleaned) { + cleaned = background.get(1).unwrap().as_str().trim(); + } else if METADATA.is_match(cleaned) { + continue; + } + while let Some(timestamp) = LEADING_TIME.find(cleaned) { + cleaned = cleaned[timestamp.end()..].trim(); + } + let without_inline = INLINE_TIME.replace_all(cleaned, ""); + cleaned = without_inline.trim(); + let lower = lowercase(cleaned); + if lower.starts_with("v1:") || lower.starts_with("v2:") { + cleaned = cleaned[3..].trim(); + } + if !cleaned.is_empty() { + return true; + } + } + false +} + +pub fn parse_synced(raw: &str) -> Option<Vec<LyricsLine>> { + let mut lines: Vec<LyricsLine> = Vec::new(); + for line in raw.split('\n').map(str::trim) { + if line.is_empty() { + continue; + } + if line.starts_with("[bg:") + && let Some(previous) = lines.last_mut() + { + previous.words.push('\n'); + previous.words.push_str(line); + continue; + } + if let Some(captures) = TIMED_LINE.captures(line) { + let words = captures[4].trim(); + if words.is_empty() { + continue; + } + let minutes = captures[1].parse::<i64>().unwrap(); + let seconds = captures[2].parse::<i64>().unwrap(); + let mut fraction = captures[3].parse::<i64>().unwrap(); + if captures[3].len() == 2 { + fraction *= 10; + } + lines.push(LyricsLine { + start_time_ms: minutes * 60_000 + seconds * 1000 + fraction, + words: words.into(), + end_time_ms: 0, + }); + } + } + for index in 1..lines.len() { + lines[index - 1].end_time_ms = lines[index].start_time_ms; + } + if let Some(last) = lines.last_mut() { + last.end_time_ms = last.start_time_ms + 5000; + } + (!lines.is_empty()).then_some(lines) +} + +pub fn plain_text_lines(raw: &str) -> Option<Vec<LyricsLine>> { + let lines: Vec<_> = raw + .split('\n') + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(|line| LyricsLine { + words: line.into(), + ..LyricsLine::default() + }) + .collect(); + (!lines.is_empty()).then_some(lines) +} + +pub fn plain_from_timed_lines(lines: &[LyricsLine]) -> String { + lines + .iter() + .map(|line| line.words.trim()) + .filter(|words| !words.is_empty()) + .collect::<Vec<_>>() + .join("\n") +} + +pub fn timestamp_inline(ms: i64) -> String { + let seconds = ms / 1000; + format!( + "{:02}:{:02}.{:02}", + seconds / 60, + seconds % 60, + ms % 1000 / 10 + ) +} + +pub fn timestamp(ms: i64) -> String { + format!("[{}]", timestamp_inline(ms)) +} + +pub fn source_uses_proxy(source: &str) -> bool { + let source = lowercase(source.trim()); + !source.is_empty() + && ![ + "lrclib", + "kugou direct", + "qq music direct", + "genius direct", + "extension:", + "heuristic", + ] + .iter() + .any(|prefix| source.starts_with(prefix)) +} + +pub fn extract_source(raw: &str) -> String { + for line in raw.split('\n').map(str::trim) { + if !lowercase(line).starts_with("[by:") { + continue; + } + let Some((_, source)) = line.split_once("(source: ") else { + return String::new(); + }; + let source = source.trim(); + let source = source.strip_suffix(']').unwrap_or(source); + return source.strip_suffix(')').unwrap_or(source).trim().into(); + } + String::new() +} + +pub fn with_metadata(lyrics: &LyricsResponse, track: &str, artist: &str) -> String { + if lyrics.lines().is_empty() { + return String::new(); + } + let source = if lyrics.source.trim().is_empty() { + lyrics.provider.trim() + } else { + lyrics.source.trim() + }; + let credit = if source_uses_proxy(source) { + "SpotiFLAC-Mobile via Paxsenix API" + } else { + "SpotiFLAC-Mobile" + }; + let mut output = format!("[ti:{track}]\n[ar:{artist}]\n[by:{credit}"); + if !source.is_empty() { + output.push_str(&format!(" (source: {source})")); + } + output.push_str("]\n\n"); + for line in lyrics.lines() { + if line.words.is_empty() { + continue; + } + if lyrics.sync_type == "LINE_SYNCED" { + output.push_str(×tamp(line.start_time_ms)); + } + output.push_str(&line.words); + output.push('\n'); + } + output +} diff --git a/rust_backend/crates/core/src/lyrics/lrclib.rs b/rust_backend/crates/core/src/lyrics/lrclib.rs new file mode 100644 index 00000000..732ad79d --- /dev/null +++ b/rust_backend/crates/core/src/lyrics/lrclib.rs @@ -0,0 +1,95 @@ +use super::{LyricsLine, LyricsResponse, lrc, matching}; +use serde::Serialize; + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LrcLibResponse { + pub id: isize, + pub name: String, + pub track_name: String, + pub artist_name: String, + pub album_name: String, + pub duration: f64, + pub instrumental: bool, + pub plain_lyrics: String, + pub synced_lyrics: String, +} + +super::json::go_deserialize!(LrcLibResponse { + "id" => id, "name" => name, "trackname" => track_name, + "artistname" => artist_name, "albumname" => album_name, "duration" => duration, + "instrumental" => instrumental, "plainlyrics" => plain_lyrics, + "syncedlyrics" => synced_lyrics, +}); + +impl LrcLibResponse { + pub fn track_name(&self) -> &str { + if self.track_name.trim().is_empty() { + self.name.trim() + } else { + self.track_name.trim() + } + } + + pub fn matches(&self, query: &str, track: &str, artist: &str, duration: f64) -> bool { + if !matching::duration_matches(self.duration, duration) { + return false; + } + if !track.trim().is_empty() || !artist.trim().is_empty() { + return matching::titles_match(self.track_name(), track, false) + && matching::artists_match(&self.artist_name, artist); + } + let query = matching::normalize_loose_artist(query); + let track = matching::normalize_loose_artist(&matching::simplify_track(self.track_name())); + let artist = matching::normalize_loose_artist(&matching::primary_artist(&self.artist_name)); + matching::contains_words(&query, &track) && matching::contains_words(&query, &artist) + } + + pub fn into_lyrics(&self) -> LyricsResponse { + let mut result = LyricsResponse { + instrumental: self.instrumental, + plain_lyrics: self.plain_lyrics.clone(), + provider: "LRCLIB".into(), + ..LyricsResponse::default() + }; + if !self.synced_lyrics.is_empty() { + result.lines = lrc::parse_synced(&self.synced_lyrics); + result.sync_type = "LINE_SYNCED".into(); + } else if !self.plain_lyrics.is_empty() { + result.sync_type = "UNSYNCED".into(); + let lines: Vec<_> = self + .plain_lyrics + .split('\n') + .filter(|line| !line.trim().is_empty()) + .map(|line| LyricsLine { + words: line.into(), + ..LyricsLine::default() + }) + .collect(); + result.lines = (!lines.is_empty()).then_some(lines); + } + result + } +} + +pub fn select<'a>( + results: &'a [LrcLibResponse], + query: &str, + track: &str, + artist: &str, + duration: f64, +) -> Option<&'a LrcLibResponse> { + let mut synced = None; + let mut plain = None; + for result in results { + if !result.matches(query, track, artist, duration) { + continue; + } + if !result.synced_lyrics.is_empty() && synced.is_none() { + synced = Some(result); + } else if !result.plain_lyrics.is_empty() && plain.is_none() { + plain = Some(result); + } + } + synced.or(plain) +} diff --git a/rust_backend/crates/core/src/lyrics/matching.rs b/rust_backend/crates/core/src/lyrics/matching.rs new file mode 100644 index 00000000..d7df7818 --- /dev/null +++ b/rust_backend/crates/core/src/lyrics/matching.rs @@ -0,0 +1,223 @@ +use crate::matching::lowercase; +use regex::Regex; +use std::borrow::Cow; +use std::sync::LazyLock; +use unicode_general_category::{GeneralCategory, get_general_category}; +use unicode_normalization::UnicodeNormalization; + +static SIMPLIFY: LazyLock<Vec<Regex>> = LazyLock::new(|| { + [ + r"\s*\(feat\.?.*?\)", + r"\s*\(ft\.?.*?\)", + r"\s*\(featuring.*?\)", + r"\s*\(with.*?\)", + r"\s*-\s*Remaster(ed)?.*$", + r"\s*-\s*\d{4}\s*Remaster.*$", + r"\s*\(Remaster(ed)?.*?\)", + r"\s*\(Deluxe.*?\)", + r"\s*\(Bonus.*?\)", + r"\s*\(Live.*?\)", + r"\s*\(Acoustic.*?\)", + r"\s*\(Radio Edit\)", + r"\s*\(Single Version\)", + ] + .iter() + .map(|pattern| { + // Go's Perl character classes are ASCII even with Unicode case folding. + Regex::new(&format!( + "(?i){}", + pattern + .replace(r"\s", r"[\t\n\f\r ]") + .replace(r"\d", "[0-9]") + )) + .unwrap() + }) + .collect() +}); +static INSTRUMENTAL: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"(?i)(?:^|[\t\n\f\r \[(\-])(?:instrumental|inst\.?)(?:[\t\n\f\r \])]|$)").unwrap() +}); + +fn normalized(value: &str, artist: bool) -> String { + use GeneralCategory::*; + let lower = lowercase(value.trim()); + let mut output = String::with_capacity(lower.len()); + let characters: Box<dyn Iterator<Item = char>> = if artist { + Box::new(lower.nfd()) + } else { + Box::new(lower.chars()) + }; + for ch in characters { + match get_general_category(ch) { + UppercaseLetter | LowercaseLetter | TitlecaseLetter | ModifierLetter | OtherLetter + | DecimalNumber | LetterNumber | OtherNumber => match (artist, ch) { + (true, 'đ') => output.push_str("dj"), + (true, 'ß') => output.push_str("ss"), + (true, 'æ') => output.push_str("ae"), + (true, 'œ') => output.push_str("oe"), + _ => output.push(ch), + }, + _ if ch.is_whitespace() + || matches!(ch, '/' | '\\' | '_' | '-' | '|' | '.' | '&' | '+') => + { + output.push(' ') + } + _ => {} + } + } + output.split_whitespace().collect::<Vec<_>>().join(" ") +} + +pub fn normalize_title(value: &str) -> String { + normalized(value, false) +} + +pub fn normalize_loose_artist(value: &str) -> String { + normalized(value, true) +} + +pub fn simplify_track(value: &str) -> String { + let mut output = value.to_owned(); + for pattern in SIMPLIFY.iter() { + if let Cow::Owned(replaced) = pattern.replace_all(&output, "") { + output = replaced; + } + } + let normalized = normalize_title(output.trim()); + if normalized.is_empty() { + output.trim().into() + } else { + normalized + } +} + +pub fn primary_artist(value: &str) -> String { + let lowered = lowercase(value); + for separator in [ + ", ", + "; ", + " & ", + " feat. ", + " ft. ", + " featuring ", + " with ", + ] { + if let Some(index) = lowered.find(separator) + && index > 0 + { + // Go slices the original by the lowercased byte offset. Preserve + // replacement bytes if a lowercase mapping changed its UTF-8 width. + return crate::text::utf8(&value.as_bytes()[..index.min(value.len())]) + .trim() + .into(); + } + } + value.trim().into() +} + +pub fn contains_words(value: &str, sequence: &str) -> bool { + let value: Vec<_> = value.split_whitespace().collect(); + let sequence: Vec<_> = sequence.split_whitespace().collect(); + !sequence.is_empty() && value.windows(sequence.len()).any(|words| words == sequence) +} + +pub fn titles_match(candidate: &str, expected: &str, decorated: bool) -> bool { + let candidate = lowercase(simplify_track(candidate).trim()); + let expected = lowercase(simplify_track(expected).trim()); + !candidate.is_empty() + && !expected.is_empty() + && (candidate == expected || (decorated && contains_words(&candidate, &expected))) +} + +pub fn artists_match(candidate: &str, expected: &str) -> bool { + let expected = normalize_loose_artist(&primary_artist(expected)); + if expected.is_empty() { + return true; + } + let candidate = normalize_loose_artist(&primary_artist(candidate)); + let mut candidate_words: Vec<_> = candidate.split_whitespace().collect(); + let mut expected_words: Vec<_> = expected.split_whitespace().collect(); + candidate_words.sort_unstable(); + expected_words.sort_unstable(); + candidate_words == expected_words +} + +pub fn duration_matches(candidate: f64, expected: f64) -> bool { + candidate <= 0.0 || expected <= 0.0 || (candidate - expected).abs() <= 10.0 +} + +pub fn artist_in_title(candidate: &str, artist: &str) -> bool { + let expected = normalize_loose_artist(&primary_artist(artist)); + contains_words(&normalize_loose_artist(candidate), &expected) +} + +pub fn is_likely_instrumental(title: &str) -> bool { + INSTRUMENTAL.is_match(title.trim()) +} + +pub fn score( + candidate_title: &str, + candidate_artist: &str, + candidate_duration: f64, + title: &str, + artist: &str, + duration: f64, +) -> i32 { + let title = lowercase(simplify_track(title).trim()); + let candidate_title = lowercase(simplify_track(candidate_title).trim()); + let artist = lowercase(primary_artist(artist).trim()); + let candidate_artist = lowercase(primary_artist(candidate_artist).trim()); + let component = |candidate: &str, expected: &str, weight| { + if candidate == expected { + weight + } else if candidate.contains(expected) || expected.contains(candidate) { + weight / 2 + } else { + 0 + } + }; + component(&candidate_title, &title, 50) + + component(&candidate_artist, &artist, 60) + + if duration > 0.0 + && candidate_duration > 0.0 + && (candidate_duration - duration).abs() <= 10.0 + { + 20 + } else { + 0 + } +} + +pub fn clock_duration(value: &str) -> f64 { + let mut total = 0_isize; + if value.trim().is_empty() { + return 0.0; + } + for part in value.trim().split(':') { + let Ok(value) = part.trim().parse::<isize>() else { + return 0.0; + }; + total = total.wrapping_mul(60).wrapping_add(value); + } + total as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn simplify_and_primary_artist_keep_go_order_and_offsets() { + assert_eq!(simplify_track("Song (feat. Guest) (Live)"), "song"); + assert_eq!( + primary_artist("First with Guest, Second"), + "First with Guest" + ); + // '&' wins before 'with'; Go applies its lowercased byte offset to the + // original string even when lowercasing changes the UTF-8 width. + assert_eq!( + primary_artist("İ Artist with Guest & Other"), + "İ Artist with Gues" + ); + } +} diff --git a/rust_backend/crates/core/src/lyrics/models.rs b/rust_backend/crates/core/src/lyrics/models.rs new file mode 100644 index 00000000..5db7ba12 --- /dev/null +++ b/rust_backend/crates/core/src/lyrics/models.rs @@ -0,0 +1,192 @@ +//! Typed built-in search schemas, retaining Go field folding and null behavior. + +use super::json::go_deserialize; +use std::collections::BTreeMap; + +#[derive(Clone, Debug, Default)] +pub struct Artist { + pub name: String, +} +go_deserialize!(Artist { "name" => name, }); + +#[derive(Clone, Debug, Default)] +pub struct SpotifySong { + pub track_id: String, + pub name: String, + pub artist_name: String, + pub duration: String, +} +go_deserialize!(SpotifySong { "trackid" => track_id, "name" => name, "artistname" => artist_name, "duration" => duration, }); + +#[derive(Clone, Debug, Default)] +pub struct YouTubeSong { + pub video_id: String, + pub title: String, + pub author: String, + pub duration: String, +} +go_deserialize!(YouTubeSong { "videoid" => video_id, "title" => title, "author" => author, "duration" => duration, }); + +#[derive(Clone, Debug, Default)] +pub struct NeteaseSong { + pub name: String, + pub id: i64, + pub artists: Option<Vec<Artist>>, +} +go_deserialize!(NeteaseSong { "name" => name, "id" => id, "artists" => artists, }); +#[derive(Clone, Debug, Default)] +pub struct NeteaseResults { + pub songs: Option<Vec<NeteaseSong>>, + pub song_count: isize, +} +go_deserialize!(NeteaseResults { "songs" => songs, "songcount" => song_count, }); +#[derive(Clone, Debug, Default)] +pub struct NeteaseSearch { + pub result: NeteaseResults, + pub code: isize, + pub message: String, + pub msg: String, +} +go_deserialize!(NeteaseSearch { "result" => result, "code" => code, "message" => message, "msg" => msg, }); +#[derive(Clone, Debug, Default)] +pub struct LyricField { + pub lyric: String, +} +go_deserialize!(LyricField { "lyric" => lyric, }); +#[derive(Clone, Debug, Default)] +pub struct NeteaseLyrics { + pub lrc: Option<LyricField>, + pub tlyric: Option<LyricField>, + pub romalrc: Option<LyricField>, + pub code: isize, +} +go_deserialize!(NeteaseLyrics { "lrc" => lrc, "tlyric" => tlyric, "romalrc" => romalrc, "code" => code, }); + +#[derive(Clone, Debug, Default)] +pub struct AppleSongId { + pub id: String, +} +go_deserialize!(AppleSongId { "id" => id, }); +#[derive(Clone, Debug, Default)] +pub struct AppleSongs { + pub data: Option<Vec<AppleSongId>>, +} +go_deserialize!(AppleSongs { "data" => data, }); +#[derive(Clone, Debug, Default)] +pub struct AppleResults { + pub songs: Option<AppleSongs>, +} +go_deserialize!(AppleResults { "songs" => songs, }); +#[derive(Clone, Debug, Default)] +pub struct AppleAttributes { + pub name: String, + pub artist_name: String, + pub album_name: String, + pub duration_in_millis: isize, +} +go_deserialize!(AppleAttributes { "name" => name, "artistname" => artist_name, "albumname" => album_name, "durationinmillis" => duration_in_millis, }); +#[derive(Clone, Debug, Default)] +pub struct AppleSong { + pub attributes: AppleAttributes, +} +go_deserialize!(AppleSong { "attributes" => attributes, }); +#[derive(Clone, Debug, Default)] +pub struct AppleResources { + pub songs: Option<BTreeMap<String, AppleSong>>, +} +go_deserialize!(AppleResources { "songs" => songs, }); +#[derive(Clone, Debug, Default)] +pub struct AppleSearch { + pub results: AppleResults, + pub resources: Option<AppleResources>, +} +go_deserialize!(AppleSearch { "results" => results, "resources" => resources, }); + +#[derive(Clone, Debug, Default)] +pub struct QqSong { + pub mid: String, + pub id: i64, + pub name: String, + pub interval: isize, + pub singer: Option<Vec<Artist>>, +} +go_deserialize!(QqSong { "mid" => mid, "id" => id, "name" => name, "interval" => interval, "singer" => singer, }); +#[derive(Clone, Debug, Default)] +pub struct QqSongs { + pub list: Option<Vec<QqSong>>, +} +go_deserialize!(QqSongs { "list" => list, }); +#[derive(Clone, Debug, Default)] +pub struct QqData { + pub song: QqSongs, +} +go_deserialize!(QqData { "song" => song, }); +#[derive(Clone, Debug, Default)] +pub struct QqSearch { + pub code: isize, + pub data: QqData, +} +go_deserialize!(QqSearch { "code" => code, "data" => data, }); +#[derive(Clone, Debug, Default)] +pub struct QqLyrics { + pub retcode: isize, + pub code: isize, + pub lyric: String, +} +go_deserialize!(QqLyrics { "retcode" => retcode, "code" => code, "lyric" => lyric, }); + +#[derive(Clone, Debug, Default)] +pub struct KugouSong { + pub id: String, + pub accesskey: String, + pub song: String, + pub singer: String, + pub duration: f64, +} +go_deserialize!(KugouSong { "id" => id, "accesskey" => accesskey, "song" => song, "singer" => singer, "duration" => duration, }); +#[derive(Clone, Debug, Default)] +pub struct KugouSearch { + pub status: isize, + pub errcode: isize, + pub errmsg: String, + pub candidates: Option<Vec<KugouSong>>, +} +go_deserialize!(KugouSearch { "status" => status, "errcode" => errcode, "errmsg" => errmsg, "candidates" => candidates, }); +#[derive(Clone, Debug, Default)] +pub struct KugouLyrics { + pub status: isize, + pub error_code: isize, + pub info: String, + pub content: String, +} +go_deserialize!(KugouLyrics { "status" => status, "error_code" => error_code, "info" => info, "content" => content, }); + +#[derive(Clone, Debug, Default)] +pub struct GeniusSong { + pub title: String, + pub artist_names: String, + pub primary_artist_names: String, + pub url: String, +} +go_deserialize!(GeniusSong { "title" => title, "artist_names" => artist_names, "primary_artist_names" => primary_artist_names, "url" => url, }); +#[derive(Clone, Debug, Default)] +pub struct GeniusHit { + pub kind: String, + pub result: GeniusSong, +} +go_deserialize!(GeniusHit { "type" => kind, "result" => result, }); +#[derive(Clone, Debug, Default)] +pub struct GeniusSection { + pub hits: Option<Vec<GeniusHit>>, +} +go_deserialize!(GeniusSection { "hits" => hits, }); +#[derive(Clone, Debug, Default)] +pub struct GeniusResults { + pub sections: Option<Vec<GeniusSection>>, +} +go_deserialize!(GeniusResults { "sections" => sections, }); +#[derive(Clone, Debug, Default)] +pub struct GeniusSearch { + pub response: GeniusResults, +} +go_deserialize!(GeniusSearch { "response" => response, }); diff --git a/rust_backend/crates/core/src/lyrics/payloads.rs b/rust_backend/crates/core/src/lyrics/payloads.rs new file mode 100644 index 00000000..e4081ee5 --- /dev/null +++ b/rust_backend/crates/core/src/lyrics/payloads.rs @@ -0,0 +1,312 @@ +use super::{LyricsResponse, json, lrc}; +use serde::Serialize; +use serde_json::value::RawValue; +use std::collections::BTreeMap; + +#[derive(Clone, Debug, Default, Serialize)] +pub struct PaxDetail { + pub text: String, + pub part: bool, + pub timestamp: Option<isize>, + pub endtime: Option<isize>, +} +json::go_deserialize!(PaxDetail { + "text" => text, "part" => part, "timestamp" => timestamp, "endtime" => endtime, +}); + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PaxLine { + pub text: Option<Vec<PaxDetail>>, + pub timestamp: isize, + pub opposite_turn: bool, + pub background: bool, + pub background_text: Option<Vec<PaxDetail>>, + pub endtime: isize, +} +json::go_deserialize!(PaxLine { + "text" => text, "timestamp" => timestamp, "oppositeturn" => opposite_turn, + "background" => background, "backgroundtext" => background_text, "endtime" => endtime, +}); + +#[derive(Default)] +struct ApplePayload { + kind: String, + content: Option<Vec<PaxLine>>, + elrc: String, + elrc_multi_person: String, + plain: String, + ttml_content: String, +} +json::go_deserialize!(ApplePayload { + "type" => kind, "content" => content, "elrc" => elrc, + "elrcmultiperson" => elrc_multi_person, "plain" => plain, "ttmlcontent" => ttml_content, +}); + +#[derive(Default)] +struct ProxyPayload { + kind: String, + content: Option<Vec<PaxLine>>, + lyrics: Option<Vec<PaxLine>>, + lyrics_text: String, + plain_lyrics: String, +} +json::go_deserialize!(ProxyPayload { + "type" => kind, "content" => content, "lyrics" => lyrics, + "lyrics_text" => lyrics_text, "plain_lyrics" => plain_lyrics, +}); + +fn append_detail(output: &mut String, details: &[PaxDetail], word_timing: bool) { + let mut last_start = String::new(); + for detail in details { + if word_timing && let Some(time) = detail.timestamp { + let start = format!("<{}>", lrc::timestamp_inline(time as i64)); + if start != last_start { + output.push_str(&start); + last_start = start; + } + } + output.push_str(&detail.text); + if !detail.part { + output.push(' '); + } + if word_timing && let Some(time) = detail.endtime { + output.push_str(&format!("<{}>", lrc::timestamp_inline(time as i64))); + } + } +} + +pub fn format_pax_content( + kind: &str, + lines: &[PaxLine], + multi_person: bool, + word_timing: bool, +) -> String { + let mut output = String::new(); + for (index, line) in lines.iter().enumerate() { + if index > 0 { + output.push('\n'); + } + if json::field_name(kind) == "syllable" { + output.push_str(&lrc::timestamp(line.timestamp as i64)); + if multi_person { + output.push_str(if line.opposite_turn { "v2:" } else { "v1:" }); + } + append_detail( + &mut output, + line.text.as_deref().unwrap_or_default(), + word_timing, + ); + if line.background + && multi_person + && let Some(background) = line + .background_text + .as_ref() + .filter(|lines| !lines.is_empty()) + { + output.push_str("\n[bg:"); + append_detail(&mut output, background, word_timing); + output.push(']'); + } + } else if let Some(text) = line.text.as_ref().and_then(|text| text.first()) { + output.push_str(&lrc::timestamp(line.timestamp as i64)); + output.push_str(&text.text); + } + } + output.trim().into() +} + +pub fn format_apple(raw: &str, multi_person: bool, word_timing: bool) -> Result<String, String> { + if let Ok(Some(value)) = json::decode::<Option<String>>(raw) + && !value.trim().is_empty() + { + return Ok(value.trim().into()); + } + if let Ok(value) = json::decode::<ApplePayload>(raw) + && (value.content.is_some() + || [ + &value.elrc_multi_person, + &value.elrc, + &value.plain, + &value.ttml_content, + ] + .iter() + .any(|value| !value.trim().is_empty())) + { + if word_timing && multi_person && !value.elrc_multi_person.trim().is_empty() { + return Ok(value.elrc_multi_person.trim().into()); + } + if word_timing && !value.elrc.trim().is_empty() { + return Ok(value.elrc.trim().into()); + } + let content = value.content.as_deref().unwrap_or_default(); + if !value.plain.trim().is_empty() && content.is_empty() { + return Ok(value.plain.trim().into()); + } + if content.is_empty() { + return Err("unsupported apple music lyrics payload".into()); + } + return Ok(format_pax_content( + &value.kind, + content, + multi_person, + word_timing, + )); + } + if let Ok(Some(lines)) = json::decode::<Option<Vec<PaxLine>>>(raw) + && !lines.is_empty() + { + return Ok(format_pax_content( + "Syllable", + &lines, + multi_person, + word_timing, + )); + } + Err("failed to parse pax lyrics response".into()) +} + +/// Failure strings describe unavailable proxy payloads, not an absent track. +pub fn parse_proxy( + raw: &str, + provider: &str, + multi_person: bool, +) -> Result<LyricsResponse, String> { + if let Ok(value) = json::decode::<Option<String>>(raw) { + let value = value.unwrap_or_default(); + if value.trim().is_empty() { + return Err(format!("{provider} returned empty lyrics")); + } + return Ok(LyricsResponse::from_text(value.trim(), provider, provider)); + } + // Go checks these aliases through json.RawMessage. An unused number may + // exceed float64's range without invalidating an otherwise usable payload. + if let Ok(value) = json::decode::<BTreeMap<String, Box<RawValue>>>(raw) { + for key in ["lyrics", "lyric", "lyrics_text", "plain_lyrics"] { + if let Some(text) = value + .get(key) + .and_then(|value| json::decode::<Option<String>>(value.get()).ok().flatten()) + .filter(|value| !value.trim().is_empty()) + { + return Ok(LyricsResponse::from_text(text.trim(), provider, provider)); + } + } + } + if let Ok(value) = json::decode::<ProxyPayload>(raw) { + let content = value.content.as_deref().unwrap_or_default(); + let lyrics = value.lyrics.as_deref().unwrap_or_default(); + let text = if !value.lyrics_text.trim().is_empty() { + Some(value.lyrics_text) + } else if !lyrics.is_empty() { + Some(format_pax_content("Syllable", lyrics, multi_person, true)) + } else if !content.is_empty() { + Some(format_pax_content( + if value.kind.is_empty() { + "Syllable" + } else { + &value.kind + }, + content, + multi_person, + true, + )) + } else if !value.plain_lyrics.trim().is_empty() { + Some(value.plain_lyrics) + } else { + None + }; + if let Some(text) = text { + return Ok(LyricsResponse::from_text(&text, provider, provider)); + } + } + let raw = raw.trim(); + if !raw.is_empty() && !raw.starts_with(['{', '[']) { + return Ok(LyricsResponse::from_text(raw, provider, provider)); + } + if json::decode::<Box<RawValue>>(raw).is_ok() { + return Err(format!( + "{provider} returned a response without usable lyrics" + )); + } + Err(format!("failed to decode {provider} lyrics response")) +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct KpoeSyllable { + pub text: String, + pub time: f64, + pub duration: f64, + pub is_background: bool, +} +json::go_deserialize!(KpoeSyllable { + "text" => text, "time" => time, "duration" => duration, "isbackground" => is_background, +}); + +#[derive(Clone, Debug, Default, Serialize)] +pub struct KpoeLine { + pub time: f64, + pub duration: f64, + pub text: String, + pub syllabus: Option<Vec<KpoeSyllable>>, +} +json::go_deserialize!(KpoeLine { + "time" => time, "duration" => duration, "text" => text, "syllabus" => syllabus, +}); + +#[derive(Clone, Debug, Default, Serialize)] +pub struct KpoeResponse { + #[serde(rename = "type")] + pub kind: String, + pub lyrics: Option<Vec<KpoeLine>>, +} +json::go_deserialize!(KpoeResponse { "type" => kind, "lyrics" => lyrics, }); + +pub fn format_kpoe(response: &KpoeResponse, multi_person: bool, word_timing: bool) -> String { + let words = matches!( + json::field_name(&response.kind).as_str(), + "word" | "syllable" + ); + let mut lines = Vec::new(); + let append = |details: &[&KpoeSyllable]| { + let mut result = String::new(); + for detail in details { + result.push_str(&format!( + "<{}>{}", + lrc::timestamp_inline(detail.time as i64), + detail.text + )); + } + result + }; + for line in response.lyrics.as_deref().unwrap_or_default() { + let syllabus = line.syllabus.as_deref().unwrap_or_default(); + let timestamp = lrc::timestamp(line.time as i64); + if words && word_timing && !syllabus.is_empty() { + let (mut main, mut background): (Vec<_>, Vec<_>) = + syllabus.iter().partition(|detail| !detail.is_background); + if main.is_empty() { + main = syllabus.iter().collect(); + background.clear(); + } + let mut output = format!("{timestamp}{}", append(&main)); + if multi_person && !background.is_empty() { + output.push_str(&format!("\n[bg:{}]", append(&background))); + } + lines.push(output); + } else { + let text = if line.text.trim().is_empty() && !syllabus.is_empty() { + syllabus + .iter() + .map(|detail| detail.text.as_str()) + .collect::<String>() + } else { + line.text.clone() + }; + if !text.trim().is_empty() { + lines.push(format!("{timestamp}{}", text.trim())); + } + } + } + lines.join("\n").trim().into() +} diff --git a/rust_backend/crates/core/src/matching.rs b/rust_backend/crates/core/src/matching.rs new file mode 100644 index 00000000..b310fb88 --- /dev/null +++ b/rust_backend/crates/core/src/matching.rs @@ -0,0 +1,147 @@ +//! Matching compatibility uses Go's simple case mapping and UTF-8 byte distance. + +/// Unlike String::to_lowercase, Go's unicode.ToLower maps one rune to one rune. +pub fn lowercase(value: &str) -> String { + value + .chars() + .map(|ch| match ch { + // Go 1.26 uses Unicode 15.0; these lowercase mappings were added + // in Unicode 16/17, used by the pinned Rust toolchain. + '\u{1c89}' + | '\u{a7cb}' + | '\u{a7cc}' + | '\u{a7ce}' + | '\u{a7d2}' + | '\u{a7d4}' + | '\u{a7da}' + | '\u{a7dc}' + | '\u{10d50}'..='\u{10d65}' + | '\u{16ea0}'..='\u{16eb8}' => ch, + _ => ch.to_lowercase().next().expect("lowercase character"), + }) + .collect() +} + +/// Go's Unicode 15 simple uppercase does not expand characters such as ß. +pub fn uppercase(value: &str) -> String { + value + .chars() + .map(|ch| match ch { + '\u{019b}' + | '\u{0264}' + | '\u{1c8a}' + | '\u{a7cd}' + | '\u{a7cf}' + | '\u{a7d3}' + | '\u{a7d5}' + | '\u{a7db}' + | '\u{10d70}'..='\u{10d85}' + | '\u{16ebb}'..='\u{16ed3}' => ch, + // Full uppercase expands these Greek letters; simple uppercase uses + // the corresponding single precomposed capital instead. + '\u{1f80}'..='\u{1f87}' | '\u{1f90}'..='\u{1f97}' | '\u{1fa0}'..='\u{1fa7}' => { + char::from_u32(ch as u32 + 8).expect("Greek capital") + } + '\u{1fb3}' => '\u{1fbc}', + '\u{1fc3}' => '\u{1fcc}', + '\u{1ff3}' => '\u{1ffc}', + _ => { + let mut mapped = ch.to_uppercase(); + let first = mapped.next().expect("uppercase character"); + if mapped.next().is_none() { first } else { ch } + } + }) + .collect() +} + +pub fn normalize(value: &str) -> String { + let mut value = lowercase(value); + for suffix in [ + " (remastered)", + " (remaster)", + " - remastered", + " - remaster", + " (deluxe)", + " (deluxe edition)", + " - deluxe", + " - deluxe edition", + " (explicit)", + " (clean)", + " [explicit]", + " [clean]", + " (album version)", + " (single version)", + " (radio edit)", + " (feat.", + " (ft.", + " feat.", + " ft.", + ] { + if let Some(index) = value.find(suffix) { + value.truncate(index); + } + } + let filtered: String = value + .chars() + .filter(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || *ch == ' ') + .collect(); + filtered.split_whitespace().collect::<Vec<_>>().join(" ") +} + +pub fn compare_strings( + first: &str, + second: &str, + check: &dyn Fn() -> Result<(), String>, +) -> Result<f64, String> { + check()?; + let first = lowercase(first.trim()); + let second = lowercase(second.trim()); + if first == second { + return Ok(1.0); + } + let maximum = first.len().max(second.len()); + if first.is_empty() || second.is_empty() { + return Ok(0.0); + } + let mut first = first.as_bytes(); + let mut second = second.as_bytes(); + let prefix = first.iter().zip(second).take_while(|(a, b)| a == b).count(); + first = &first[prefix..]; + second = &second[prefix..]; + let suffix = first + .iter() + .rev() + .zip(second.iter().rev()) + .take_while(|(a, b)| a == b) + .count(); + first = &first[..first.len() - suffix]; + second = &second[..second.len() - suffix]; + if first.len() < second.len() { + std::mem::swap(&mut first, &mut second); + } + let mut row: Vec<_> = (0..=second.len()).collect(); + for (i, a) in first.iter().enumerate() { + check()?; + let mut diagonal = row[0]; + row[0] = i + 1; + for (j, b) in second.iter().enumerate() { + if j % 4096 == 0 { + check()?; + } + let old = row[j + 1]; + row[j + 1] = (old + 1) + .min(row[j] + 1) + .min(diagonal + usize::from(a != b)); + diagonal = old; + } + } + Ok(1.0 - row[second.len()] as f64 / maximum as f64) +} + +/// Go's `int` is native-width and arithmetic wraps, including abs(MIN). +pub fn compare_duration(first: i64, second: i64, tolerance: i64) -> bool { + let difference = (first as isize) + .wrapping_sub(second as isize) + .wrapping_abs(); + difference <= tolerance as isize +} diff --git a/rust_backend/crates/core/src/media/audio.rs b/rust_backend/crates/core/src/media/audio.rs new file mode 100644 index 00000000..113b639f --- /dev/null +++ b/rust_backend/crates/core/src/media/audio.rs @@ -0,0 +1,292 @@ +//! Quality probes used by the file metadata export. Callers supply a checked +//! reader; large audio payloads are skipped instead of buffered. + +use super::AudioQuality; +use crate::tags::{ogg_packets, ogg_stream_is_opus}; +use std::io::{BufReader, Read, Seek, SeekFrom}; + +pub(crate) fn mp3_quality( + file: &mut (impl Read + Seek), + size: i64, +) -> Result<AudioQuality, String> { + let mut quality = AudioQuality::default(); + let header = read::<10>(file)?; + let start = if &header[..3] == b"ID3" { + 10 + ((i64::from(header[6]) << 21) + | (i64::from(header[7]) << 14) + | (i64::from(header[8]) << 7) + | i64::from(header[9])) + } else { + 0 + }; + seek(file, SeekFrom::Start(start as u64))?; + let Ok(header) = read::<4>(file) else { + return Ok(quality); + }; + let frame = if header[0] == 0xff && header[1] & 0xe0 == 0xe0 { + Some((header, seek(file, SeekFrom::Current(0))? - 4)) + } else { + // Allocate only for resynchronization; normal MP3s need no read-ahead. + let mut buffered = BufReader::new(&mut *file); + let mut frame = None; + for _ in 1..10000 { + buffered + .seek_relative(-3) + .map_err(|error| error.to_string())?; + let Ok(header) = read::<4>(&mut buffered) else { + break; + }; + if header[0] == 0xff && header[1] & 0xe0 == 0xe0 { + frame = Some((header, seek(&mut buffered, SeekFrom::Current(0))? - 4)); + break; + } + } + frame + }; + let Some((header, frame_start)) = frame else { + return Ok(quality); + }; + let version = (header[1] >> 3) & 3; + let layer = (header[1] >> 1) & 3; + let bitrate_index = (header[2] >> 4) as usize; + let rate_index = ((header[2] >> 2) & 3) as usize; + let mono = (header[3] >> 6) == 3; + let rates = [ + [11025, 12000, 8000], + [0, 0, 0], + [22050, 24000, 16000], + [44100, 48000, 32000], + ]; + if rate_index < 3 { + quality.sample_rate = rates[version as usize][rate_index]; + } + if layer == 1 { + let rates = if version == 3 { + [ + 0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0, + ] + } else if version == 0 || version == 2 { + [ + 0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0, + ] + } else { + [0; 16] + }; + quality.bitrate = rates[bitrate_index] * 1000; + } + let samples_per_frame = if version == 0 || version == 2 { + 576 + } else { + 1152 + }; + let xing_offset = match (version == 3, mono) { + (true, false) => 32, + (false, true) => 9, + _ => 17, + }; + seek(file, SeekFrom::Start(frame_start + 4))?; + let mut xing = [0; 200]; + let mut length = 0; + while length < xing.len() { + match file.read(&mut xing[length..]) { + Ok(0) | Err(_) => break, + Ok(n) => length += n, + } + } + let mut frames = 0; + let mut bytes = 0; + if xing_offset + 8 <= length && matches!(&xing[xing_offset..xing_offset + 4], b"Xing" | b"Info") + { + let flags = be32(&xing[xing_offset + 4..xing_offset + 8]); + let mut offset = xing_offset + 8; + if flags & 1 != 0 && offset + 4 <= length { + frames = be32(&xing[offset..offset + 4]); + offset += 4; + } + if flags & 2 != 0 && offset + 4 <= length { + bytes = be32(&xing[offset..offset + 4]); + } + } + if frames == 0 && length >= 62 && &xing[32..36] == b"VBRI" { + bytes = be32(&xing[42..46]); + frames = be32(&xing[46..50]); + } + if frames > 0 && quality.sample_rate > 0 { + quality.duration = frames * samples_per_frame / quality.sample_rate; + if quality.duration > 0 { + quality.bitrate = if bytes > 0 { bytes } else { size - start } * 8 / quality.duration; + } + } else if quality.bitrate > 0 && size - start - 128 > 0 { + quality.duration = (size - start - 128) * 8 / quality.bitrate; + } + Ok(quality) +} + +pub(crate) fn ogg_quality( + file: &mut (impl Read + Seek), + size: i64, + path: &str, +) -> Result<AudioQuality, String> { + let packets = ogg_packets(file, 5, 10)?; + let opus = + ogg_stream_is_opus(&packets).unwrap_or_else(|| path.to_lowercase().ends_with(".opus")); + let mut quality = AudioQuality::default(); + let mut pre_skip = 0; + for packet in packets { + if opus && packet.len() >= 19 && packet.starts_with(b"OpusHead") { + quality.sample_rate = le32(&packet[12..16]); + if quality.sample_rate == 0 { + quality.sample_rate = 48000; + } + pre_skip = i64::from(u16::from_le_bytes(packet[10..12].try_into().unwrap())); + break; + } + if !opus && packet.len() > 29 && packet.starts_with(b"\x01vorbis") { + quality.sample_rate = le32(&packet[12..16]); + break; + } + } + let length = size.clamp(0, 65536) as usize; + seek(file, SeekFrom::Start((size - length as i64) as u64))?; + let mut tail = vec![0; length]; + file.read_exact(&mut tail).map_err(|e| e.to_string())?; + let granule = (0..length.saturating_sub(3)) + .rev() + .find_map(|offset| { + let data = &tail[offset..]; + if data.len() < 27 || &data[..4] != b"OggS" || data[4] != 0 || data[5] > 7 { + return None; + } + let header_length = 27 + data[26] as usize; + if data.len() < header_length { + return None; + } + let payload: usize = data[27..header_length].iter().map(|v| *v as usize).sum(); + (data.len() >= header_length + payload) + .then(|| u64::from_le_bytes(data[6..14].try_into().unwrap()) as i64) + }) + .unwrap_or(0); + let seconds = if granule <= 0 { + 0.0 + } else if opus { + (granule - pre_skip) as f64 / 48000.0 + } else if quality.sample_rate > 0 { + granule as f64 / quality.sample_rate as f64 + } else { + 0.0 + }; + if seconds > 0.0 { + quality.duration = seconds.round() as i64; + quality.bitrate = (size as f64 * 8.0 / seconds) as i64; + } + if quality.bitrate <= 0 && quality.duration > 0 { + quality.bitrate = size * 8 / quality.duration; + } + if quality.duration > 86400 { + quality.duration = 0; + quality.bitrate = 0; + } + if quality.bitrate > 0 && quality.bitrate < 8000 { + quality.bitrate = 0; + } + Ok(quality) +} + +pub(crate) fn riff_quality( + file: &mut (impl Read + Seek), + size: i64, + aiff: bool, +) -> Result<AudioQuality, String> { + let header = read::<12>(file)?; + if if aiff { + &header[..4] != b"FORM" || !matches!(&header[8..], b"AIFF" | b"AIFC") + } else { + &header[..4] != b"RIFF" || &header[8..] != b"WAVE" + } { + return Err("invalid audio container".into()); + } + let mut quality = AudioQuality::default(); + let (mut channels, mut byte_rate, mut data_size, mut frames) = (0, 0, 0, 0); + while let Ok(header) = read::<8>(file) { + let length = if aiff { + be32(&header[4..]) + } else { + le32(&header[4..]) + }; + let start = seek(file, SeekFrom::Current(0))?; + if (!aiff && &header[..4] == b"fmt ") || (aiff && &header[..4] == b"COMM") { + // Go returns the previous probe on a truncated format chunk. + if start + length as u64 > size as u64 { + break; + } + let mut data = [0; 26]; + let count = length.min(data.len() as i64) as usize; + if file.read_exact(&mut data[..count]).is_err() { + break; + } + if aiff && length >= 18 { + channels = i64::from(u16::from_be_bytes(data[..2].try_into().unwrap())); + frames = be32(&data[2..6]); + quality.bit_depth = i64::from(u16::from_be_bytes(data[6..8].try_into().unwrap())); + let exponent = (i32::from(data[8] & 0x7f) << 8) | i32::from(data[9]); + let mantissa = u64::from_be_bytes(data[10..18].try_into().unwrap()); + let sign = if data[8] & 0x80 == 0 { 1.0 } else { -1.0 }; + quality.sample_rate = + (sign * mantissa as f64 * 2.0_f64.powi(exponent - 16446) + 0.5) as i64; + } else if !aiff && length >= 16 { + channels = i64::from(u16::from_le_bytes(data[2..4].try_into().unwrap())); + quality.sample_rate = le32(&data[4..8]); + byte_rate = le32(&data[8..12]); + quality.bit_depth = i64::from(u16::from_le_bytes(data[14..16].try_into().unwrap())); + if data[..2] == [0xfe, 0xff] && length >= 26 { + let valid_bits = + i64::from(u16::from_le_bytes(data[18..20].try_into().unwrap())); + if valid_bits > 0 { + quality.bit_depth = valid_bits; + } + } + } + } else if !aiff && &header[..4] == b"data" { + data_size = length; + } + seek( + file, + SeekFrom::Start(start + length as u64 + (length & 1) as u64), + )?; + } + if aiff { + if quality.sample_rate > 0 && frames > 0 { + quality.duration = frames / quality.sample_rate; + } + } else if byte_rate > 0 && data_size > 0 { + quality.duration = data_size / byte_rate; + } else if quality.sample_rate > 0 && channels > 0 && quality.bit_depth > 0 && data_size > 0 { + let bytes_per_second = quality + .sample_rate + .wrapping_mul(channels) + .wrapping_mul(quality.bit_depth) + / 8; + if bytes_per_second > 0 { + quality.duration = data_size / bytes_per_second; + } + } + Ok(quality) +} + +fn read<const N: usize>(file: &mut impl Read) -> Result<[u8; N], String> { + let mut data = [0; N]; + file.read_exact(&mut data).map_err(|e| e.to_string())?; + Ok(data) +} + +fn seek(file: &mut impl Seek, position: SeekFrom) -> Result<u64, String> { + file.seek(position).map_err(|e| e.to_string()) +} + +fn be32(data: &[u8]) -> i64 { + i64::from(u32::from_be_bytes(data.try_into().unwrap())) +} + +fn le32(data: &[u8]) -> i64 { + i64::from(u32::from_le_bytes(data.try_into().unwrap())) +} diff --git a/rust_backend/crates/core/src/media/mod.rs b/rust_backend/crates/core/src/media/mod.rs new file mode 100644 index 00000000..66b65404 --- /dev/null +++ b/rust_backend/crates/core/src/media/mod.rs @@ -0,0 +1,95 @@ +//! Bounded, seek-based implementation of Go's FLAC/MP4 quality probe. + +mod audio; +pub(crate) mod mp4; + +pub(crate) use audio::{mp3_quality, ogg_quality, riff_quality}; + +use serde::Serialize; +use std::io::{Read, Seek, SeekFrom}; + +#[derive(Debug, Default, PartialEq, Serialize)] +pub struct AudioQuality { + pub bit_depth: i64, + pub sample_rate: i64, + pub total_samples: i64, + pub duration: i64, + #[serde(skip_serializing_if = "is_zero")] + pub bitrate: i64, + #[serde(skip_serializing_if = "String::is_empty")] + pub codec: String, +} + +fn is_zero(value: &i64) -> bool { + *value == 0 +} + +pub fn probe_quality( + file: &mut (impl Read + Seek), + check: &dyn Fn() -> Result<(), String>, +) -> Result<AudioQuality, String> { + check()?; + file.seek(SeekFrom::Start(0)).map_err(|e| e.to_string())?; + let marker = read_go::<4>(file).map_err(|e| format!("failed to read marker: {e}"))?; + if &marker == b"fLaC" { + let header = read_go::<4>(file).map_err(|e| format!("failed to read header: {e}"))?; + if header[0] & 0x7f != 0 { + return Err("first block is not STREAMINFO".into()); + } + let info = read_go::<34>(file).map_err(|e| format!("failed to read STREAMINFO: {e}"))?; + return Ok(flac_quality(&info)); + } + file.seek(SeekFrom::Start(0)).map_err(|e| e.to_string())?; + let header = read_go::<8>(file).map_err(|e| format!("failed to read header: {e}"))?; + if &header[4..] == b"ftyp" { + return probe_mp4_quality(file, check); + } + Err("unsupported file format (not FLAC or M4A)".into()) +} + +/// MP4-family descriptor paths may lack an ftyp box, as in Go's GetM4AQuality. +pub fn probe_mp4_quality( + file: &mut (impl Read + Seek), + check: &dyn Fn() -> Result<(), String>, +) -> Result<AudioQuality, String> { + mp4::probe(file, check) +} + +fn flac_quality(info: &[u8; 34]) -> AudioQuality { + let packed = u64::from_be_bytes(info[10..18].try_into().expect("STREAMINFO fields")); + let rate = (packed >> 44) as i64; + let samples = (packed & ((1 << 36) - 1)) as i64; + AudioQuality { + bit_depth: ((packed >> 36) & 31) as i64 + 1, + sample_rate: rate, + total_samples: samples, + duration: if rate > 0 { samples / rate } else { 0 }, + codec: "flac".into(), + ..AudioQuality::default() + } +} + +// Go's marker/STREAMINFO reader accepts a short successful Read and leaves the +// rest of its zero-initialized buffer untouched. Preserve that legacy behavior. +fn read_go<const N: usize>(file: &mut impl Read) -> Result<[u8; N], String> { + let mut bytes = [0; N]; + match file.read(&mut bytes) { + Ok(0) => Err("EOF".into()), + Ok(_) => Ok(bytes), + Err(error) => Err(error.to_string()), + } +} + +fn read_at<const N: usize>(file: &mut (impl Read + Seek), offset: u64) -> Result<[u8; N], String> { + file.seek(SeekFrom::Start(offset)) + .map_err(|e| e.to_string())?; + let mut bytes = [0; N]; + file.read_exact(&mut bytes).map_err(|e| { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + "EOF".into() + } else { + e.to_string() + } + })?; + Ok(bytes) +} diff --git a/rust_backend/crates/core/src/media/mp4.rs b/rust_backend/crates/core/src/media/mp4.rs new file mode 100644 index 00000000..db18eb18 --- /dev/null +++ b/rust_backend/crates/core/src/media/mp4.rs @@ -0,0 +1,271 @@ +use super::{AudioQuality, flac_quality, read_at}; +use std::io::{Read, Seek, SeekFrom}; + +pub(crate) struct Reader<'a, R> { + pub(crate) file: &'a mut R, + pub(crate) size: u64, + pub(crate) check: &'a dyn Fn() -> Result<(), String>, +} + +#[derive(Clone, Copy)] +pub(crate) struct Atom { + pub(crate) start: u64, + pub(crate) end: u64, + pub(crate) payload: u64, + pub(crate) kind: [u8; 4], +} + +pub(super) fn probe( + file: &mut (impl Read + Seek), + check: &dyn Fn() -> Result<(), String>, +) -> Result<AudioQuality, String> { + let size = file.seek(SeekFrom::End(0)).map_err(|e| e.to_string())?; + let mut reader = Reader { file, size, check }; + let moov = reader + .find(0, size, b"moov") + .map_err(|e| format!("failed to find moov atom: {e}"))? + .ok_or("moov atom not found")?; + let mut duration = reader + .find(moov.payload, moov.end, b"mvhd") + .ok() + .flatten() + .map_or(0, |atom| reader.duration(atom).unwrap_or(0)); + if duration <= 0 { + let _ = reader.track_duration(moov.payload, moov.end, 0, &mut duration); + } + let (offset, kind) = reader.sample_entry(moov)?; + let bytes = read_at::<32>(reader.file, offset) + .map_err(|e| format!("failed to read audio sample entry: {e}"))?; + let mut rate = u16::from_be_bytes(bytes[28..30].try_into().unwrap()) as i64; + let mut depth = 0; + if &kind == b"alac" || &kind == b"fLaC" { + depth = u16::from_be_bytes(bytes[22..24].try_into().unwrap()) as i64; + if let Some((bits, hz, samples)) = reader.specific_config(offset, &kind) { + if bits > 0 { + depth = bits; + } + if hz > 0 { + rate = hz; + } + if samples > 0 && rate > 0 && duration <= 0 { + duration = samples / rate; + } + } + } + (reader.check)()?; + let bitrate = if duration > 0 { + (size as f64 * 8.0 / duration as f64 / 1000.0).round() as i64 + } else { + 0 + }; + Ok(AudioQuality { + bit_depth: depth, + sample_rate: rate, + duration, + bitrate: if bitrate < 16 { 0 } else { bitrate }, + codec: match &kind { + b"mp4a" => "aac", + b"alac" => "alac", + b"fLaC" => "flac", + b"ec-3" => "eac3", + b"ac-3" => "ac3", + b"ac-4" => "ac4", + b"Opus" => "opus", + _ => unreachable!("recognized sample entry"), + } + .into(), + // Go leaves total_samples zero for MP4, even when dfLa contains it. + total_samples: 0, + }) +} + +impl<R: Read + Seek> Reader<'_, R> { + pub(crate) fn atom(&mut self, offset: u64, end: u64) -> Result<Atom, String> { + (self.check)()?; + if offset.checked_add(8).is_none_or(|value| value > self.size) { + return Err("unexpected EOF".into()); + } + let bytes = read_at::<8>(self.file, offset)?; + let kind: [u8; 4] = bytes[4..8].try_into().unwrap(); + let mut size = u32::from_be_bytes(bytes[..4].try_into().unwrap()) as u64; + let mut header = 8; + if size == 1 { + if offset.checked_add(16).is_none_or(|value| value > self.size) { + return Err("unexpected EOF".into()); + } + size = u64::from_be_bytes(read_at::<8>(self.file, offset + 8)?); + header = 16; + } else if size == 0 { + size = end.saturating_sub(offset); + } + if size < header || offset.checked_add(size).is_none() || size > i64::MAX as u64 { + return Err(format!( + "invalid atom size for {}", + String::from_utf8_lossy(&kind) + )); + } + Ok(Atom { + start: offset, + end: offset + size, + payload: offset + header, + kind, + }) + } + + fn find(&mut self, mut start: u64, end: u64, kind: &[u8; 4]) -> Result<Option<Atom>, String> { + while start.checked_add(8).is_some_and(|value| value <= end) { + let atom = self.atom(start, end)?; + if &atom.kind == kind { + return Ok(Some(atom)); + } + start = atom.end; + } + Ok(None) + } + + fn duration(&mut self, atom: Atom) -> Result<i64, String> { + let version = read_at::<1>(self.file, atom.payload)?[0]; + let (scale, duration) = if version == 1 { + let bytes = read_at::<32>(self.file, atom.payload)?; + ( + u32::from_be_bytes(bytes[20..24].try_into().unwrap()), + u64::from_be_bytes(bytes[24..32].try_into().unwrap()), + ) + } else { + let bytes = read_at::<20>(self.file, atom.payload)?; + ( + u32::from_be_bytes(bytes[12..16].try_into().unwrap()), + u32::from_be_bytes(bytes[16..20].try_into().unwrap()) as u64, + ) + }; + Ok(if scale > 0 { + (duration as f64 / scale as f64).round() as i64 + } else { + 0 + }) + } + + fn track_duration( + &mut self, + mut start: u64, + end: u64, + depth: usize, + best: &mut i64, + ) -> Result<(), String> { + if depth >= 32 { + return Err("MP4 nesting exceeds 32 levels".into()); + } + while start.checked_add(8).is_some_and(|value| value <= end) { + let atom = self.atom(start, end)?; + match &atom.kind { + b"mdhd" => *best = (*best).max(self.duration(atom).unwrap_or(0)), + b"trak" | b"mdia" => { + self.track_duration(atom.payload, atom.end, depth + 1, best)? + } + _ => {} + } + start = atom.end; + } + Ok(()) + } + + // Match Go's earliest recognized byte pattern, including chunk boundaries. + // Do not load moov/mdat in full: some files put large covers in moov. + fn sample_entry(&mut self, atom: Atom) -> Result<(u64, [u8; 4]), String> { + let mut offset = atom.start; + let mut data = vec![0; 65536 + 3]; + let mut tail = 0; + while offset < atom.end { + (self.check)()?; + let length = (atom.end - offset).min(65536) as usize; + self.file + .seek(SeekFrom::Start(offset)) + .map_err(|e| e.to_string())?; + let n = self + .file + .read(&mut data[tail..tail + length]) + .map_err(|e| format!("failed to read M4A atom data: {e}"))?; + if n == 0 { + break; + } + for (index, bytes) in data[..tail + n].windows(4).enumerate() { + if matches!( + bytes, + b"mp4a" | b"alac" | b"fLaC" | b"ec-3" | b"ac-3" | b"ac-4" | b"Opus" + ) { + let absolute = offset - tail as u64 + index as u64; + if absolute + .checked_add(32) + .is_none_or(|value| value > self.size) + { + return Err("audio info not found in M4A file".into()); + } + return Ok((absolute, bytes.try_into().unwrap())); + } + } + let end = tail + n; + tail = end.min(3); + data.copy_within(end - tail..end, 0); + offset += n as u64; + } + Err("audio info not found in M4A file".into()) + } + + fn specific_config(&mut self, offset: u64, kind: &[u8; 4]) -> Option<(i64, i64, i64)> { + let entry = self.atom(offset.checked_sub(4)?, self.size).ok()?; + let config = self + .find( + offset + 32, + entry.end, + if kind == b"alac" { b"alac" } else { b"dfLa" }, + ) + .ok()??; + let size = config.end.saturating_sub(config.payload); + // Go reads the entire config. Only these fixed fields are needed, and + // malformed size declarations must not cause file-sized allocations. + if config.end > self.size { + return None; + } + if kind == b"alac" { + if size < 24 { + return None; + } + let bytes = read_at::<24>(self.file, config.payload).ok()?; + let bits = bytes[5] as i64; + let rate = u32::from_be_bytes(bytes[20..24].try_into().unwrap()) as i64; + if bits > 0 && rate > 0 { + return Some((bits, rate, 0)); + } + if size < 28 { + return None; + } + let bytes = read_at::<24>(self.file, config.payload + 4).ok()?; + let bits = bytes[5] as i64; + let rate = u32::from_be_bytes(bytes[20..24].try_into().unwrap()) as i64; + return (bits > 0 && rate > 0).then_some((bits, rate, 0)); + } + let mut position = config.payload + 4; + while position + .checked_add(4) + .is_some_and(|value| value <= config.end) + { + (self.check)().ok()?; + let bytes = read_at::<4>(self.file, position).ok()?; + let length = u32::from_be_bytes([0, bytes[1], bytes[2], bytes[3]]) as u64; + let next = position.checked_add(4 + length)?; + if next > config.end { + return None; + } + if bytes[0] & 0x7f == 0 && length >= 34 { + let quality = flac_quality(&read_at::<34>(self.file, position + 4).ok()?); + return Some(( + quality.bit_depth, + quality.sample_rate, + quality.total_samples, + )); + } + position = next; + } + None + } +} diff --git a/rust_backend/crates/core/src/metadata.rs b/rust_backend/crates/core/src/metadata.rs new file mode 100644 index 00000000..80acdfb8 --- /dev/null +++ b/rust_backend/crates/core/src/metadata.rs @@ -0,0 +1,47 @@ +//! Application metadata response schemas and built-in provider conversion. + +mod catalog; +pub mod deezer; +pub mod musicbrainz; +pub mod reenrich; +pub mod share; +pub use catalog::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct TrackMetadata { + #[serde(skip_serializing_if = "String::is_empty")] + pub spotify_id: String, + pub artists: String, + pub name: String, + pub album_name: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub album_artist: String, + pub duration_ms: isize, + pub images: String, + pub release_date: String, + pub track_number: isize, + #[serde(skip_serializing_if = "zero")] + pub total_tracks: isize, + #[serde(skip_serializing_if = "zero")] + pub disc_number: isize, + #[serde(skip_serializing_if = "zero")] + pub total_discs: isize, + pub external_urls: String, + pub isrc: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub album_id: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub artist_id: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub album_type: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub composer: String, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub explicit: bool, +} + +fn zero(value: &isize) -> bool { + *value == 0 +} diff --git a/rust_backend/crates/core/src/metadata/catalog.rs b/rust_backend/crates/core/src/metadata/catalog.rs new file mode 100644 index 00000000..7afec51d --- /dev/null +++ b/rust_backend/crates/core/src/metadata/catalog.rs @@ -0,0 +1,164 @@ +use super::{TrackMetadata, zero}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct AlbumTrackMetadata { + #[serde(skip_serializing_if = "String::is_empty")] + pub spotify_id: String, + pub artists: String, + pub name: String, + pub album_name: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub album_artist: String, + pub duration_ms: isize, + pub images: String, + pub release_date: String, + pub track_number: isize, + #[serde(skip_serializing_if = "zero")] + pub total_tracks: isize, + #[serde(skip_serializing_if = "zero")] + pub disc_number: isize, + #[serde(skip_serializing_if = "zero")] + pub total_discs: isize, + pub external_urls: String, + pub isrc: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub album_id: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub album_url: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub album_type: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub composer: String, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub explicit: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct AlbumInfoMetadata { + pub total_tracks: isize, + pub name: String, + pub release_date: String, + pub artists: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub artist_id: String, + pub images: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub genre: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub label: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub copyright: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct AlbumResponsePayload { + pub album_info: AlbumInfoMetadata, + pub track_list: Vec<AlbumTrackMetadata>, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct PlaylistTrackCount { + pub total: isize, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct PlaylistOwner { + pub display_name: String, + pub name: String, + pub images: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct PlaylistInfoMetadata { + #[serde(skip_serializing_if = "String::is_empty")] + pub name: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub images: String, + pub tracks: PlaylistTrackCount, + pub owner: PlaylistOwner, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct PlaylistResponsePayload { + pub playlist_info: PlaylistInfoMetadata, + pub track_list: Vec<AlbumTrackMetadata>, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct ArtistInfoMetadata { + pub id: String, + pub name: String, + pub images: String, + pub followers: isize, + pub popularity: isize, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct ArtistAlbumMetadata { + pub id: String, + pub name: String, + pub release_date: String, + pub total_tracks: isize, + pub images: String, + pub album_type: String, + pub artists: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct ArtistResponsePayload { + pub artist_info: ArtistInfoMetadata, + pub albums: Vec<ArtistAlbumMetadata>, +} + +pub type SearchArtistResult = ArtistInfoMetadata; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct SearchAlbumResult { + pub id: String, + pub name: String, + pub artists: String, + pub images: String, + pub release_date: String, + pub total_tracks: isize, + pub album_type: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct SearchPlaylistResult { + pub id: String, + pub name: String, + pub owner: String, + pub images: String, + pub total_tracks: isize, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct SearchAllResult { + pub tracks: Vec<TrackMetadata>, + pub artists: Vec<SearchArtistResult>, + pub albums: Vec<SearchAlbumResult>, + pub playlists: Vec<SearchPlaylistResult>, +} + +// Go's internal extended-metadata response has exported, untagged fields. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct AlbumExtendedMetadata { + pub genre: String, + pub label: String, + pub copyright: String, +} diff --git a/rust_backend/crates/core/src/metadata/deezer.rs b/rust_backend/crates/core/src/metadata/deezer.rs new file mode 100644 index 00000000..cdd44f69 --- /dev/null +++ b/rust_backend/crates/core/src/metadata/deezer.rs @@ -0,0 +1,146 @@ +use super::TrackMetadata; +use crate::lyrics::json::go_deserialize; + +mod catalog; +pub use catalog::*; + +#[derive(Clone, Debug, Default)] +pub struct Track { + pub id: i64, + pub title: String, + pub duration: isize, + pub track_position: isize, + pub disk_number: isize, + pub isrc: String, + pub link: String, + pub release_date: String, + pub explicit_lyrics: bool, + pub explicit_content_lyrics: isize, + pub artist: Artist, + pub album: Album, + pub contributors: Option<Vec<Artist>>, +} +go_deserialize!(Track { + "id" => id, "title" => title, "duration" => duration, "track_position" => track_position, + "disk_number" => disk_number, "isrc" => isrc, "link" => link, "release_date" => release_date, + "explicit_lyrics" => explicit_lyrics, "explicit_content_lyrics" => explicit_content_lyrics, + "artist" => artist, "album" => album, "contributors" => contributors, +}); + +#[derive(Clone, Debug, Default)] +pub struct Artist { + pub id: i64, + pub name: String, + pub picture: String, + pub picture_medium: String, + pub picture_big: String, + pub picture_xl: String, + pub nb_fan: isize, +} +go_deserialize!(Artist { + "id" => id, "name" => name, "picture" => picture, "picture_medium" => picture_medium, + "picture_big" => picture_big, "picture_xl" => picture_xl, "nb_fan" => nb_fan, +}); + +#[derive(Clone, Debug, Default)] +pub struct Album { + pub id: i64, + pub title: String, + pub cover: String, + pub cover_medium: String, + pub cover_big: String, + pub cover_xl: String, + pub release_date: String, + pub record_type: String, +} +go_deserialize!(Album { + "id" => id, "title" => title, "cover" => cover, "cover_medium" => cover_medium, + "cover_big" => cover_big, "cover_xl" => cover_xl, "release_date" => release_date, "record_type" => record_type, +}); + +#[derive(Clone, Debug, Default)] +pub struct Tracks { + pub data: Option<Vec<Track>>, +} +go_deserialize!(Tracks { "data" => data, }); + +impl Track { + pub fn artist_display(&self) -> String { + match self.contributors.as_deref() { + Some(artists) if !artists.is_empty() => artists + .iter() + .map(|artist| artist.name.as_str()) + .collect::<Vec<_>>() + .join(", "), + _ => self.artist.name.clone(), + } + } + + pub fn is_explicit(&self) -> bool { + self.explicit_lyrics || self.explicit_content_lyrics == 1 + } + + pub fn metadata(&self) -> TrackMetadata { + TrackMetadata { + spotify_id: format!("deezer:{}", self.id), + artists: self.artist_display(), + name: self.title.clone(), + album_name: self.album.title.clone(), + album_artist: self.artist.name.clone(), + duration_ms: self.duration.wrapping_mul(1000), + images: self.album.image(), + release_date: if self.release_date.is_empty() { + self.album.release_date.clone() + } else { + self.release_date.clone() + }, + track_number: self.track_position, + disc_number: self.disk_number, + external_urls: self.link.clone(), + isrc: self.isrc.clone(), + album_id: format!("deezer:{}", self.album.id), + artist_id: format!("deezer:{}", self.artist.id), + explicit: self.is_explicit(), + ..TrackMetadata::default() + } + } +} + +pub fn best_image<'a>(images: impl IntoIterator<Item = &'a str>) -> String { + images + .into_iter() + .find(|image| !image.is_empty()) + .unwrap_or_default() + .into() +} + +pub fn album_type(record_type: &str) -> String { + if record_type == "compile" { + "compilation" + } else { + record_type + } + .into() +} + +impl Album { + pub fn image(&self) -> String { + best_image([ + self.cover_xl.as_str(), + &self.cover_big, + &self.cover_medium, + &self.cover, + ]) + } +} + +impl Artist { + pub fn image(&self) -> String { + best_image([ + self.picture_xl.as_str(), + &self.picture_big, + &self.picture_medium, + &self.picture, + ]) + } +} diff --git a/rust_backend/crates/core/src/metadata/deezer/catalog.rs b/rust_backend/crates/core/src/metadata/deezer/catalog.rs new file mode 100644 index 00000000..7ee0de52 --- /dev/null +++ b/rust_backend/crates/core/src/metadata/deezer/catalog.rs @@ -0,0 +1,241 @@ +use super::{Artist, Track, Tracks, best_image}; +use crate::lyrics::json::{Update, go_deserialize}; +use serde::Deserialize; + +#[derive(Clone, Debug, Default)] +pub struct ApiError { + pub error_type: String, + pub message: String, + pub code: isize, +} +go_deserialize!(ApiError { "type" => error_type, "message" => message, "code" => code, }); + +/// A repeated non-null Go pointer field merges into the existing object. +#[derive(Clone, Debug, Default)] +pub struct OptionalApiError(pub Option<ApiError>); + +impl<'de> Update<'de> for OptionalApiError { + fn update<D: serde::Deserializer<'de>>(&mut self, decoder: D) -> Result<(), D::Error> { + match Option::<&serde_json::value::RawValue>::deserialize(decoder)? { + None => self.0 = None, + Some(raw) => { + self.0 + .get_or_insert_default() + .update(&mut serde_json::Deserializer::from_str(raw.get())) + .map_err(serde::de::Error::custom)?; + } + } + Ok(()) + } +} + +macro_rules! search_page { + ($name:ident, $item:ty) => { + #[derive(Clone, Debug, Default)] + pub struct $name { + pub data: Option<Vec<$item>>, + pub error: OptionalApiError, + } + go_deserialize!($name { "data" => data, "error" => error, }); + }; +} +search_page!(TrackSearch, Track); +search_page!(ArtistSearch, Artist); +search_page!(AlbumSearch, SearchAlbum); +search_page!(PlaylistSearch, SearchPlaylist); + +#[derive(Clone, Debug, Default)] +pub struct TrackPage { + pub data: Option<Vec<Track>>, + pub next: String, +} +go_deserialize!(TrackPage { "data" => data, "next" => next, }); + +#[derive(Clone, Debug, Default)] +pub struct Genre { + pub id: isize, + pub name: String, +} +go_deserialize!(Genre { "id" => id, "name" => name, }); + +#[derive(Clone, Debug, Default)] +pub struct Genres { + pub data: Option<Vec<Genre>>, +} +go_deserialize!(Genres { "data" => data, }); + +impl Genres { + pub fn display(&self) -> String { + self.data + .iter() + .flatten() + .map(|genre| genre.name.as_str()) + .filter(|name| !name.is_empty()) + .collect::<Vec<_>>() + .join(", ") + } +} + +#[derive(Clone, Debug, Default)] +pub struct FullAlbum { + pub id: i64, + pub title: String, + pub cover: String, + pub cover_medium: String, + pub cover_big: String, + pub cover_xl: String, + pub release_date: String, + pub nb_tracks: isize, + pub record_type: String, + pub label: String, + pub copyright: String, + pub genres: Genres, + pub artist: Artist, + pub contributors: Option<Vec<Artist>>, + pub tracks: Tracks, +} +go_deserialize!(FullAlbum { + "id" => id, "title" => title, "cover" => cover, "cover_medium" => cover_medium, + "cover_big" => cover_big, "cover_xl" => cover_xl, "release_date" => release_date, + "nb_tracks" => nb_tracks, "record_type" => record_type, "label" => label, + "copyright" => copyright, "genres" => genres, "artist" => artist, + "contributors" => contributors, "tracks" => tracks, +}); + +#[derive(Clone, Debug, Default)] +pub struct FullArtist { + pub id: i64, + pub name: String, + pub picture: String, + pub picture_medium: String, + pub picture_big: String, + pub picture_xl: String, + pub nb_fan: isize, + pub nb_album: isize, +} +go_deserialize!(FullArtist { + "id" => id, "name" => name, "picture" => picture, "picture_medium" => picture_medium, + "picture_big" => picture_big, "picture_xl" => picture_xl, "nb_fan" => nb_fan, "nb_album" => nb_album, +}); + +#[derive(Clone, Debug, Default)] +pub struct Name { + pub name: String, +} +go_deserialize!(Name { "name" => name, }); + +#[derive(Clone, Debug, Default)] +pub struct FullPlaylist { + pub id: i64, + pub title: String, + pub picture: String, + pub picture_medium: String, + pub picture_big: String, + pub picture_xl: String, + pub nb_tracks: isize, + pub creator: Name, + pub tracks: Tracks, +} +go_deserialize!(FullPlaylist { + "id" => id, "title" => title, "picture" => picture, "picture_medium" => picture_medium, + "picture_big" => picture_big, "picture_xl" => picture_xl, "nb_tracks" => nb_tracks, + "creator" => creator, "tracks" => tracks, +}); + +#[derive(Clone, Debug, Default)] +pub struct SearchAlbum { + pub id: i64, + pub title: String, + pub cover: String, + pub cover_medium: String, + pub cover_big: String, + pub cover_xl: String, + pub nb_tracks: isize, + pub release_date: String, + pub record_type: String, + pub artist: Artist, +} +go_deserialize!(SearchAlbum { + "id" => id, "title" => title, "cover" => cover, "cover_medium" => cover_medium, + "cover_big" => cover_big, "cover_xl" => cover_xl, "nb_tracks" => nb_tracks, + "release_date" => release_date, "record_type" => record_type, "artist" => artist, +}); + +#[derive(Clone, Debug, Default)] +pub struct ArtistAlbum { + pub id: i64, + pub title: String, + pub release_date: String, + pub nb_tracks: isize, + pub cover: String, + pub cover_medium: String, + pub cover_big: String, + pub cover_xl: String, + pub record_type: String, +} +go_deserialize!(ArtistAlbum { + "id" => id, "title" => title, "release_date" => release_date, "nb_tracks" => nb_tracks, + "cover" => cover, "cover_medium" => cover_medium, "cover_big" => cover_big, + "cover_xl" => cover_xl, "record_type" => record_type, +}); + +#[derive(Clone, Debug, Default)] +pub struct ArtistAlbums { + pub data: Option<Vec<ArtistAlbum>>, +} +go_deserialize!(ArtistAlbums { "data" => data, }); + +#[derive(Clone, Debug, Default)] +pub struct AlbumTrackCount { + pub nb_tracks: isize, +} +go_deserialize!(AlbumTrackCount { "nb_tracks" => nb_tracks, }); + +#[derive(Clone, Debug, Default)] +pub struct SearchPlaylist { + pub id: i64, + pub title: String, + pub picture: String, + pub picture_medium: String, + pub picture_big: String, + pub picture_xl: String, + pub nb_tracks: isize, + pub user: Name, +} +go_deserialize!(SearchPlaylist { + "id" => id, "title" => title, "picture" => picture, "picture_medium" => picture_medium, + "picture_big" => picture_big, "picture_xl" => picture_xl, "nb_tracks" => nb_tracks, "user" => user, +}); + +macro_rules! cover_image { + ($($name:ty),*) => {$( + impl $name { + pub fn image(&self) -> String { + best_image([self.cover_xl.as_str(), &self.cover_big, &self.cover_medium, &self.cover]) + } + } + )*}; +} +cover_image!(FullAlbum, SearchAlbum, ArtistAlbum); + +impl FullArtist { + pub fn image(&self) -> String { + best_image([ + self.picture_xl.as_str(), + &self.picture_big, + &self.picture_medium, + &self.picture, + ]) + } +} + +impl SearchPlaylist { + pub fn image(&self) -> String { + best_image([ + self.picture_xl.as_str(), + &self.picture_big, + &self.picture_medium, + &self.picture, + ]) + } +} diff --git a/rust_backend/crates/core/src/metadata/musicbrainz.rs b/rust_backend/crates/core/src/metadata/musicbrainz.rs new file mode 100644 index 00000000..1c2c0ce9 --- /dev/null +++ b/rust_backend/crates/core/src/metadata/musicbrainz.rs @@ -0,0 +1,117 @@ +//! Recording metadata and formatting used by the native MusicBrainz getters. + +mod case_data; +mod casing; + +use crate::lyrics::json::go_deserialize; +use crate::matching::lowercase; +use std::collections::BTreeSet; + +#[derive(Clone, Debug, Default)] +pub struct Tag { + pub count: isize, + pub name: String, +} +go_deserialize!(Tag { "count" => count, "name" => name, }); + +#[derive(Clone, Debug, Default)] +pub struct ArtistCredit { + pub name: String, + pub joinphrase: String, +} +go_deserialize!(ArtistCredit { "name" => name, "joinphrase" => joinphrase, }); + +#[derive(Clone, Debug, Default)] +pub struct Release { + pub title: String, + pub artist_credit: Option<Vec<ArtistCredit>>, +} +go_deserialize!(Release { "title" => title, "artist-credit" => artist_credit, }); + +#[derive(Clone, Debug, Default)] +pub struct Recording { + pub tags: Option<Vec<Tag>>, + pub releases: Option<Vec<Release>>, +} +go_deserialize!(Recording { "tags" => tags, "releases" => releases, }); + +#[derive(Clone, Debug, Default)] +pub struct Response { + pub recordings: Option<Vec<Recording>>, +} +go_deserialize!(Response { "recordings" => recordings, }); + +pub fn genre(tags: &[Tag]) -> String { + let mut seen = BTreeSet::new(); + let mut max_count = -1; + let mut best = String::new(); + for tag in tags { + let name = tag.name.trim(); + if name.is_empty() || !seen.insert(lowercase(name)) { + continue; + } + if tag.count > max_count { + max_count = tag.count; + best = casing::title(name); + } + } + best +} + +pub fn artist_credit(credits: &[ArtistCredit]) -> String { + let mut output = String::new(); + for credit in credits { + let name = credit.name.trim(); + if !name.is_empty() { + output.push_str(name); + output.push_str(&credit.joinphrase); + } + } + output.trim().into() +} + +pub fn album_artist(releases: &[Release], album_name: &str) -> String { + let album = lowercase(album_name.trim()); + if !album.is_empty() { + for release in releases { + if lowercase(release.title.trim()) == album { + let artist = artist_credit(release.artist_credit.as_deref().unwrap_or_default()); + if !artist.is_empty() { + return artist; + } + } + } + } + releases + .iter() + .map(|release| artist_credit(release.artist_credit.as_deref().unwrap_or_default())) + .find(|artist| !artist.is_empty()) + .unwrap_or_default() +} + +impl Response { + pub fn genre(&self, isrc: &str) -> Result<String, String> { + let first = self + .recordings + .as_deref() + .unwrap_or_default() + .first() + .ok_or_else(|| format!("no recordings found for ISRC: {isrc}"))?; + let value = genre(first.tags.as_deref().unwrap_or_default()); + if value.is_empty() { + Err(format!("no MusicBrainz genre tags found for ISRC: {isrc}")) + } else { + Ok(value) + } + } + + pub fn album_artist(&self, isrc: &str, album: &str) -> Result<String, String> { + self.recordings + .as_deref() + .unwrap_or_default() + .iter() + .map(|recording| album_artist(recording.releases.as_deref().unwrap_or_default(), album)) + .find(|artist| !artist.is_empty()) + .ok_or_else(|| format!("no MusicBrainz album artist found for ISRC: {isrc}")) + } +} diff --git a/rust_backend/crates/core/src/metadata/musicbrainz/case_data.rs b/rust_backend/crates/core/src/metadata/musicbrainz/case_data.rs new file mode 100644 index 00000000..d31e399a --- /dev/null +++ b/rust_backend/crates/core/src/metadata/musicbrainz/case_data.rs @@ -0,0 +1,4131 @@ +// Generated by rust_backend/tools/generate_case_data.go. DO NOT EDIT. +// Unicode 15.0.0, golang.org/x/text v0.41.0; see NOTICE. +// Flags: cased=1, case-ignorable=2, word break=4, mid punctuation=8. +#[rustfmt::skip] +pub(super) const RANGES: &[(u32, u32, u8)] = &[ + (0x27, 0x27, 10), + (0x2e, 0x2e, 10), + (0x30, 0x39, 0), + (0x3a, 0x3a, 10), + (0x41, 0x5a, 1), + (0x5e, 0x5e, 6), + (0x5f, 0x5f, 0), + (0x60, 0x60, 6), + (0x61, 0x7a, 1), + (0xa8, 0xa8, 6), + (0xaa, 0xaa, 1), + (0xad, 0xad, 2), + (0xaf, 0xaf, 6), + (0xb4, 0xb4, 6), + (0xb5, 0xb5, 1), + (0xb7, 0xb7, 10), + (0xb8, 0xb8, 6), + (0xba, 0xba, 1), + (0xc0, 0xd6, 1), + (0xd8, 0xf6, 1), + (0xf8, 0x1ba, 1), + (0x1bb, 0x1bb, 0), + (0x1bc, 0x1bf, 1), + (0x1c0, 0x1c3, 0), + (0x1c4, 0x293, 1), + (0x294, 0x294, 0), + (0x295, 0x2af, 1), + (0x2b0, 0x2b8, 3), + (0x2b9, 0x2bf, 2), + (0x2c0, 0x2c1, 3), + (0x2c2, 0x2d7, 2), + (0x2d8, 0x2dd, 6), + (0x2de, 0x2df, 2), + (0x2e0, 0x2e4, 3), + (0x2e5, 0x344, 2), + (0x345, 0x345, 3), + (0x346, 0x36f, 2), + (0x370, 0x373, 1), + (0x374, 0x374, 2), + (0x375, 0x375, 6), + (0x376, 0x377, 1), + (0x37a, 0x37a, 3), + (0x37b, 0x37d, 1), + (0x37f, 0x37f, 1), + (0x384, 0x385, 6), + (0x386, 0x386, 1), + (0x387, 0x387, 10), + (0x388, 0x38a, 1), + (0x38c, 0x38c, 1), + (0x38e, 0x3a1, 1), + (0x3a3, 0x3f5, 1), + (0x3f7, 0x481, 1), + (0x483, 0x489, 2), + (0x48a, 0x52f, 1), + (0x531, 0x556, 1), + (0x559, 0x559, 2), + (0x55a, 0x55c, 0), + (0x55e, 0x55e, 0), + (0x55f, 0x55f, 10), + (0x560, 0x588, 1), + (0x58a, 0x58a, 0), + (0x591, 0x5bd, 2), + (0x5bf, 0x5bf, 2), + (0x5c1, 0x5c2, 2), + (0x5c4, 0x5c5, 2), + (0x5c7, 0x5c7, 2), + (0x5d0, 0x5ea, 0), + (0x5ef, 0x5f3, 0), + (0x5f4, 0x5f4, 10), + (0x600, 0x605, 2), + (0x610, 0x61a, 2), + (0x61c, 0x61c, 2), + (0x620, 0x63f, 0), + (0x640, 0x640, 2), + (0x641, 0x64a, 0), + (0x64b, 0x65f, 2), + (0x660, 0x669, 0), + (0x66b, 0x66b, 0), + (0x66e, 0x66f, 0), + (0x670, 0x670, 2), + (0x671, 0x6d3, 0), + (0x6d5, 0x6d5, 0), + (0x6d6, 0x6dd, 2), + (0x6df, 0x6e8, 2), + (0x6ea, 0x6ed, 2), + (0x6ee, 0x6fc, 0), + (0x6ff, 0x6ff, 0), + (0x70f, 0x70f, 2), + (0x710, 0x710, 0), + (0x711, 0x711, 2), + (0x712, 0x72f, 0), + (0x730, 0x74a, 2), + (0x74d, 0x7a5, 0), + (0x7a6, 0x7b0, 2), + (0x7b1, 0x7b1, 0), + (0x7c0, 0x7ea, 0), + (0x7eb, 0x7f5, 2), + (0x7fa, 0x7fa, 2), + (0x7fd, 0x7fd, 2), + (0x800, 0x815, 0), + (0x816, 0x82d, 2), + (0x840, 0x858, 0), + (0x859, 0x85b, 2), + (0x860, 0x86a, 0), + (0x870, 0x887, 0), + (0x888, 0x888, 6), + (0x889, 0x88e, 0), + (0x890, 0x891, 2), + (0x898, 0x89f, 2), + (0x8a0, 0x8c8, 0), + (0x8c9, 0x902, 2), + (0x903, 0x939, 0), + (0x93a, 0x93a, 2), + (0x93b, 0x93b, 0), + (0x93c, 0x93c, 2), + (0x93d, 0x940, 0), + (0x941, 0x948, 2), + (0x949, 0x94c, 0), + (0x94d, 0x94d, 2), + (0x94e, 0x950, 0), + (0x951, 0x957, 2), + (0x958, 0x961, 0), + (0x962, 0x963, 2), + (0x966, 0x96f, 0), + (0x971, 0x971, 2), + (0x972, 0x980, 0), + (0x981, 0x981, 2), + (0x982, 0x983, 0), + (0x985, 0x98c, 0), + (0x98f, 0x990, 0), + (0x993, 0x9a8, 0), + (0x9aa, 0x9b0, 0), + (0x9b2, 0x9b2, 0), + (0x9b6, 0x9b9, 0), + (0x9bc, 0x9bc, 2), + (0x9bd, 0x9c0, 0), + (0x9c1, 0x9c4, 2), + (0x9c7, 0x9c8, 0), + (0x9cb, 0x9cc, 0), + (0x9cd, 0x9cd, 2), + (0x9ce, 0x9ce, 0), + (0x9d7, 0x9d7, 0), + (0x9dc, 0x9dd, 0), + (0x9df, 0x9e1, 0), + (0x9e2, 0x9e3, 2), + (0x9e6, 0x9f1, 0), + (0x9fc, 0x9fc, 0), + (0x9fe, 0x9fe, 2), + (0xa01, 0xa02, 2), + (0xa03, 0xa03, 0), + (0xa05, 0xa0a, 0), + (0xa0f, 0xa10, 0), + (0xa13, 0xa28, 0), + (0xa2a, 0xa30, 0), + (0xa32, 0xa33, 0), + (0xa35, 0xa36, 0), + (0xa38, 0xa39, 0), + (0xa3c, 0xa3c, 2), + (0xa3e, 0xa40, 0), + (0xa41, 0xa42, 2), + (0xa47, 0xa48, 2), + (0xa4b, 0xa4d, 2), + (0xa51, 0xa51, 2), + (0xa59, 0xa5c, 0), + (0xa5e, 0xa5e, 0), + (0xa66, 0xa6f, 0), + (0xa70, 0xa71, 2), + (0xa72, 0xa74, 0), + (0xa75, 0xa75, 2), + (0xa81, 0xa82, 2), + (0xa83, 0xa83, 0), + (0xa85, 0xa8d, 0), + (0xa8f, 0xa91, 0), + (0xa93, 0xaa8, 0), + (0xaaa, 0xab0, 0), + (0xab2, 0xab3, 0), + (0xab5, 0xab9, 0), + (0xabc, 0xabc, 2), + (0xabd, 0xac0, 0), + (0xac1, 0xac5, 2), + (0xac7, 0xac8, 2), + (0xac9, 0xac9, 0), + (0xacb, 0xacc, 0), + (0xacd, 0xacd, 2), + (0xad0, 0xad0, 0), + (0xae0, 0xae1, 0), + (0xae2, 0xae3, 2), + (0xae6, 0xaef, 0), + (0xaf9, 0xaf9, 0), + (0xafa, 0xaff, 2), + (0xb01, 0xb01, 2), + (0xb02, 0xb03, 0), + (0xb05, 0xb0c, 0), + (0xb0f, 0xb10, 0), + (0xb13, 0xb28, 0), + (0xb2a, 0xb30, 0), + (0xb32, 0xb33, 0), + (0xb35, 0xb39, 0), + (0xb3c, 0xb3c, 2), + (0xb3d, 0xb3e, 0), + (0xb3f, 0xb3f, 2), + (0xb40, 0xb40, 0), + (0xb41, 0xb44, 2), + (0xb47, 0xb48, 0), + (0xb4b, 0xb4c, 0), + (0xb4d, 0xb4d, 2), + (0xb55, 0xb56, 2), + (0xb57, 0xb57, 0), + (0xb5c, 0xb5d, 0), + (0xb5f, 0xb61, 0), + (0xb62, 0xb63, 2), + (0xb66, 0xb6f, 0), + (0xb71, 0xb71, 0), + (0xb82, 0xb82, 2), + (0xb83, 0xb83, 0), + (0xb85, 0xb8a, 0), + (0xb8e, 0xb90, 0), + (0xb92, 0xb95, 0), + (0xb99, 0xb9a, 0), + (0xb9c, 0xb9c, 0), + (0xb9e, 0xb9f, 0), + (0xba3, 0xba4, 0), + (0xba8, 0xbaa, 0), + (0xbae, 0xbb9, 0), + (0xbbe, 0xbbf, 0), + (0xbc0, 0xbc0, 2), + (0xbc1, 0xbc2, 0), + (0xbc6, 0xbc8, 0), + (0xbca, 0xbcc, 0), + (0xbcd, 0xbcd, 2), + (0xbd0, 0xbd0, 0), + (0xbd7, 0xbd7, 0), + (0xbe6, 0xbef, 0), + (0xc00, 0xc00, 2), + (0xc01, 0xc03, 0), + (0xc04, 0xc04, 2), + (0xc05, 0xc0c, 0), + (0xc0e, 0xc10, 0), + (0xc12, 0xc28, 0), + (0xc2a, 0xc39, 0), + (0xc3c, 0xc3c, 2), + (0xc3d, 0xc3d, 0), + (0xc3e, 0xc40, 2), + (0xc41, 0xc44, 0), + (0xc46, 0xc48, 2), + (0xc4a, 0xc4d, 2), + (0xc55, 0xc56, 2), + (0xc58, 0xc5a, 0), + (0xc5d, 0xc5d, 0), + (0xc60, 0xc61, 0), + (0xc62, 0xc63, 2), + (0xc66, 0xc6f, 0), + (0xc80, 0xc80, 0), + (0xc81, 0xc81, 2), + (0xc82, 0xc83, 0), + (0xc85, 0xc8c, 0), + (0xc8e, 0xc90, 0), + (0xc92, 0xca8, 0), + (0xcaa, 0xcb3, 0), + (0xcb5, 0xcb9, 0), + (0xcbc, 0xcbc, 2), + (0xcbd, 0xcbe, 0), + (0xcbf, 0xcbf, 2), + (0xcc0, 0xcc4, 0), + (0xcc6, 0xcc6, 2), + (0xcc7, 0xcc8, 0), + (0xcca, 0xccb, 0), + (0xccc, 0xccd, 2), + (0xcd5, 0xcd6, 0), + (0xcdd, 0xcde, 0), + (0xce0, 0xce1, 0), + (0xce2, 0xce3, 2), + (0xce6, 0xcef, 0), + (0xcf1, 0xcf3, 0), + (0xd00, 0xd01, 2), + (0xd02, 0xd0c, 0), + (0xd0e, 0xd10, 0), + (0xd12, 0xd3a, 0), + (0xd3b, 0xd3c, 2), + (0xd3d, 0xd40, 0), + (0xd41, 0xd44, 2), + (0xd46, 0xd48, 0), + (0xd4a, 0xd4c, 0), + (0xd4d, 0xd4d, 2), + (0xd4e, 0xd4e, 0), + (0xd54, 0xd57, 0), + (0xd5f, 0xd61, 0), + (0xd62, 0xd63, 2), + (0xd66, 0xd6f, 0), + (0xd7a, 0xd7f, 0), + (0xd81, 0xd81, 2), + (0xd82, 0xd83, 0), + (0xd85, 0xd96, 0), + (0xd9a, 0xdb1, 0), + (0xdb3, 0xdbb, 0), + (0xdbd, 0xdbd, 0), + (0xdc0, 0xdc6, 0), + (0xdca, 0xdca, 2), + (0xdcf, 0xdd1, 0), + (0xdd2, 0xdd4, 2), + (0xdd6, 0xdd6, 2), + (0xdd8, 0xddf, 0), + (0xde6, 0xdef, 0), + (0xdf2, 0xdf3, 0), + (0xe31, 0xe31, 2), + (0xe34, 0xe3a, 2), + (0xe46, 0xe46, 6), + (0xe47, 0xe4e, 2), + (0xe50, 0xe59, 0), + (0xeb1, 0xeb1, 2), + (0xeb4, 0xebc, 2), + (0xec6, 0xec6, 6), + (0xec8, 0xece, 2), + (0xed0, 0xed9, 0), + (0xf00, 0xf00, 0), + (0xf18, 0xf19, 2), + (0xf20, 0xf29, 0), + (0xf35, 0xf35, 2), + (0xf37, 0xf37, 2), + (0xf39, 0xf39, 2), + (0xf3e, 0xf47, 0), + (0xf49, 0xf6c, 0), + (0xf71, 0xf7e, 2), + (0xf7f, 0xf7f, 0), + (0xf80, 0xf84, 2), + (0xf86, 0xf87, 2), + (0xf88, 0xf8c, 0), + (0xf8d, 0xf97, 2), + (0xf99, 0xfbc, 2), + (0xfc6, 0xfc6, 2), + (0x102b, 0x102c, 0), + (0x102d, 0x1030, 2), + (0x1031, 0x1031, 0), + (0x1032, 0x1037, 2), + (0x1038, 0x1038, 0), + (0x1039, 0x103a, 2), + (0x103b, 0x103c, 0), + (0x103d, 0x103e, 2), + (0x1040, 0x1049, 0), + (0x1056, 0x1057, 0), + (0x1058, 0x1059, 2), + (0x105e, 0x1060, 2), + (0x1062, 0x1064, 0), + (0x1067, 0x106d, 0), + (0x1071, 0x1074, 2), + (0x1082, 0x1082, 2), + (0x1083, 0x1084, 0), + (0x1085, 0x1086, 2), + (0x1087, 0x108c, 0), + (0x108d, 0x108d, 2), + (0x108f, 0x109c, 0), + (0x109d, 0x109d, 2), + (0x10a0, 0x10c5, 1), + (0x10c7, 0x10c7, 1), + (0x10cd, 0x10cd, 1), + (0x10d0, 0x10fa, 1), + (0x10fc, 0x10fc, 3), + (0x10fd, 0x10ff, 1), + (0x1100, 0x1248, 0), + (0x124a, 0x124d, 0), + (0x1250, 0x1256, 0), + (0x1258, 0x1258, 0), + (0x125a, 0x125d, 0), + (0x1260, 0x1288, 0), + (0x128a, 0x128d, 0), + (0x1290, 0x12b0, 0), + (0x12b2, 0x12b5, 0), + (0x12b8, 0x12be, 0), + (0x12c0, 0x12c0, 0), + (0x12c2, 0x12c5, 0), + (0x12c8, 0x12d6, 0), + (0x12d8, 0x1310, 0), + (0x1312, 0x1315, 0), + (0x1318, 0x135a, 0), + (0x135d, 0x135f, 2), + (0x1380, 0x138f, 0), + (0x13a0, 0x13f5, 1), + (0x13f8, 0x13fd, 1), + (0x1401, 0x166c, 0), + (0x166f, 0x167f, 0), + (0x1681, 0x169a, 0), + (0x16a0, 0x16ea, 0), + (0x16ee, 0x16f8, 0), + (0x1700, 0x1711, 0), + (0x1712, 0x1714, 2), + (0x1715, 0x1715, 0), + (0x171f, 0x1731, 0), + (0x1732, 0x1733, 2), + (0x1734, 0x1734, 0), + (0x1740, 0x1751, 0), + (0x1752, 0x1753, 2), + (0x1760, 0x176c, 0), + (0x176e, 0x1770, 0), + (0x1772, 0x1773, 2), + (0x17b4, 0x17b5, 2), + (0x17b6, 0x17b6, 0), + (0x17b7, 0x17bd, 2), + (0x17be, 0x17c5, 0), + (0x17c6, 0x17c6, 2), + (0x17c7, 0x17c8, 0), + (0x17c9, 0x17d3, 2), + (0x17d7, 0x17d7, 6), + (0x17dd, 0x17dd, 2), + (0x17e0, 0x17e9, 0), + (0x180b, 0x180f, 2), + (0x1810, 0x1819, 0), + (0x1820, 0x1842, 0), + (0x1843, 0x1843, 2), + (0x1844, 0x1878, 0), + (0x1880, 0x1884, 0), + (0x1885, 0x1886, 2), + (0x1887, 0x18a8, 0), + (0x18a9, 0x18a9, 2), + (0x18aa, 0x18aa, 0), + (0x18b0, 0x18f5, 0), + (0x1900, 0x191e, 0), + (0x1920, 0x1922, 2), + (0x1923, 0x1926, 0), + (0x1927, 0x1928, 2), + (0x1929, 0x192b, 0), + (0x1930, 0x1931, 0), + (0x1932, 0x1932, 2), + (0x1933, 0x1938, 0), + (0x1939, 0x193b, 2), + (0x1946, 0x194f, 0), + (0x19d0, 0x19d9, 0), + (0x1a00, 0x1a16, 0), + (0x1a17, 0x1a18, 2), + (0x1a19, 0x1a1a, 0), + (0x1a1b, 0x1a1b, 2), + (0x1a55, 0x1a55, 0), + (0x1a56, 0x1a56, 2), + (0x1a57, 0x1a57, 0), + (0x1a58, 0x1a5e, 2), + (0x1a60, 0x1a60, 2), + (0x1a61, 0x1a61, 0), + (0x1a62, 0x1a62, 2), + (0x1a63, 0x1a64, 0), + (0x1a65, 0x1a6c, 2), + (0x1a6d, 0x1a72, 0), + (0x1a73, 0x1a7c, 2), + (0x1a7f, 0x1a7f, 2), + (0x1a80, 0x1a89, 0), + (0x1a90, 0x1a99, 0), + (0x1aa7, 0x1aa7, 6), + (0x1ab0, 0x1ace, 2), + (0x1b00, 0x1b03, 2), + (0x1b04, 0x1b33, 0), + (0x1b34, 0x1b34, 2), + (0x1b35, 0x1b35, 0), + (0x1b36, 0x1b3a, 2), + (0x1b3b, 0x1b3b, 0), + (0x1b3c, 0x1b3c, 2), + (0x1b3d, 0x1b41, 0), + (0x1b42, 0x1b42, 2), + (0x1b43, 0x1b4c, 0), + (0x1b50, 0x1b59, 0), + (0x1b6b, 0x1b73, 2), + (0x1b80, 0x1b81, 2), + (0x1b82, 0x1ba1, 0), + (0x1ba2, 0x1ba5, 2), + (0x1ba6, 0x1ba7, 0), + (0x1ba8, 0x1ba9, 2), + (0x1baa, 0x1baa, 0), + (0x1bab, 0x1bad, 2), + (0x1bae, 0x1be5, 0), + (0x1be6, 0x1be6, 2), + (0x1be7, 0x1be7, 0), + (0x1be8, 0x1be9, 2), + (0x1bea, 0x1bec, 0), + (0x1bed, 0x1bed, 2), + (0x1bee, 0x1bee, 0), + (0x1bef, 0x1bf1, 2), + (0x1bf2, 0x1bf3, 0), + (0x1c00, 0x1c2b, 0), + (0x1c2c, 0x1c33, 2), + (0x1c34, 0x1c35, 0), + (0x1c36, 0x1c37, 2), + (0x1c40, 0x1c49, 0), + (0x1c4d, 0x1c77, 0), + (0x1c78, 0x1c7d, 2), + (0x1c80, 0x1c88, 1), + (0x1c90, 0x1cba, 1), + (0x1cbd, 0x1cbf, 1), + (0x1cd0, 0x1cd2, 2), + (0x1cd4, 0x1ce0, 2), + (0x1ce1, 0x1ce1, 0), + (0x1ce2, 0x1ce8, 2), + (0x1ce9, 0x1cec, 0), + (0x1ced, 0x1ced, 2), + (0x1cee, 0x1cf3, 0), + (0x1cf4, 0x1cf4, 2), + (0x1cf5, 0x1cf7, 0), + (0x1cf8, 0x1cf9, 2), + (0x1cfa, 0x1cfa, 0), + (0x1d00, 0x1d2b, 1), + (0x1d2c, 0x1d6a, 3), + (0x1d6b, 0x1d77, 1), + (0x1d78, 0x1d78, 3), + (0x1d79, 0x1d9a, 1), + (0x1d9b, 0x1dbf, 3), + (0x1dc0, 0x1dff, 2), + (0x1e00, 0x1f15, 1), + (0x1f18, 0x1f1d, 1), + (0x1f20, 0x1f45, 1), + (0x1f48, 0x1f4d, 1), + (0x1f50, 0x1f57, 1), + (0x1f59, 0x1f59, 1), + (0x1f5b, 0x1f5b, 1), + (0x1f5d, 0x1f5d, 1), + (0x1f5f, 0x1f7d, 1), + (0x1f80, 0x1fb4, 1), + (0x1fb6, 0x1fbc, 1), + (0x1fbd, 0x1fbd, 6), + (0x1fbe, 0x1fbe, 1), + (0x1fbf, 0x1fc1, 6), + (0x1fc2, 0x1fc4, 1), + (0x1fc6, 0x1fcc, 1), + (0x1fcd, 0x1fcf, 6), + (0x1fd0, 0x1fd3, 1), + (0x1fd6, 0x1fdb, 1), + (0x1fdd, 0x1fdf, 6), + (0x1fe0, 0x1fec, 1), + (0x1fed, 0x1fef, 6), + (0x1ff2, 0x1ff4, 1), + (0x1ff6, 0x1ffc, 1), + (0x1ffd, 0x1ffe, 6), + (0x200b, 0x200b, 6), + (0x200c, 0x200f, 2), + (0x2018, 0x2019, 10), + (0x2024, 0x2024, 10), + (0x2027, 0x2027, 10), + (0x202a, 0x202e, 2), + (0x202f, 0x202f, 0), + (0x203f, 0x2040, 0), + (0x2054, 0x2054, 0), + (0x2060, 0x2064, 2), + (0x2066, 0x206f, 2), + (0x2071, 0x2071, 3), + (0x207f, 0x207f, 3), + (0x2090, 0x209c, 3), + (0x20d0, 0x20f0, 2), + (0x2102, 0x2102, 1), + (0x2107, 0x2107, 1), + (0x210a, 0x2113, 1), + (0x2115, 0x2115, 1), + (0x2119, 0x211d, 1), + (0x2124, 0x2124, 1), + (0x2126, 0x2126, 1), + (0x2128, 0x2128, 1), + (0x212a, 0x212d, 1), + (0x212f, 0x2134, 1), + (0x2135, 0x2138, 0), + (0x2139, 0x2139, 1), + (0x213c, 0x213f, 1), + (0x2145, 0x2149, 1), + (0x214e, 0x214e, 1), + (0x2160, 0x217f, 1), + (0x2180, 0x2182, 0), + (0x2183, 0x2184, 1), + (0x2185, 0x2188, 0), + (0x24b6, 0x24e9, 1), + (0x2c00, 0x2c7b, 1), + (0x2c7c, 0x2c7d, 3), + (0x2c7e, 0x2ce4, 1), + (0x2ceb, 0x2cee, 1), + (0x2cef, 0x2cf1, 2), + (0x2cf2, 0x2cf3, 1), + (0x2d00, 0x2d25, 1), + (0x2d27, 0x2d27, 1), + (0x2d2d, 0x2d2d, 1), + (0x2d30, 0x2d67, 0), + (0x2d6f, 0x2d6f, 2), + (0x2d7f, 0x2d7f, 2), + (0x2d80, 0x2d96, 0), + (0x2da0, 0x2da6, 0), + (0x2da8, 0x2dae, 0), + (0x2db0, 0x2db6, 0), + (0x2db8, 0x2dbe, 0), + (0x2dc0, 0x2dc6, 0), + (0x2dc8, 0x2dce, 0), + (0x2dd0, 0x2dd6, 0), + (0x2dd8, 0x2dde, 0), + (0x2de0, 0x2dff, 2), + (0x2e2f, 0x2e2f, 2), + (0x3005, 0x3005, 2), + (0x302a, 0x302d, 2), + (0x302e, 0x302f, 0), + (0x3031, 0x3035, 6), + (0x303b, 0x303b, 2), + (0x303c, 0x303c, 0), + (0x3099, 0x309a, 2), + (0x309b, 0x309e, 6), + (0x30fc, 0x30fe, 6), + (0x3105, 0x312f, 0), + (0x3131, 0x318e, 0), + (0x31a0, 0x31bf, 0), + (0xa000, 0xa014, 0), + (0xa015, 0xa015, 2), + (0xa016, 0xa48c, 0), + (0xa4d0, 0xa4f7, 0), + (0xa4f8, 0xa4fd, 2), + (0xa500, 0xa60b, 0), + (0xa60c, 0xa60c, 2), + (0xa610, 0xa62b, 0), + (0xa640, 0xa66d, 1), + (0xa66e, 0xa66e, 0), + (0xa66f, 0xa672, 2), + (0xa674, 0xa67d, 2), + (0xa67f, 0xa67f, 2), + (0xa680, 0xa69b, 1), + (0xa69c, 0xa69d, 3), + (0xa69e, 0xa69f, 2), + (0xa6a0, 0xa6ef, 0), + (0xa6f0, 0xa6f1, 2), + (0xa700, 0xa707, 6), + (0xa708, 0xa721, 2), + (0xa722, 0xa76f, 1), + (0xa770, 0xa770, 3), + (0xa771, 0xa787, 1), + (0xa788, 0xa78a, 2), + (0xa78b, 0xa78e, 1), + (0xa78f, 0xa78f, 0), + (0xa790, 0xa7ca, 1), + (0xa7d0, 0xa7d1, 1), + (0xa7d3, 0xa7d3, 1), + (0xa7d5, 0xa7d9, 1), + (0xa7f2, 0xa7f4, 3), + (0xa7f5, 0xa7f6, 1), + (0xa7f7, 0xa7f7, 0), + (0xa7f8, 0xa7f9, 3), + (0xa7fa, 0xa7fa, 1), + (0xa7fb, 0xa801, 0), + (0xa802, 0xa802, 2), + (0xa803, 0xa805, 0), + (0xa806, 0xa806, 2), + (0xa807, 0xa80a, 0), + (0xa80b, 0xa80b, 2), + (0xa80c, 0xa824, 0), + (0xa825, 0xa826, 2), + (0xa827, 0xa827, 0), + (0xa82c, 0xa82c, 2), + (0xa840, 0xa873, 0), + (0xa880, 0xa8c3, 0), + (0xa8c4, 0xa8c5, 2), + (0xa8d0, 0xa8d9, 0), + (0xa8e0, 0xa8f1, 2), + (0xa8f2, 0xa8f7, 0), + (0xa8fb, 0xa8fb, 0), + (0xa8fd, 0xa8fe, 0), + (0xa8ff, 0xa8ff, 2), + (0xa900, 0xa925, 0), + (0xa926, 0xa92d, 2), + (0xa930, 0xa946, 0), + (0xa947, 0xa951, 2), + (0xa952, 0xa953, 0), + (0xa960, 0xa97c, 0), + (0xa980, 0xa982, 2), + (0xa983, 0xa9b2, 0), + (0xa9b3, 0xa9b3, 2), + (0xa9b4, 0xa9b5, 0), + (0xa9b6, 0xa9b9, 2), + (0xa9ba, 0xa9bb, 0), + (0xa9bc, 0xa9bd, 2), + (0xa9be, 0xa9c0, 0), + (0xa9cf, 0xa9cf, 2), + (0xa9d0, 0xa9d9, 0), + (0xa9e5, 0xa9e5, 2), + (0xa9e6, 0xa9e6, 6), + (0xa9f0, 0xa9f9, 0), + (0xaa00, 0xaa28, 0), + (0xaa29, 0xaa2e, 2), + (0xaa2f, 0xaa30, 0), + (0xaa31, 0xaa32, 2), + (0xaa33, 0xaa34, 0), + (0xaa35, 0xaa36, 2), + (0xaa40, 0xaa42, 0), + (0xaa43, 0xaa43, 2), + (0xaa44, 0xaa4b, 0), + (0xaa4c, 0xaa4c, 2), + (0xaa4d, 0xaa4d, 0), + (0xaa50, 0xaa59, 0), + (0xaa70, 0xaa70, 6), + (0xaa7b, 0xaa7b, 0), + (0xaa7c, 0xaa7c, 2), + (0xaa7d, 0xaa7d, 0), + (0xaab0, 0xaab0, 2), + (0xaab2, 0xaab4, 2), + (0xaab7, 0xaab8, 2), + (0xaabe, 0xaabf, 2), + (0xaac1, 0xaac1, 2), + (0xaadd, 0xaadd, 6), + (0xaae0, 0xaaeb, 0), + (0xaaec, 0xaaed, 2), + (0xaaee, 0xaaef, 0), + (0xaaf2, 0xaaf2, 0), + (0xaaf3, 0xaaf4, 2), + (0xaaf5, 0xaaf5, 0), + (0xaaf6, 0xaaf6, 2), + (0xab01, 0xab06, 0), + (0xab09, 0xab0e, 0), + (0xab11, 0xab16, 0), + (0xab20, 0xab26, 0), + (0xab28, 0xab2e, 0), + (0xab30, 0xab5a, 1), + (0xab5b, 0xab5b, 2), + (0xab5c, 0xab5f, 3), + (0xab60, 0xab68, 1), + (0xab69, 0xab69, 3), + (0xab6a, 0xab6b, 6), + (0xab70, 0xabbf, 1), + (0xabc0, 0xabe4, 0), + (0xabe5, 0xabe5, 2), + (0xabe6, 0xabe7, 0), + (0xabe8, 0xabe8, 2), + (0xabe9, 0xabea, 0), + (0xabec, 0xabec, 0), + (0xabed, 0xabed, 2), + (0xabf0, 0xabf9, 0), + (0xac00, 0xd7a3, 0), + (0xd7b0, 0xd7c6, 0), + (0xd7cb, 0xd7fb, 0), + (0xfb00, 0xfb06, 1), + (0xfb13, 0xfb17, 1), + (0xfb1d, 0xfb1d, 0), + (0xfb1e, 0xfb1e, 2), + (0xfb1f, 0xfb28, 0), + (0xfb2a, 0xfb36, 0), + (0xfb38, 0xfb3c, 0), + (0xfb3e, 0xfb3e, 0), + (0xfb40, 0xfb41, 0), + (0xfb43, 0xfb44, 0), + (0xfb46, 0xfbb1, 0), + (0xfbb2, 0xfbc2, 6), + (0xfbd3, 0xfd3d, 0), + (0xfd50, 0xfd8f, 0), + (0xfd92, 0xfdc7, 0), + (0xfdf0, 0xfdfb, 0), + (0xfe00, 0xfe0f, 2), + (0xfe13, 0xfe13, 10), + (0xfe20, 0xfe2f, 2), + (0xfe33, 0xfe34, 0), + (0xfe4d, 0xfe4f, 0), + (0xfe52, 0xfe52, 10), + (0xfe55, 0xfe55, 10), + (0xfe70, 0xfe74, 0), + (0xfe76, 0xfefc, 0), + (0xfeff, 0xfeff, 2), + (0xff07, 0xff07, 10), + (0xff0e, 0xff0e, 10), + (0xff10, 0xff19, 0), + (0xff1a, 0xff1a, 10), + (0xff21, 0xff3a, 1), + (0xff3e, 0xff3e, 6), + (0xff3f, 0xff3f, 0), + (0xff40, 0xff40, 6), + (0xff41, 0xff5a, 1), + (0xff70, 0xff70, 6), + (0xff9e, 0xff9f, 2), + (0xffa0, 0xffbe, 0), + (0xffc2, 0xffc7, 0), + (0xffca, 0xffcf, 0), + (0xffd2, 0xffd7, 0), + (0xffda, 0xffdc, 0), + (0xffe3, 0xffe3, 6), + (0xfff9, 0xfffb, 2), + (0x10000, 0x1000b, 0), + (0x1000d, 0x10026, 0), + (0x10028, 0x1003a, 0), + (0x1003c, 0x1003d, 0), + (0x1003f, 0x1004d, 0), + (0x10050, 0x1005d, 0), + (0x10080, 0x100fa, 0), + (0x10140, 0x10174, 0), + (0x101fd, 0x101fd, 2), + (0x10280, 0x1029c, 0), + (0x102a0, 0x102d0, 0), + (0x102e0, 0x102e0, 2), + (0x10300, 0x1031f, 0), + (0x1032d, 0x1034a, 0), + (0x10350, 0x10375, 0), + (0x10376, 0x1037a, 2), + (0x10380, 0x1039d, 0), + (0x103a0, 0x103c3, 0), + (0x103c8, 0x103cf, 0), + (0x103d1, 0x103d5, 0), + (0x10400, 0x1044f, 1), + (0x10450, 0x1049d, 0), + (0x104a0, 0x104a9, 0), + (0x104b0, 0x104d3, 1), + (0x104d8, 0x104fb, 1), + (0x10500, 0x10527, 0), + (0x10530, 0x10563, 0), + (0x10570, 0x1057a, 1), + (0x1057c, 0x1058a, 1), + (0x1058c, 0x10592, 1), + (0x10594, 0x10595, 1), + (0x10597, 0x105a1, 1), + (0x105a3, 0x105b1, 1), + (0x105b3, 0x105b9, 1), + (0x105bb, 0x105bc, 1), + (0x10600, 0x10736, 0), + (0x10740, 0x10755, 0), + (0x10760, 0x10767, 0), + (0x10780, 0x10780, 3), + (0x10781, 0x10782, 2), + (0x10783, 0x10785, 3), + (0x10787, 0x107b0, 3), + (0x107b2, 0x107ba, 3), + (0x10800, 0x10805, 0), + (0x10808, 0x10808, 0), + (0x1080a, 0x10835, 0), + (0x10837, 0x10838, 0), + (0x1083c, 0x1083c, 0), + (0x1083f, 0x10855, 0), + (0x10860, 0x10876, 0), + (0x10880, 0x1089e, 0), + (0x108e0, 0x108f2, 0), + (0x108f4, 0x108f5, 0), + (0x10900, 0x10915, 0), + (0x10920, 0x10939, 0), + (0x10980, 0x109b7, 0), + (0x109be, 0x109bf, 0), + (0x10a00, 0x10a00, 0), + (0x10a01, 0x10a03, 2), + (0x10a05, 0x10a06, 2), + (0x10a0c, 0x10a0f, 2), + (0x10a10, 0x10a13, 0), + (0x10a15, 0x10a17, 0), + (0x10a19, 0x10a35, 0), + (0x10a38, 0x10a3a, 2), + (0x10a3f, 0x10a3f, 2), + (0x10a60, 0x10a7c, 0), + (0x10a80, 0x10a9c, 0), + (0x10ac0, 0x10ac7, 0), + (0x10ac9, 0x10ae4, 0), + (0x10ae5, 0x10ae6, 2), + (0x10b00, 0x10b35, 0), + (0x10b40, 0x10b55, 0), + (0x10b60, 0x10b72, 0), + (0x10b80, 0x10b91, 0), + (0x10c00, 0x10c48, 0), + (0x10c80, 0x10cb2, 1), + (0x10cc0, 0x10cf2, 1), + (0x10d00, 0x10d23, 0), + (0x10d24, 0x10d27, 2), + (0x10d30, 0x10d39, 0), + (0x10e80, 0x10ea9, 0), + (0x10eab, 0x10eac, 2), + (0x10eb0, 0x10eb1, 0), + (0x10efd, 0x10eff, 2), + (0x10f00, 0x10f1c, 0), + (0x10f27, 0x10f27, 0), + (0x10f30, 0x10f45, 0), + (0x10f46, 0x10f50, 2), + (0x10f70, 0x10f81, 0), + (0x10f82, 0x10f85, 2), + (0x10fb0, 0x10fc4, 0), + (0x10fe0, 0x10ff6, 0), + (0x11000, 0x11000, 0), + (0x11001, 0x11001, 2), + (0x11002, 0x11037, 0), + (0x11038, 0x11046, 2), + (0x11066, 0x1106f, 0), + (0x11070, 0x11070, 2), + (0x11071, 0x11072, 0), + (0x11073, 0x11074, 2), + (0x11075, 0x11075, 0), + (0x1107f, 0x11081, 2), + (0x11082, 0x110b2, 0), + (0x110b3, 0x110b6, 2), + (0x110b7, 0x110b8, 0), + (0x110b9, 0x110ba, 2), + (0x110bd, 0x110bd, 2), + (0x110c2, 0x110c2, 2), + (0x110cd, 0x110cd, 2), + (0x110d0, 0x110e8, 0), + (0x110f0, 0x110f9, 0), + (0x11100, 0x11102, 2), + (0x11103, 0x11126, 0), + (0x11127, 0x1112b, 2), + (0x1112c, 0x1112c, 0), + (0x1112d, 0x11134, 2), + (0x11136, 0x1113f, 0), + (0x11144, 0x11147, 0), + (0x11150, 0x11172, 0), + (0x11173, 0x11173, 2), + (0x11176, 0x11176, 0), + (0x11180, 0x11181, 2), + (0x11182, 0x111b5, 0), + (0x111b6, 0x111be, 2), + (0x111bf, 0x111c4, 0), + (0x111c9, 0x111cc, 2), + (0x111ce, 0x111ce, 0), + (0x111cf, 0x111cf, 2), + (0x111d0, 0x111da, 0), + (0x111dc, 0x111dc, 0), + (0x11200, 0x11211, 0), + (0x11213, 0x1122e, 0), + (0x1122f, 0x11231, 2), + (0x11232, 0x11233, 0), + (0x11234, 0x11234, 2), + (0x11235, 0x11235, 0), + (0x11236, 0x11237, 2), + (0x1123e, 0x1123e, 2), + (0x1123f, 0x11240, 0), + (0x11241, 0x11241, 2), + (0x11280, 0x11286, 0), + (0x11288, 0x11288, 0), + (0x1128a, 0x1128d, 0), + (0x1128f, 0x1129d, 0), + (0x1129f, 0x112a8, 0), + (0x112b0, 0x112de, 0), + (0x112df, 0x112df, 2), + (0x112e0, 0x112e2, 0), + (0x112e3, 0x112ea, 2), + (0x112f0, 0x112f9, 0), + (0x11300, 0x11301, 2), + (0x11302, 0x11303, 0), + (0x11305, 0x1130c, 0), + (0x1130f, 0x11310, 0), + (0x11313, 0x11328, 0), + (0x1132a, 0x11330, 0), + (0x11332, 0x11333, 0), + (0x11335, 0x11339, 0), + (0x1133b, 0x1133c, 2), + (0x1133d, 0x1133f, 0), + (0x11340, 0x11340, 2), + (0x11341, 0x11344, 0), + (0x11347, 0x11348, 0), + (0x1134b, 0x1134d, 0), + (0x11350, 0x11350, 0), + (0x11357, 0x11357, 0), + (0x1135d, 0x11363, 0), + (0x11366, 0x1136c, 2), + (0x11370, 0x11374, 2), + (0x11400, 0x11437, 0), + (0x11438, 0x1143f, 2), + (0x11440, 0x11441, 0), + (0x11442, 0x11444, 2), + (0x11445, 0x11445, 0), + (0x11446, 0x11446, 2), + (0x11447, 0x1144a, 0), + (0x11450, 0x11459, 0), + (0x1145e, 0x1145e, 2), + (0x1145f, 0x11461, 0), + (0x11480, 0x114b2, 0), + (0x114b3, 0x114b8, 2), + (0x114b9, 0x114b9, 0), + (0x114ba, 0x114ba, 2), + (0x114bb, 0x114be, 0), + (0x114bf, 0x114c0, 2), + (0x114c1, 0x114c1, 0), + (0x114c2, 0x114c3, 2), + (0x114c4, 0x114c5, 0), + (0x114c7, 0x114c7, 0), + (0x114d0, 0x114d9, 0), + (0x11580, 0x115b1, 0), + (0x115b2, 0x115b5, 2), + (0x115b8, 0x115bb, 0), + (0x115bc, 0x115bd, 2), + (0x115be, 0x115be, 0), + (0x115bf, 0x115c0, 2), + (0x115d8, 0x115db, 0), + (0x115dc, 0x115dd, 2), + (0x11600, 0x11632, 0), + (0x11633, 0x1163a, 2), + (0x1163b, 0x1163c, 0), + (0x1163d, 0x1163d, 2), + (0x1163e, 0x1163e, 0), + (0x1163f, 0x11640, 2), + (0x11644, 0x11644, 0), + (0x11650, 0x11659, 0), + (0x11680, 0x116aa, 0), + (0x116ab, 0x116ab, 2), + (0x116ac, 0x116ac, 0), + (0x116ad, 0x116ad, 2), + (0x116ae, 0x116af, 0), + (0x116b0, 0x116b5, 2), + (0x116b6, 0x116b6, 0), + (0x116b7, 0x116b7, 2), + (0x116b8, 0x116b8, 0), + (0x116c0, 0x116c9, 0), + (0x1171d, 0x1171f, 2), + (0x11720, 0x11721, 0), + (0x11722, 0x11725, 2), + (0x11726, 0x11726, 0), + (0x11727, 0x1172b, 2), + (0x11730, 0x11739, 0), + (0x11800, 0x1182e, 0), + (0x1182f, 0x11837, 2), + (0x11838, 0x11838, 0), + (0x11839, 0x1183a, 2), + (0x118a0, 0x118df, 1), + (0x118e0, 0x118e9, 0), + (0x118ff, 0x11906, 0), + (0x11909, 0x11909, 0), + (0x1190c, 0x11913, 0), + (0x11915, 0x11916, 0), + (0x11918, 0x11935, 0), + (0x11937, 0x11938, 0), + (0x1193b, 0x1193c, 2), + (0x1193d, 0x1193d, 0), + (0x1193e, 0x1193e, 2), + (0x1193f, 0x11942, 0), + (0x11943, 0x11943, 2), + (0x11950, 0x11959, 0), + (0x119a0, 0x119a7, 0), + (0x119aa, 0x119d3, 0), + (0x119d4, 0x119d7, 2), + (0x119da, 0x119db, 2), + (0x119dc, 0x119df, 0), + (0x119e0, 0x119e0, 2), + (0x119e1, 0x119e1, 0), + (0x119e3, 0x119e4, 0), + (0x11a00, 0x11a00, 0), + (0x11a01, 0x11a0a, 2), + (0x11a0b, 0x11a32, 0), + (0x11a33, 0x11a38, 2), + (0x11a39, 0x11a3a, 0), + (0x11a3b, 0x11a3e, 2), + (0x11a47, 0x11a47, 2), + (0x11a50, 0x11a50, 0), + (0x11a51, 0x11a56, 2), + (0x11a57, 0x11a58, 0), + (0x11a59, 0x11a5b, 2), + (0x11a5c, 0x11a89, 0), + (0x11a8a, 0x11a96, 2), + (0x11a97, 0x11a97, 0), + (0x11a98, 0x11a99, 2), + (0x11a9d, 0x11a9d, 0), + (0x11ab0, 0x11af8, 0), + (0x11c00, 0x11c08, 0), + (0x11c0a, 0x11c2f, 0), + (0x11c30, 0x11c36, 2), + (0x11c38, 0x11c3d, 2), + (0x11c3e, 0x11c3e, 0), + (0x11c3f, 0x11c3f, 2), + (0x11c40, 0x11c40, 0), + (0x11c50, 0x11c59, 0), + (0x11c72, 0x11c8f, 0), + (0x11c92, 0x11ca7, 2), + (0x11ca9, 0x11ca9, 0), + (0x11caa, 0x11cb0, 2), + (0x11cb1, 0x11cb1, 0), + (0x11cb2, 0x11cb3, 2), + (0x11cb4, 0x11cb4, 0), + (0x11cb5, 0x11cb6, 2), + (0x11d00, 0x11d06, 0), + (0x11d08, 0x11d09, 0), + (0x11d0b, 0x11d30, 0), + (0x11d31, 0x11d36, 2), + (0x11d3a, 0x11d3a, 2), + (0x11d3c, 0x11d3d, 2), + (0x11d3f, 0x11d45, 2), + (0x11d46, 0x11d46, 0), + (0x11d47, 0x11d47, 2), + (0x11d50, 0x11d59, 0), + (0x11d60, 0x11d65, 0), + (0x11d67, 0x11d68, 0), + (0x11d6a, 0x11d8e, 0), + (0x11d90, 0x11d91, 2), + (0x11d93, 0x11d94, 0), + (0x11d95, 0x11d95, 2), + (0x11d96, 0x11d96, 0), + (0x11d97, 0x11d97, 2), + (0x11d98, 0x11d98, 0), + (0x11da0, 0x11da9, 0), + (0x11ee0, 0x11ef2, 0), + (0x11ef3, 0x11ef4, 2), + (0x11ef5, 0x11ef6, 0), + (0x11f00, 0x11f01, 2), + (0x11f02, 0x11f10, 0), + (0x11f12, 0x11f35, 0), + (0x11f36, 0x11f3a, 2), + (0x11f3e, 0x11f3f, 0), + (0x11f40, 0x11f40, 2), + (0x11f41, 0x11f41, 0), + (0x11f42, 0x11f42, 2), + (0x11f50, 0x11f59, 0), + (0x11fb0, 0x11fb0, 0), + (0x12000, 0x12399, 0), + (0x12400, 0x1246e, 0), + (0x12480, 0x12543, 0), + (0x12f90, 0x12ff0, 0), + (0x13000, 0x1342f, 0), + (0x13430, 0x13440, 2), + (0x13441, 0x13446, 0), + (0x13447, 0x13455, 2), + (0x14400, 0x14646, 0), + (0x16800, 0x16a38, 0), + (0x16a40, 0x16a5e, 0), + (0x16a60, 0x16a69, 0), + (0x16a70, 0x16abe, 0), + (0x16ac0, 0x16ac9, 0), + (0x16ad0, 0x16aed, 0), + (0x16af0, 0x16af4, 2), + (0x16b00, 0x16b2f, 0), + (0x16b30, 0x16b36, 2), + (0x16b40, 0x16b43, 2), + (0x16b50, 0x16b59, 0), + (0x16b63, 0x16b77, 0), + (0x16b7d, 0x16b8f, 0), + (0x16e40, 0x16e7f, 1), + (0x16f00, 0x16f4a, 0), + (0x16f4f, 0x16f4f, 2), + (0x16f50, 0x16f87, 0), + (0x16f8f, 0x16f9f, 2), + (0x16fe0, 0x16fe1, 2), + (0x16fe3, 0x16fe4, 2), + (0x16ff0, 0x16ff1, 0), + (0x1aff0, 0x1aff3, 6), + (0x1aff5, 0x1affb, 6), + (0x1affd, 0x1affe, 6), + (0x1bc00, 0x1bc6a, 0), + (0x1bc70, 0x1bc7c, 0), + (0x1bc80, 0x1bc88, 0), + (0x1bc90, 0x1bc99, 0), + (0x1bc9d, 0x1bc9e, 2), + (0x1bca0, 0x1bca3, 2), + (0x1cf00, 0x1cf2d, 2), + (0x1cf30, 0x1cf46, 2), + (0x1d165, 0x1d166, 0), + (0x1d167, 0x1d169, 2), + (0x1d16d, 0x1d172, 0), + (0x1d173, 0x1d182, 2), + (0x1d185, 0x1d18b, 2), + (0x1d1aa, 0x1d1ad, 2), + (0x1d242, 0x1d244, 2), + (0x1d400, 0x1d454, 1), + (0x1d456, 0x1d49c, 1), + (0x1d49e, 0x1d49f, 1), + (0x1d4a2, 0x1d4a2, 1), + (0x1d4a5, 0x1d4a6, 1), + (0x1d4a9, 0x1d4ac, 1), + (0x1d4ae, 0x1d4b9, 1), + (0x1d4bb, 0x1d4bb, 1), + (0x1d4bd, 0x1d4c3, 1), + (0x1d4c5, 0x1d505, 1), + (0x1d507, 0x1d50a, 1), + (0x1d50d, 0x1d514, 1), + (0x1d516, 0x1d51c, 1), + (0x1d51e, 0x1d539, 1), + (0x1d53b, 0x1d53e, 1), + (0x1d540, 0x1d544, 1), + (0x1d546, 0x1d546, 1), + (0x1d54a, 0x1d550, 1), + (0x1d552, 0x1d6a5, 1), + (0x1d6a8, 0x1d6c0, 1), + (0x1d6c2, 0x1d6da, 1), + (0x1d6dc, 0x1d6fa, 1), + (0x1d6fc, 0x1d714, 1), + (0x1d716, 0x1d734, 1), + (0x1d736, 0x1d74e, 1), + (0x1d750, 0x1d76e, 1), + (0x1d770, 0x1d788, 1), + (0x1d78a, 0x1d7a8, 1), + (0x1d7aa, 0x1d7c2, 1), + (0x1d7c4, 0x1d7cb, 1), + (0x1d7ce, 0x1d7ff, 0), + (0x1da00, 0x1da36, 2), + (0x1da3b, 0x1da6c, 2), + (0x1da75, 0x1da75, 2), + (0x1da84, 0x1da84, 2), + (0x1da9b, 0x1da9f, 2), + (0x1daa1, 0x1daaf, 2), + (0x1df00, 0x1df09, 1), + (0x1df0a, 0x1df0a, 0), + (0x1df0b, 0x1df1e, 1), + (0x1df25, 0x1df2a, 1), + (0x1e000, 0x1e006, 2), + (0x1e008, 0x1e018, 2), + (0x1e01b, 0x1e021, 2), + (0x1e023, 0x1e024, 2), + (0x1e026, 0x1e02a, 2), + (0x1e030, 0x1e06d, 3), + (0x1e08f, 0x1e08f, 2), + (0x1e100, 0x1e12c, 0), + (0x1e130, 0x1e13d, 2), + (0x1e140, 0x1e149, 0), + (0x1e14e, 0x1e14e, 0), + (0x1e290, 0x1e2ad, 0), + (0x1e2ae, 0x1e2ae, 2), + (0x1e2c0, 0x1e2eb, 0), + (0x1e2ec, 0x1e2ef, 2), + (0x1e2f0, 0x1e2f9, 0), + (0x1e4d0, 0x1e4ea, 0), + (0x1e4eb, 0x1e4ef, 2), + (0x1e4f0, 0x1e4f9, 0), + (0x1e7e0, 0x1e7e6, 0), + (0x1e7e8, 0x1e7eb, 0), + (0x1e7ed, 0x1e7ee, 0), + (0x1e7f0, 0x1e7fe, 0), + (0x1e800, 0x1e8c4, 0), + (0x1e8d0, 0x1e8d6, 2), + (0x1e900, 0x1e943, 1), + (0x1e944, 0x1e94b, 2), + (0x1e950, 0x1e959, 0), + (0x1ee00, 0x1ee03, 0), + (0x1ee05, 0x1ee1f, 0), + (0x1ee21, 0x1ee22, 0), + (0x1ee24, 0x1ee24, 0), + (0x1ee27, 0x1ee27, 0), + (0x1ee29, 0x1ee32, 0), + (0x1ee34, 0x1ee37, 0), + (0x1ee39, 0x1ee39, 0), + (0x1ee3b, 0x1ee3b, 0), + (0x1ee42, 0x1ee42, 0), + (0x1ee47, 0x1ee47, 0), + (0x1ee49, 0x1ee49, 0), + (0x1ee4b, 0x1ee4b, 0), + (0x1ee4d, 0x1ee4f, 0), + (0x1ee51, 0x1ee52, 0), + (0x1ee54, 0x1ee54, 0), + (0x1ee57, 0x1ee57, 0), + (0x1ee59, 0x1ee59, 0), + (0x1ee5b, 0x1ee5b, 0), + (0x1ee5d, 0x1ee5d, 0), + (0x1ee5f, 0x1ee5f, 0), + (0x1ee61, 0x1ee62, 0), + (0x1ee64, 0x1ee64, 0), + (0x1ee67, 0x1ee6a, 0), + (0x1ee6c, 0x1ee72, 0), + (0x1ee74, 0x1ee77, 0), + (0x1ee79, 0x1ee7c, 0), + (0x1ee7e, 0x1ee7e, 0), + (0x1ee80, 0x1ee89, 0), + (0x1ee8b, 0x1ee9b, 0), + (0x1eea1, 0x1eea3, 0), + (0x1eea5, 0x1eea9, 0), + (0x1eeab, 0x1eebb, 0), + (0x1f130, 0x1f149, 1), + (0x1f150, 0x1f169, 1), + (0x1f170, 0x1f189, 1), + (0x1f3fb, 0x1f3ff, 2), + (0x1fbf0, 0x1fbf9, 0), + (0xe0001, 0xe0001, 2), + (0xe0020, 0xe007f, 2), + (0xe0100, 0xe01ef, 2), +]; + +// An empty mapping preserves the input rune. +#[rustfmt::skip] +pub(super) const MAPPINGS: &[(u32, &str, &str)] = &[ + (0x41, "", "a"), + (0x42, "", "b"), + (0x43, "", "c"), + (0x44, "", "d"), + (0x45, "", "e"), + (0x46, "", "f"), + (0x47, "", "g"), + (0x48, "", "h"), + (0x49, "", "i"), + (0x4a, "", "j"), + (0x4b, "", "k"), + (0x4c, "", "l"), + (0x4d, "", "m"), + (0x4e, "", "n"), + (0x4f, "", "o"), + (0x50, "", "p"), + (0x51, "", "q"), + (0x52, "", "r"), + (0x53, "", "s"), + (0x54, "", "t"), + (0x55, "", "u"), + (0x56, "", "v"), + (0x57, "", "w"), + (0x58, "", "x"), + (0x59, "", "y"), + (0x5a, "", "z"), + (0x61, "A", ""), + (0x62, "B", ""), + (0x63, "C", ""), + (0x64, "D", ""), + (0x65, "E", ""), + (0x66, "F", ""), + (0x67, "G", ""), + (0x68, "H", ""), + (0x69, "I", ""), + (0x6a, "J", ""), + (0x6b, "K", ""), + (0x6c, "L", ""), + (0x6d, "M", ""), + (0x6e, "N", ""), + (0x6f, "O", ""), + (0x70, "P", ""), + (0x71, "Q", ""), + (0x72, "R", ""), + (0x73, "S", ""), + (0x74, "T", ""), + (0x75, "U", ""), + (0x76, "V", ""), + (0x77, "W", ""), + (0x78, "X", ""), + (0x79, "Y", ""), + (0x7a, "Z", ""), + (0xb5, "Μ", ""), + (0xc0, "", "à"), + (0xc1, "", "á"), + (0xc2, "", "â"), + (0xc3, "", "ã"), + (0xc4, "", "ä"), + (0xc5, "", "å"), + (0xc6, "", "æ"), + (0xc7, "", "ç"), + (0xc8, "", "è"), + (0xc9, "", "é"), + (0xca, "", "ê"), + (0xcb, "", "ë"), + (0xcc, "", "ì"), + (0xcd, "", "í"), + (0xce, "", "î"), + (0xcf, "", "ï"), + (0xd0, "", "ð"), + (0xd1, "", "ñ"), + (0xd2, "", "ò"), + (0xd3, "", "ó"), + (0xd4, "", "ô"), + (0xd5, "", "õ"), + (0xd6, "", "ö"), + (0xd8, "", "ø"), + (0xd9, "", "ù"), + (0xda, "", "ú"), + (0xdb, "", "û"), + (0xdc, "", "ü"), + (0xdd, "", "ý"), + (0xde, "", "þ"), + (0xdf, "Ss", ""), + (0xe0, "À", ""), + (0xe1, "Á", ""), + (0xe2, "Â", ""), + (0xe3, "Ã", ""), + (0xe4, "Ä", ""), + (0xe5, "Å", ""), + (0xe6, "Æ", ""), + (0xe7, "Ç", ""), + (0xe8, "È", ""), + (0xe9, "É", ""), + (0xea, "Ê", ""), + (0xeb, "Ë", ""), + (0xec, "Ì", ""), + (0xed, "Í", ""), + (0xee, "Î", ""), + (0xef, "Ï", ""), + (0xf0, "Ð", ""), + (0xf1, "Ñ", ""), + (0xf2, "Ò", ""), + (0xf3, "Ó", ""), + (0xf4, "Ô", ""), + (0xf5, "Õ", ""), + (0xf6, "Ö", ""), + (0xf8, "Ø", ""), + (0xf9, "Ù", ""), + (0xfa, "Ú", ""), + (0xfb, "Û", ""), + (0xfc, "Ü", ""), + (0xfd, "Ý", ""), + (0xfe, "Þ", ""), + (0xff, "Ÿ", ""), + (0x100, "", "ā"), + (0x101, "Ā", ""), + (0x102, "", "ă"), + (0x103, "Ă", ""), + (0x104, "", "ą"), + (0x105, "Ą", ""), + (0x106, "", "ć"), + (0x107, "Ć", ""), + (0x108, "", "ĉ"), + (0x109, "Ĉ", ""), + (0x10a, "", "ċ"), + (0x10b, "Ċ", ""), + (0x10c, "", "č"), + (0x10d, "Č", ""), + (0x10e, "", "ď"), + (0x10f, "Ď", ""), + (0x110, "", "đ"), + (0x111, "Đ", ""), + (0x112, "", "ē"), + (0x113, "Ē", ""), + (0x114, "", "ĕ"), + (0x115, "Ĕ", ""), + (0x116, "", "ė"), + (0x117, "Ė", ""), + (0x118, "", "ę"), + (0x119, "Ę", ""), + (0x11a, "", "ě"), + (0x11b, "Ě", ""), + (0x11c, "", "ĝ"), + (0x11d, "Ĝ", ""), + (0x11e, "", "ğ"), + (0x11f, "Ğ", ""), + (0x120, "", "ġ"), + (0x121, "Ġ", ""), + (0x122, "", "ģ"), + (0x123, "Ģ", ""), + (0x124, "", "ĥ"), + (0x125, "Ĥ", ""), + (0x126, "", "ħ"), + (0x127, "Ħ", ""), + (0x128, "", "ĩ"), + (0x129, "Ĩ", ""), + (0x12a, "", "ī"), + (0x12b, "Ī", ""), + (0x12c, "", "ĭ"), + (0x12d, "Ĭ", ""), + (0x12e, "", "į"), + (0x12f, "Į", ""), + (0x130, "", "i̇"), + (0x131, "I", ""), + (0x132, "", "ij"), + (0x133, "IJ", ""), + (0x134, "", "ĵ"), + (0x135, "Ĵ", ""), + (0x136, "", "ķ"), + (0x137, "Ķ", ""), + (0x139, "", "ĺ"), + (0x13a, "Ĺ", ""), + (0x13b, "", "ļ"), + (0x13c, "Ļ", ""), + (0x13d, "", "ľ"), + (0x13e, "Ľ", ""), + (0x13f, "", "ŀ"), + (0x140, "Ŀ", ""), + (0x141, "", "ł"), + (0x142, "Ł", ""), + (0x143, "", "ń"), + (0x144, "Ń", ""), + (0x145, "", "ņ"), + (0x146, "Ņ", ""), + (0x147, "", "ň"), + (0x148, "Ň", ""), + (0x149, "ʼN", ""), + (0x14a, "", "ŋ"), + (0x14b, "Ŋ", ""), + (0x14c, "", "ō"), + (0x14d, "Ō", ""), + (0x14e, "", "ŏ"), + (0x14f, "Ŏ", ""), + (0x150, "", "ő"), + (0x151, "Ő", ""), + (0x152, "", "œ"), + (0x153, "Œ", ""), + (0x154, "", "ŕ"), + (0x155, "Ŕ", ""), + (0x156, "", "ŗ"), + (0x157, "Ŗ", ""), + (0x158, "", "ř"), + (0x159, "Ř", ""), + (0x15a, "", "ś"), + (0x15b, "Ś", ""), + (0x15c, "", "ŝ"), + (0x15d, "Ŝ", ""), + (0x15e, "", "ş"), + (0x15f, "Ş", ""), + (0x160, "", "š"), + (0x161, "Š", ""), + (0x162, "", "ţ"), + (0x163, "Ţ", ""), + (0x164, "", "ť"), + (0x165, "Ť", ""), + (0x166, "", "ŧ"), + (0x167, "Ŧ", ""), + (0x168, "", "ũ"), + (0x169, "Ũ", ""), + (0x16a, "", "ū"), + (0x16b, "Ū", ""), + (0x16c, "", "ŭ"), + (0x16d, "Ŭ", ""), + (0x16e, "", "ů"), + (0x16f, "Ů", ""), + (0x170, "", "ű"), + (0x171, "Ű", ""), + (0x172, "", "ų"), + (0x173, "Ų", ""), + (0x174, "", "ŵ"), + (0x175, "Ŵ", ""), + (0x176, "", "ŷ"), + (0x177, "Ŷ", ""), + (0x178, "", "ÿ"), + (0x179, "", "ź"), + (0x17a, "Ź", ""), + (0x17b, "", "ż"), + (0x17c, "Ż", ""), + (0x17d, "", "ž"), + (0x17e, "Ž", ""), + (0x17f, "S", ""), + (0x180, "Ƀ", ""), + (0x181, "", "ɓ"), + (0x182, "", "ƃ"), + (0x183, "Ƃ", ""), + (0x184, "", "ƅ"), + (0x185, "Ƅ", ""), + (0x186, "", "ɔ"), + (0x187, "", "ƈ"), + (0x188, "Ƈ", ""), + (0x189, "", "ɖ"), + (0x18a, "", "ɗ"), + (0x18b, "", "ƌ"), + (0x18c, "Ƌ", ""), + (0x18e, "", "ǝ"), + (0x18f, "", "ə"), + (0x190, "", "ɛ"), + (0x191, "", "ƒ"), + (0x192, "Ƒ", ""), + (0x193, "", "ɠ"), + (0x194, "", "ɣ"), + (0x195, "Ƕ", ""), + (0x196, "", "ɩ"), + (0x197, "", "ɨ"), + (0x198, "", "ƙ"), + (0x199, "Ƙ", ""), + (0x19a, "Ƚ", ""), + (0x19c, "", "ɯ"), + (0x19d, "", "ɲ"), + (0x19e, "Ƞ", ""), + (0x19f, "", "ɵ"), + (0x1a0, "", "ơ"), + (0x1a1, "Ơ", ""), + (0x1a2, "", "ƣ"), + (0x1a3, "Ƣ", ""), + (0x1a4, "", "ƥ"), + (0x1a5, "Ƥ", ""), + (0x1a6, "", "ʀ"), + (0x1a7, "", "ƨ"), + (0x1a8, "Ƨ", ""), + (0x1a9, "", "ʃ"), + (0x1ac, "", "ƭ"), + (0x1ad, "Ƭ", ""), + (0x1ae, "", "ʈ"), + (0x1af, "", "ư"), + (0x1b0, "Ư", ""), + (0x1b1, "", "ʊ"), + (0x1b2, "", "ʋ"), + (0x1b3, "", "ƴ"), + (0x1b4, "Ƴ", ""), + (0x1b5, "", "ƶ"), + (0x1b6, "Ƶ", ""), + (0x1b7, "", "ʒ"), + (0x1b8, "", "ƹ"), + (0x1b9, "Ƹ", ""), + (0x1bc, "", "ƽ"), + (0x1bd, "Ƽ", ""), + (0x1bf, "Ƿ", ""), + (0x1c4, "Dž", "dž"), + (0x1c5, "", "dž"), + (0x1c6, "Dž", ""), + (0x1c7, "Lj", "lj"), + (0x1c8, "", "lj"), + (0x1c9, "Lj", ""), + (0x1ca, "Nj", "nj"), + (0x1cb, "", "nj"), + (0x1cc, "Nj", ""), + (0x1cd, "", "ǎ"), + (0x1ce, "Ǎ", ""), + (0x1cf, "", "ǐ"), + (0x1d0, "Ǐ", ""), + (0x1d1, "", "ǒ"), + (0x1d2, "Ǒ", ""), + (0x1d3, "", "ǔ"), + (0x1d4, "Ǔ", ""), + (0x1d5, "", "ǖ"), + (0x1d6, "Ǖ", ""), + (0x1d7, "", "ǘ"), + (0x1d8, "Ǘ", ""), + (0x1d9, "", "ǚ"), + (0x1da, "Ǚ", ""), + (0x1db, "", "ǜ"), + (0x1dc, "Ǜ", ""), + (0x1dd, "Ǝ", ""), + (0x1de, "", "ǟ"), + (0x1df, "Ǟ", ""), + (0x1e0, "", "ǡ"), + (0x1e1, "Ǡ", ""), + (0x1e2, "", "ǣ"), + (0x1e3, "Ǣ", ""), + (0x1e4, "", "ǥ"), + (0x1e5, "Ǥ", ""), + (0x1e6, "", "ǧ"), + (0x1e7, "Ǧ", ""), + (0x1e8, "", "ǩ"), + (0x1e9, "Ǩ", ""), + (0x1ea, "", "ǫ"), + (0x1eb, "Ǫ", ""), + (0x1ec, "", "ǭ"), + (0x1ed, "Ǭ", ""), + (0x1ee, "", "ǯ"), + (0x1ef, "Ǯ", ""), + (0x1f0, "J̌", ""), + (0x1f1, "Dz", "dz"), + (0x1f2, "", "dz"), + (0x1f3, "Dz", ""), + (0x1f4, "", "ǵ"), + (0x1f5, "Ǵ", ""), + (0x1f6, "", "ƕ"), + (0x1f7, "", "ƿ"), + (0x1f8, "", "ǹ"), + (0x1f9, "Ǹ", ""), + (0x1fa, "", "ǻ"), + (0x1fb, "Ǻ", ""), + (0x1fc, "", "ǽ"), + (0x1fd, "Ǽ", ""), + (0x1fe, "", "ǿ"), + (0x1ff, "Ǿ", ""), + (0x200, "", "ȁ"), + (0x201, "Ȁ", ""), + (0x202, "", "ȃ"), + (0x203, "Ȃ", ""), + (0x204, "", "ȅ"), + (0x205, "Ȅ", ""), + (0x206, "", "ȇ"), + (0x207, "Ȇ", ""), + (0x208, "", "ȉ"), + (0x209, "Ȉ", ""), + (0x20a, "", "ȋ"), + (0x20b, "Ȋ", ""), + (0x20c, "", "ȍ"), + (0x20d, "Ȍ", ""), + (0x20e, "", "ȏ"), + (0x20f, "Ȏ", ""), + (0x210, "", "ȑ"), + (0x211, "Ȑ", ""), + (0x212, "", "ȓ"), + (0x213, "Ȓ", ""), + (0x214, "", "ȕ"), + (0x215, "Ȕ", ""), + (0x216, "", "ȗ"), + (0x217, "Ȗ", ""), + (0x218, "", "ș"), + (0x219, "Ș", ""), + (0x21a, "", "ț"), + (0x21b, "Ț", ""), + (0x21c, "", "ȝ"), + (0x21d, "Ȝ", ""), + (0x21e, "", "ȟ"), + (0x21f, "Ȟ", ""), + (0x220, "", "ƞ"), + (0x222, "", "ȣ"), + (0x223, "Ȣ", ""), + (0x224, "", "ȥ"), + (0x225, "Ȥ", ""), + (0x226, "", "ȧ"), + (0x227, "Ȧ", ""), + (0x228, "", "ȩ"), + (0x229, "Ȩ", ""), + (0x22a, "", "ȫ"), + (0x22b, "Ȫ", ""), + (0x22c, "", "ȭ"), + (0x22d, "Ȭ", ""), + (0x22e, "", "ȯ"), + (0x22f, "Ȯ", ""), + (0x230, "", "ȱ"), + (0x231, "Ȱ", ""), + (0x232, "", "ȳ"), + (0x233, "Ȳ", ""), + (0x23a, "", "ⱥ"), + (0x23b, "", "ȼ"), + (0x23c, "Ȼ", ""), + (0x23d, "", "ƚ"), + (0x23e, "", "ⱦ"), + (0x23f, "Ȿ", ""), + (0x240, "Ɀ", ""), + (0x241, "", "ɂ"), + (0x242, "Ɂ", ""), + (0x243, "", "ƀ"), + (0x244, "", "ʉ"), + (0x245, "", "ʌ"), + (0x246, "", "ɇ"), + (0x247, "Ɇ", ""), + (0x248, "", "ɉ"), + (0x249, "Ɉ", ""), + (0x24a, "", "ɋ"), + (0x24b, "Ɋ", ""), + (0x24c, "", "ɍ"), + (0x24d, "Ɍ", ""), + (0x24e, "", "ɏ"), + (0x24f, "Ɏ", ""), + (0x250, "Ɐ", ""), + (0x251, "Ɑ", ""), + (0x252, "Ɒ", ""), + (0x253, "Ɓ", ""), + (0x254, "Ɔ", ""), + (0x256, "Ɖ", ""), + (0x257, "Ɗ", ""), + (0x259, "Ə", ""), + (0x25b, "Ɛ", ""), + (0x25c, "Ɜ", ""), + (0x260, "Ɠ", ""), + (0x261, "Ɡ", ""), + (0x263, "Ɣ", ""), + (0x265, "Ɥ", ""), + (0x266, "Ɦ", ""), + (0x268, "Ɨ", ""), + (0x269, "Ɩ", ""), + (0x26a, "Ɪ", ""), + (0x26b, "Ɫ", ""), + (0x26c, "Ɬ", ""), + (0x26f, "Ɯ", ""), + (0x271, "Ɱ", ""), + (0x272, "Ɲ", ""), + (0x275, "Ɵ", ""), + (0x27d, "Ɽ", ""), + (0x280, "Ʀ", ""), + (0x282, "Ʂ", ""), + (0x283, "Ʃ", ""), + (0x287, "Ʇ", ""), + (0x288, "Ʈ", ""), + (0x289, "Ʉ", ""), + (0x28a, "Ʊ", ""), + (0x28b, "Ʋ", ""), + (0x28c, "Ʌ", ""), + (0x292, "Ʒ", ""), + (0x29d, "Ʝ", ""), + (0x29e, "Ʞ", ""), + (0x345, "Ι", ""), + (0x370, "", "ͱ"), + (0x371, "Ͱ", ""), + (0x372, "", "ͳ"), + (0x373, "Ͳ", ""), + (0x376, "", "ͷ"), + (0x377, "Ͷ", ""), + (0x37b, "Ͻ", ""), + (0x37c, "Ͼ", ""), + (0x37d, "Ͽ", ""), + (0x37f, "", "ϳ"), + (0x386, "", "ά"), + (0x388, "", "έ"), + (0x389, "", "ή"), + (0x38a, "", "ί"), + (0x38c, "", "ό"), + (0x38e, "", "ύ"), + (0x38f, "", "ώ"), + (0x390, "Ϊ́", ""), + (0x391, "", "α"), + (0x392, "", "β"), + (0x393, "", "γ"), + (0x394, "", "δ"), + (0x395, "", "ε"), + (0x396, "", "ζ"), + (0x397, "", "η"), + (0x398, "", "θ"), + (0x399, "", "ι"), + (0x39a, "", "κ"), + (0x39b, "", "λ"), + (0x39c, "", "μ"), + (0x39d, "", "ν"), + (0x39e, "", "ξ"), + (0x39f, "", "ο"), + (0x3a0, "", "π"), + (0x3a1, "", "ρ"), + (0x3a3, "", "σ"), + (0x3a4, "", "τ"), + (0x3a5, "", "υ"), + (0x3a6, "", "φ"), + (0x3a7, "", "χ"), + (0x3a8, "", "ψ"), + (0x3a9, "", "ω"), + (0x3aa, "", "ϊ"), + (0x3ab, "", "ϋ"), + (0x3ac, "Ά", ""), + (0x3ad, "Έ", ""), + (0x3ae, "Ή", ""), + (0x3af, "Ί", ""), + (0x3b0, "Ϋ́", ""), + (0x3b1, "Α", ""), + (0x3b2, "Β", ""), + (0x3b3, "Γ", ""), + (0x3b4, "Δ", ""), + (0x3b5, "Ε", ""), + (0x3b6, "Ζ", ""), + (0x3b7, "Η", ""), + (0x3b8, "Θ", ""), + (0x3b9, "Ι", ""), + (0x3ba, "Κ", ""), + (0x3bb, "Λ", ""), + (0x3bc, "Μ", ""), + (0x3bd, "Ν", ""), + (0x3be, "Ξ", ""), + (0x3bf, "Ο", ""), + (0x3c0, "Π", ""), + (0x3c1, "Ρ", ""), + (0x3c2, "Σ", ""), + (0x3c3, "Σ", ""), + (0x3c4, "Τ", ""), + (0x3c5, "Υ", ""), + (0x3c6, "Φ", ""), + (0x3c7, "Χ", ""), + (0x3c8, "Ψ", ""), + (0x3c9, "Ω", ""), + (0x3ca, "Ϊ", ""), + (0x3cb, "Ϋ", ""), + (0x3cc, "Ό", ""), + (0x3cd, "Ύ", ""), + (0x3ce, "Ώ", ""), + (0x3cf, "", "ϗ"), + (0x3d0, "Β", ""), + (0x3d1, "Θ", ""), + (0x3d5, "Φ", ""), + (0x3d6, "Π", ""), + (0x3d7, "Ϗ", ""), + (0x3d8, "", "ϙ"), + (0x3d9, "Ϙ", ""), + (0x3da, "", "ϛ"), + (0x3db, "Ϛ", ""), + (0x3dc, "", "ϝ"), + (0x3dd, "Ϝ", ""), + (0x3de, "", "ϟ"), + (0x3df, "Ϟ", ""), + (0x3e0, "", "ϡ"), + (0x3e1, "Ϡ", ""), + (0x3e2, "", "ϣ"), + (0x3e3, "Ϣ", ""), + (0x3e4, "", "ϥ"), + (0x3e5, "Ϥ", ""), + (0x3e6, "", "ϧ"), + (0x3e7, "Ϧ", ""), + (0x3e8, "", "ϩ"), + (0x3e9, "Ϩ", ""), + (0x3ea, "", "ϫ"), + (0x3eb, "Ϫ", ""), + (0x3ec, "", "ϭ"), + (0x3ed, "Ϭ", ""), + (0x3ee, "", "ϯ"), + (0x3ef, "Ϯ", ""), + (0x3f0, "Κ", ""), + (0x3f1, "Ρ", ""), + (0x3f2, "Ϲ", ""), + (0x3f3, "Ϳ", ""), + (0x3f4, "", "θ"), + (0x3f5, "Ε", ""), + (0x3f7, "", "ϸ"), + (0x3f8, "Ϸ", ""), + (0x3f9, "", "ϲ"), + (0x3fa, "", "ϻ"), + (0x3fb, "Ϻ", ""), + (0x3fd, "", "ͻ"), + (0x3fe, "", "ͼ"), + (0x3ff, "", "ͽ"), + (0x400, "", "ѐ"), + (0x401, "", "ё"), + (0x402, "", "ђ"), + (0x403, "", "ѓ"), + (0x404, "", "є"), + (0x405, "", "ѕ"), + (0x406, "", "і"), + (0x407, "", "ї"), + (0x408, "", "ј"), + (0x409, "", "љ"), + (0x40a, "", "њ"), + (0x40b, "", "ћ"), + (0x40c, "", "ќ"), + (0x40d, "", "ѝ"), + (0x40e, "", "ў"), + (0x40f, "", "џ"), + (0x410, "", "а"), + (0x411, "", "б"), + (0x412, "", "в"), + (0x413, "", "г"), + (0x414, "", "д"), + (0x415, "", "е"), + (0x416, "", "ж"), + (0x417, "", "з"), + (0x418, "", "и"), + (0x419, "", "й"), + (0x41a, "", "к"), + (0x41b, "", "л"), + (0x41c, "", "м"), + (0x41d, "", "н"), + (0x41e, "", "о"), + (0x41f, "", "п"), + (0x420, "", "р"), + (0x421, "", "с"), + (0x422, "", "т"), + (0x423, "", "у"), + (0x424, "", "ф"), + (0x425, "", "х"), + (0x426, "", "ц"), + (0x427, "", "ч"), + (0x428, "", "ш"), + (0x429, "", "щ"), + (0x42a, "", "ъ"), + (0x42b, "", "ы"), + (0x42c, "", "ь"), + (0x42d, "", "э"), + (0x42e, "", "ю"), + (0x42f, "", "я"), + (0x430, "А", ""), + (0x431, "Б", ""), + (0x432, "В", ""), + (0x433, "Г", ""), + (0x434, "Д", ""), + (0x435, "Е", ""), + (0x436, "Ж", ""), + (0x437, "З", ""), + (0x438, "И", ""), + (0x439, "Й", ""), + (0x43a, "К", ""), + (0x43b, "Л", ""), + (0x43c, "М", ""), + (0x43d, "Н", ""), + (0x43e, "О", ""), + (0x43f, "П", ""), + (0x440, "Р", ""), + (0x441, "С", ""), + (0x442, "Т", ""), + (0x443, "У", ""), + (0x444, "Ф", ""), + (0x445, "Х", ""), + (0x446, "Ц", ""), + (0x447, "Ч", ""), + (0x448, "Ш", ""), + (0x449, "Щ", ""), + (0x44a, "Ъ", ""), + (0x44b, "Ы", ""), + (0x44c, "Ь", ""), + (0x44d, "Э", ""), + (0x44e, "Ю", ""), + (0x44f, "Я", ""), + (0x450, "Ѐ", ""), + (0x451, "Ё", ""), + (0x452, "Ђ", ""), + (0x453, "Ѓ", ""), + (0x454, "Є", ""), + (0x455, "Ѕ", ""), + (0x456, "І", ""), + (0x457, "Ї", ""), + (0x458, "Ј", ""), + (0x459, "Љ", ""), + (0x45a, "Њ", ""), + (0x45b, "Ћ", ""), + (0x45c, "Ќ", ""), + (0x45d, "Ѝ", ""), + (0x45e, "Ў", ""), + (0x45f, "Џ", ""), + (0x460, "", "ѡ"), + (0x461, "Ѡ", ""), + (0x462, "", "ѣ"), + (0x463, "Ѣ", ""), + (0x464, "", "ѥ"), + (0x465, "Ѥ", ""), + (0x466, "", "ѧ"), + (0x467, "Ѧ", ""), + (0x468, "", "ѩ"), + (0x469, "Ѩ", ""), + (0x46a, "", "ѫ"), + (0x46b, "Ѫ", ""), + (0x46c, "", "ѭ"), + (0x46d, "Ѭ", ""), + (0x46e, "", "ѯ"), + (0x46f, "Ѯ", ""), + (0x470, "", "ѱ"), + (0x471, "Ѱ", ""), + (0x472, "", "ѳ"), + (0x473, "Ѳ", ""), + (0x474, "", "ѵ"), + (0x475, "Ѵ", ""), + (0x476, "", "ѷ"), + (0x477, "Ѷ", ""), + (0x478, "", "ѹ"), + (0x479, "Ѹ", ""), + (0x47a, "", "ѻ"), + (0x47b, "Ѻ", ""), + (0x47c, "", "ѽ"), + (0x47d, "Ѽ", ""), + (0x47e, "", "ѿ"), + (0x47f, "Ѿ", ""), + (0x480, "", "ҁ"), + (0x481, "Ҁ", ""), + (0x48a, "", "ҋ"), + (0x48b, "Ҋ", ""), + (0x48c, "", "ҍ"), + (0x48d, "Ҍ", ""), + (0x48e, "", "ҏ"), + (0x48f, "Ҏ", ""), + (0x490, "", "ґ"), + (0x491, "Ґ", ""), + (0x492, "", "ғ"), + (0x493, "Ғ", ""), + (0x494, "", "ҕ"), + (0x495, "Ҕ", ""), + (0x496, "", "җ"), + (0x497, "Җ", ""), + (0x498, "", "ҙ"), + (0x499, "Ҙ", ""), + (0x49a, "", "қ"), + (0x49b, "Қ", ""), + (0x49c, "", "ҝ"), + (0x49d, "Ҝ", ""), + (0x49e, "", "ҟ"), + (0x49f, "Ҟ", ""), + (0x4a0, "", "ҡ"), + (0x4a1, "Ҡ", ""), + (0x4a2, "", "ң"), + (0x4a3, "Ң", ""), + (0x4a4, "", "ҥ"), + (0x4a5, "Ҥ", ""), + (0x4a6, "", "ҧ"), + (0x4a7, "Ҧ", ""), + (0x4a8, "", "ҩ"), + (0x4a9, "Ҩ", ""), + (0x4aa, "", "ҫ"), + (0x4ab, "Ҫ", ""), + (0x4ac, "", "ҭ"), + (0x4ad, "Ҭ", ""), + (0x4ae, "", "ү"), + (0x4af, "Ү", ""), + (0x4b0, "", "ұ"), + (0x4b1, "Ұ", ""), + (0x4b2, "", "ҳ"), + (0x4b3, "Ҳ", ""), + (0x4b4, "", "ҵ"), + (0x4b5, "Ҵ", ""), + (0x4b6, "", "ҷ"), + (0x4b7, "Ҷ", ""), + (0x4b8, "", "ҹ"), + (0x4b9, "Ҹ", ""), + (0x4ba, "", "һ"), + (0x4bb, "Һ", ""), + (0x4bc, "", "ҽ"), + (0x4bd, "Ҽ", ""), + (0x4be, "", "ҿ"), + (0x4bf, "Ҿ", ""), + (0x4c0, "", "ӏ"), + (0x4c1, "", "ӂ"), + (0x4c2, "Ӂ", ""), + (0x4c3, "", "ӄ"), + (0x4c4, "Ӄ", ""), + (0x4c5, "", "ӆ"), + (0x4c6, "Ӆ", ""), + (0x4c7, "", "ӈ"), + (0x4c8, "Ӈ", ""), + (0x4c9, "", "ӊ"), + (0x4ca, "Ӊ", ""), + (0x4cb, "", "ӌ"), + (0x4cc, "Ӌ", ""), + (0x4cd, "", "ӎ"), + (0x4ce, "Ӎ", ""), + (0x4cf, "Ӏ", ""), + (0x4d0, "", "ӑ"), + (0x4d1, "Ӑ", ""), + (0x4d2, "", "ӓ"), + (0x4d3, "Ӓ", ""), + (0x4d4, "", "ӕ"), + (0x4d5, "Ӕ", ""), + (0x4d6, "", "ӗ"), + (0x4d7, "Ӗ", ""), + (0x4d8, "", "ә"), + (0x4d9, "Ә", ""), + (0x4da, "", "ӛ"), + (0x4db, "Ӛ", ""), + (0x4dc, "", "ӝ"), + (0x4dd, "Ӝ", ""), + (0x4de, "", "ӟ"), + (0x4df, "Ӟ", ""), + (0x4e0, "", "ӡ"), + (0x4e1, "Ӡ", ""), + (0x4e2, "", "ӣ"), + (0x4e3, "Ӣ", ""), + (0x4e4, "", "ӥ"), + (0x4e5, "Ӥ", ""), + (0x4e6, "", "ӧ"), + (0x4e7, "Ӧ", ""), + (0x4e8, "", "ө"), + (0x4e9, "Ө", ""), + (0x4ea, "", "ӫ"), + (0x4eb, "Ӫ", ""), + (0x4ec, "", "ӭ"), + (0x4ed, "Ӭ", ""), + (0x4ee, "", "ӯ"), + (0x4ef, "Ӯ", ""), + (0x4f0, "", "ӱ"), + (0x4f1, "Ӱ", ""), + (0x4f2, "", "ӳ"), + (0x4f3, "Ӳ", ""), + (0x4f4, "", "ӵ"), + (0x4f5, "Ӵ", ""), + (0x4f6, "", "ӷ"), + (0x4f7, "Ӷ", ""), + (0x4f8, "", "ӹ"), + (0x4f9, "Ӹ", ""), + (0x4fa, "", "ӻ"), + (0x4fb, "Ӻ", ""), + (0x4fc, "", "ӽ"), + (0x4fd, "Ӽ", ""), + (0x4fe, "", "ӿ"), + (0x4ff, "Ӿ", ""), + (0x500, "", "ԁ"), + (0x501, "Ԁ", ""), + (0x502, "", "ԃ"), + (0x503, "Ԃ", ""), + (0x504, "", "ԅ"), + (0x505, "Ԅ", ""), + (0x506, "", "ԇ"), + (0x507, "Ԇ", ""), + (0x508, "", "ԉ"), + (0x509, "Ԉ", ""), + (0x50a, "", "ԋ"), + (0x50b, "Ԋ", ""), + (0x50c, "", "ԍ"), + (0x50d, "Ԍ", ""), + (0x50e, "", "ԏ"), + (0x50f, "Ԏ", ""), + (0x510, "", "ԑ"), + (0x511, "Ԑ", ""), + (0x512, "", "ԓ"), + (0x513, "Ԓ", ""), + (0x514, "", "ԕ"), + (0x515, "Ԕ", ""), + (0x516, "", "ԗ"), + (0x517, "Ԗ", ""), + (0x518, "", "ԙ"), + (0x519, "Ԙ", ""), + (0x51a, "", "ԛ"), + (0x51b, "Ԛ", ""), + (0x51c, "", "ԝ"), + (0x51d, "Ԝ", ""), + (0x51e, "", "ԟ"), + (0x51f, "Ԟ", ""), + (0x520, "", "ԡ"), + (0x521, "Ԡ", ""), + (0x522, "", "ԣ"), + (0x523, "Ԣ", ""), + (0x524, "", "ԥ"), + (0x525, "Ԥ", ""), + (0x526, "", "ԧ"), + (0x527, "Ԧ", ""), + (0x528, "", "ԩ"), + (0x529, "Ԩ", ""), + (0x52a, "", "ԫ"), + (0x52b, "Ԫ", ""), + (0x52c, "", "ԭ"), + (0x52d, "Ԭ", ""), + (0x52e, "", "ԯ"), + (0x52f, "Ԯ", ""), + (0x531, "", "ա"), + (0x532, "", "բ"), + (0x533, "", "գ"), + (0x534, "", "դ"), + (0x535, "", "ե"), + (0x536, "", "զ"), + (0x537, "", "է"), + (0x538, "", "ը"), + (0x539, "", "թ"), + (0x53a, "", "ժ"), + (0x53b, "", "ի"), + (0x53c, "", "լ"), + (0x53d, "", "խ"), + (0x53e, "", "ծ"), + (0x53f, "", "կ"), + (0x540, "", "հ"), + (0x541, "", "ձ"), + (0x542, "", "ղ"), + (0x543, "", "ճ"), + (0x544, "", "մ"), + (0x545, "", "յ"), + (0x546, "", "ն"), + (0x547, "", "շ"), + (0x548, "", "ո"), + (0x549, "", "չ"), + (0x54a, "", "պ"), + (0x54b, "", "ջ"), + (0x54c, "", "ռ"), + (0x54d, "", "ս"), + (0x54e, "", "վ"), + (0x54f, "", "տ"), + (0x550, "", "ր"), + (0x551, "", "ց"), + (0x552, "", "ւ"), + (0x553, "", "փ"), + (0x554, "", "ք"), + (0x555, "", "օ"), + (0x556, "", "ֆ"), + (0x561, "Ա", ""), + (0x562, "Բ", ""), + (0x563, "Գ", ""), + (0x564, "Դ", ""), + (0x565, "Ե", ""), + (0x566, "Զ", ""), + (0x567, "Է", ""), + (0x568, "Ը", ""), + (0x569, "Թ", ""), + (0x56a, "Ժ", ""), + (0x56b, "Ի", ""), + (0x56c, "Լ", ""), + (0x56d, "Խ", ""), + (0x56e, "Ծ", ""), + (0x56f, "Կ", ""), + (0x570, "Հ", ""), + (0x571, "Ձ", ""), + (0x572, "Ղ", ""), + (0x573, "Ճ", ""), + (0x574, "Մ", ""), + (0x575, "Յ", ""), + (0x576, "Ն", ""), + (0x577, "Շ", ""), + (0x578, "Ո", ""), + (0x579, "Չ", ""), + (0x57a, "Պ", ""), + (0x57b, "Ջ", ""), + (0x57c, "Ռ", ""), + (0x57d, "Ս", ""), + (0x57e, "Վ", ""), + (0x57f, "Տ", ""), + (0x580, "Ր", ""), + (0x581, "Ց", ""), + (0x582, "Ւ", ""), + (0x583, "Փ", ""), + (0x584, "Ք", ""), + (0x585, "Օ", ""), + (0x586, "Ֆ", ""), + (0x587, "Եւ", ""), + (0x10a0, "", "ⴀ"), + (0x10a1, "", "ⴁ"), + (0x10a2, "", "ⴂ"), + (0x10a3, "", "ⴃ"), + (0x10a4, "", "ⴄ"), + (0x10a5, "", "ⴅ"), + (0x10a6, "", "ⴆ"), + (0x10a7, "", "ⴇ"), + (0x10a8, "", "ⴈ"), + (0x10a9, "", "ⴉ"), + (0x10aa, "", "ⴊ"), + (0x10ab, "", "ⴋ"), + (0x10ac, "", "ⴌ"), + (0x10ad, "", "ⴍ"), + (0x10ae, "", "ⴎ"), + (0x10af, "", "ⴏ"), + (0x10b0, "", "ⴐ"), + (0x10b1, "", "ⴑ"), + (0x10b2, "", "ⴒ"), + (0x10b3, "", "ⴓ"), + (0x10b4, "", "ⴔ"), + (0x10b5, "", "ⴕ"), + (0x10b6, "", "ⴖ"), + (0x10b7, "", "ⴗ"), + (0x10b8, "", "ⴘ"), + (0x10b9, "", "ⴙ"), + (0x10ba, "", "ⴚ"), + (0x10bb, "", "ⴛ"), + (0x10bc, "", "ⴜ"), + (0x10bd, "", "ⴝ"), + (0x10be, "", "ⴞ"), + (0x10bf, "", "ⴟ"), + (0x10c0, "", "ⴠ"), + (0x10c1, "", "ⴡ"), + (0x10c2, "", "ⴢ"), + (0x10c3, "", "ⴣ"), + (0x10c4, "", "ⴤ"), + (0x10c5, "", "ⴥ"), + (0x10c7, "", "ⴧ"), + (0x10cd, "", "ⴭ"), + (0x13a0, "", "ꭰ"), + (0x13a1, "", "ꭱ"), + (0x13a2, "", "ꭲ"), + (0x13a3, "", "ꭳ"), + (0x13a4, "", "ꭴ"), + (0x13a5, "", "ꭵ"), + (0x13a6, "", "ꭶ"), + (0x13a7, "", "ꭷ"), + (0x13a8, "", "ꭸ"), + (0x13a9, "", "ꭹ"), + (0x13aa, "", "ꭺ"), + (0x13ab, "", "ꭻ"), + (0x13ac, "", "ꭼ"), + (0x13ad, "", "ꭽ"), + (0x13ae, "", "ꭾ"), + (0x13af, "", "ꭿ"), + (0x13b0, "", "ꮀ"), + (0x13b1, "", "ꮁ"), + (0x13b2, "", "ꮂ"), + (0x13b3, "", "ꮃ"), + (0x13b4, "", "ꮄ"), + (0x13b5, "", "ꮅ"), + (0x13b6, "", "ꮆ"), + (0x13b7, "", "ꮇ"), + (0x13b8, "", "ꮈ"), + (0x13b9, "", "ꮉ"), + (0x13ba, "", "ꮊ"), + (0x13bb, "", "ꮋ"), + (0x13bc, "", "ꮌ"), + (0x13bd, "", "ꮍ"), + (0x13be, "", "ꮎ"), + (0x13bf, "", "ꮏ"), + (0x13c0, "", "ꮐ"), + (0x13c1, "", "ꮑ"), + (0x13c2, "", "ꮒ"), + (0x13c3, "", "ꮓ"), + (0x13c4, "", "ꮔ"), + (0x13c5, "", "ꮕ"), + (0x13c6, "", "ꮖ"), + (0x13c7, "", "ꮗ"), + (0x13c8, "", "ꮘ"), + (0x13c9, "", "ꮙ"), + (0x13ca, "", "ꮚ"), + (0x13cb, "", "ꮛ"), + (0x13cc, "", "ꮜ"), + (0x13cd, "", "ꮝ"), + (0x13ce, "", "ꮞ"), + (0x13cf, "", "ꮟ"), + (0x13d0, "", "ꮠ"), + (0x13d1, "", "ꮡ"), + (0x13d2, "", "ꮢ"), + (0x13d3, "", "ꮣ"), + (0x13d4, "", "ꮤ"), + (0x13d5, "", "ꮥ"), + (0x13d6, "", "ꮦ"), + (0x13d7, "", "ꮧ"), + (0x13d8, "", "ꮨ"), + (0x13d9, "", "ꮩ"), + (0x13da, "", "ꮪ"), + (0x13db, "", "ꮫ"), + (0x13dc, "", "ꮬ"), + (0x13dd, "", "ꮭ"), + (0x13de, "", "ꮮ"), + (0x13df, "", "ꮯ"), + (0x13e0, "", "ꮰ"), + (0x13e1, "", "ꮱ"), + (0x13e2, "", "ꮲ"), + (0x13e3, "", "ꮳ"), + (0x13e4, "", "ꮴ"), + (0x13e5, "", "ꮵ"), + (0x13e6, "", "ꮶ"), + (0x13e7, "", "ꮷ"), + (0x13e8, "", "ꮸ"), + (0x13e9, "", "ꮹ"), + (0x13ea, "", "ꮺ"), + (0x13eb, "", "ꮻ"), + (0x13ec, "", "ꮼ"), + (0x13ed, "", "ꮽ"), + (0x13ee, "", "ꮾ"), + (0x13ef, "", "ꮿ"), + (0x13f0, "", "ᏸ"), + (0x13f1, "", "ᏹ"), + (0x13f2, "", "ᏺ"), + (0x13f3, "", "ᏻ"), + (0x13f4, "", "ᏼ"), + (0x13f5, "", "ᏽ"), + (0x13f8, "Ᏸ", ""), + (0x13f9, "Ᏹ", ""), + (0x13fa, "Ᏺ", ""), + (0x13fb, "Ᏻ", ""), + (0x13fc, "Ᏼ", ""), + (0x13fd, "Ᏽ", ""), + (0x1c80, "В", ""), + (0x1c81, "Д", ""), + (0x1c82, "О", ""), + (0x1c83, "С", ""), + (0x1c84, "Т", ""), + (0x1c85, "Т", ""), + (0x1c86, "Ъ", ""), + (0x1c87, "Ѣ", ""), + (0x1c88, "Ꙋ", ""), + (0x1c90, "", "ა"), + (0x1c91, "", "ბ"), + (0x1c92, "", "გ"), + (0x1c93, "", "დ"), + (0x1c94, "", "ე"), + (0x1c95, "", "ვ"), + (0x1c96, "", "ზ"), + (0x1c97, "", "თ"), + (0x1c98, "", "ი"), + (0x1c99, "", "კ"), + (0x1c9a, "", "ლ"), + (0x1c9b, "", "მ"), + (0x1c9c, "", "ნ"), + (0x1c9d, "", "ო"), + (0x1c9e, "", "პ"), + (0x1c9f, "", "ჟ"), + (0x1ca0, "", "რ"), + (0x1ca1, "", "ს"), + (0x1ca2, "", "ტ"), + (0x1ca3, "", "უ"), + (0x1ca4, "", "ფ"), + (0x1ca5, "", "ქ"), + (0x1ca6, "", "ღ"), + (0x1ca7, "", "ყ"), + (0x1ca8, "", "შ"), + (0x1ca9, "", "ჩ"), + (0x1caa, "", "ც"), + (0x1cab, "", "ძ"), + (0x1cac, "", "წ"), + (0x1cad, "", "ჭ"), + (0x1cae, "", "ხ"), + (0x1caf, "", "ჯ"), + (0x1cb0, "", "ჰ"), + (0x1cb1, "", "ჱ"), + (0x1cb2, "", "ჲ"), + (0x1cb3, "", "ჳ"), + (0x1cb4, "", "ჴ"), + (0x1cb5, "", "ჵ"), + (0x1cb6, "", "ჶ"), + (0x1cb7, "", "ჷ"), + (0x1cb8, "", "ჸ"), + (0x1cb9, "", "ჹ"), + (0x1cba, "", "ჺ"), + (0x1cbd, "", "ჽ"), + (0x1cbe, "", "ჾ"), + (0x1cbf, "", "ჿ"), + (0x1d79, "Ᵹ", ""), + (0x1d7d, "Ᵽ", ""), + (0x1d8e, "Ᶎ", ""), + (0x1e00, "", "ḁ"), + (0x1e01, "Ḁ", ""), + (0x1e02, "", "ḃ"), + (0x1e03, "Ḃ", ""), + (0x1e04, "", "ḅ"), + (0x1e05, "Ḅ", ""), + (0x1e06, "", "ḇ"), + (0x1e07, "Ḇ", ""), + (0x1e08, "", "ḉ"), + (0x1e09, "Ḉ", ""), + (0x1e0a, "", "ḋ"), + (0x1e0b, "Ḋ", ""), + (0x1e0c, "", "ḍ"), + (0x1e0d, "Ḍ", ""), + (0x1e0e, "", "ḏ"), + (0x1e0f, "Ḏ", ""), + (0x1e10, "", "ḑ"), + (0x1e11, "Ḑ", ""), + (0x1e12, "", "ḓ"), + (0x1e13, "Ḓ", ""), + (0x1e14, "", "ḕ"), + (0x1e15, "Ḕ", ""), + (0x1e16, "", "ḗ"), + (0x1e17, "Ḗ", ""), + (0x1e18, "", "ḙ"), + (0x1e19, "Ḙ", ""), + (0x1e1a, "", "ḛ"), + (0x1e1b, "Ḛ", ""), + (0x1e1c, "", "ḝ"), + (0x1e1d, "Ḝ", ""), + (0x1e1e, "", "ḟ"), + (0x1e1f, "Ḟ", ""), + (0x1e20, "", "ḡ"), + (0x1e21, "Ḡ", ""), + (0x1e22, "", "ḣ"), + (0x1e23, "Ḣ", ""), + (0x1e24, "", "ḥ"), + (0x1e25, "Ḥ", ""), + (0x1e26, "", "ḧ"), + (0x1e27, "Ḧ", ""), + (0x1e28, "", "ḩ"), + (0x1e29, "Ḩ", ""), + (0x1e2a, "", "ḫ"), + (0x1e2b, "Ḫ", ""), + (0x1e2c, "", "ḭ"), + (0x1e2d, "Ḭ", ""), + (0x1e2e, "", "ḯ"), + (0x1e2f, "Ḯ", ""), + (0x1e30, "", "ḱ"), + (0x1e31, "Ḱ", ""), + (0x1e32, "", "ḳ"), + (0x1e33, "Ḳ", ""), + (0x1e34, "", "ḵ"), + (0x1e35, "Ḵ", ""), + (0x1e36, "", "ḷ"), + (0x1e37, "Ḷ", ""), + (0x1e38, "", "ḹ"), + (0x1e39, "Ḹ", ""), + (0x1e3a, "", "ḻ"), + (0x1e3b, "Ḻ", ""), + (0x1e3c, "", "ḽ"), + (0x1e3d, "Ḽ", ""), + (0x1e3e, "", "ḿ"), + (0x1e3f, "Ḿ", ""), + (0x1e40, "", "ṁ"), + (0x1e41, "Ṁ", ""), + (0x1e42, "", "ṃ"), + (0x1e43, "Ṃ", ""), + (0x1e44, "", "ṅ"), + (0x1e45, "Ṅ", ""), + (0x1e46, "", "ṇ"), + (0x1e47, "Ṇ", ""), + (0x1e48, "", "ṉ"), + (0x1e49, "Ṉ", ""), + (0x1e4a, "", "ṋ"), + (0x1e4b, "Ṋ", ""), + (0x1e4c, "", "ṍ"), + (0x1e4d, "Ṍ", ""), + (0x1e4e, "", "ṏ"), + (0x1e4f, "Ṏ", ""), + (0x1e50, "", "ṑ"), + (0x1e51, "Ṑ", ""), + (0x1e52, "", "ṓ"), + (0x1e53, "Ṓ", ""), + (0x1e54, "", "ṕ"), + (0x1e55, "Ṕ", ""), + (0x1e56, "", "ṗ"), + (0x1e57, "Ṗ", ""), + (0x1e58, "", "ṙ"), + (0x1e59, "Ṙ", ""), + (0x1e5a, "", "ṛ"), + (0x1e5b, "Ṛ", ""), + (0x1e5c, "", "ṝ"), + (0x1e5d, "Ṝ", ""), + (0x1e5e, "", "ṟ"), + (0x1e5f, "Ṟ", ""), + (0x1e60, "", "ṡ"), + (0x1e61, "Ṡ", ""), + (0x1e62, "", "ṣ"), + (0x1e63, "Ṣ", ""), + (0x1e64, "", "ṥ"), + (0x1e65, "Ṥ", ""), + (0x1e66, "", "ṧ"), + (0x1e67, "Ṧ", ""), + (0x1e68, "", "ṩ"), + (0x1e69, "Ṩ", ""), + (0x1e6a, "", "ṫ"), + (0x1e6b, "Ṫ", ""), + (0x1e6c, "", "ṭ"), + (0x1e6d, "Ṭ", ""), + (0x1e6e, "", "ṯ"), + (0x1e6f, "Ṯ", ""), + (0x1e70, "", "ṱ"), + (0x1e71, "Ṱ", ""), + (0x1e72, "", "ṳ"), + (0x1e73, "Ṳ", ""), + (0x1e74, "", "ṵ"), + (0x1e75, "Ṵ", ""), + (0x1e76, "", "ṷ"), + (0x1e77, "Ṷ", ""), + (0x1e78, "", "ṹ"), + (0x1e79, "Ṹ", ""), + (0x1e7a, "", "ṻ"), + (0x1e7b, "Ṻ", ""), + (0x1e7c, "", "ṽ"), + (0x1e7d, "Ṽ", ""), + (0x1e7e, "", "ṿ"), + (0x1e7f, "Ṿ", ""), + (0x1e80, "", "ẁ"), + (0x1e81, "Ẁ", ""), + (0x1e82, "", "ẃ"), + (0x1e83, "Ẃ", ""), + (0x1e84, "", "ẅ"), + (0x1e85, "Ẅ", ""), + (0x1e86, "", "ẇ"), + (0x1e87, "Ẇ", ""), + (0x1e88, "", "ẉ"), + (0x1e89, "Ẉ", ""), + (0x1e8a, "", "ẋ"), + (0x1e8b, "Ẋ", ""), + (0x1e8c, "", "ẍ"), + (0x1e8d, "Ẍ", ""), + (0x1e8e, "", "ẏ"), + (0x1e8f, "Ẏ", ""), + (0x1e90, "", "ẑ"), + (0x1e91, "Ẑ", ""), + (0x1e92, "", "ẓ"), + (0x1e93, "Ẓ", ""), + (0x1e94, "", "ẕ"), + (0x1e95, "Ẕ", ""), + (0x1e96, "H̱", ""), + (0x1e97, "T̈", ""), + (0x1e98, "W̊", ""), + (0x1e99, "Y̊", ""), + (0x1e9a, "Aʾ", ""), + (0x1e9b, "Ṡ", ""), + (0x1e9e, "", "ß"), + (0x1ea0, "", "ạ"), + (0x1ea1, "Ạ", ""), + (0x1ea2, "", "ả"), + (0x1ea3, "Ả", ""), + (0x1ea4, "", "ấ"), + (0x1ea5, "Ấ", ""), + (0x1ea6, "", "ầ"), + (0x1ea7, "Ầ", ""), + (0x1ea8, "", "ẩ"), + (0x1ea9, "Ẩ", ""), + (0x1eaa, "", "ẫ"), + (0x1eab, "Ẫ", ""), + (0x1eac, "", "ậ"), + (0x1ead, "Ậ", ""), + (0x1eae, "", "ắ"), + (0x1eaf, "Ắ", ""), + (0x1eb0, "", "ằ"), + (0x1eb1, "Ằ", ""), + (0x1eb2, "", "ẳ"), + (0x1eb3, "Ẳ", ""), + (0x1eb4, "", "ẵ"), + (0x1eb5, "Ẵ", ""), + (0x1eb6, "", "ặ"), + (0x1eb7, "Ặ", ""), + (0x1eb8, "", "ẹ"), + (0x1eb9, "Ẹ", ""), + (0x1eba, "", "ẻ"), + (0x1ebb, "Ẻ", ""), + (0x1ebc, "", "ẽ"), + (0x1ebd, "Ẽ", ""), + (0x1ebe, "", "ế"), + (0x1ebf, "Ế", ""), + (0x1ec0, "", "ề"), + (0x1ec1, "Ề", ""), + (0x1ec2, "", "ể"), + (0x1ec3, "Ể", ""), + (0x1ec4, "", "ễ"), + (0x1ec5, "Ễ", ""), + (0x1ec6, "", "ệ"), + (0x1ec7, "Ệ", ""), + (0x1ec8, "", "ỉ"), + (0x1ec9, "Ỉ", ""), + (0x1eca, "", "ị"), + (0x1ecb, "Ị", ""), + (0x1ecc, "", "ọ"), + (0x1ecd, "Ọ", ""), + (0x1ece, "", "ỏ"), + (0x1ecf, "Ỏ", ""), + (0x1ed0, "", "ố"), + (0x1ed1, "Ố", ""), + (0x1ed2, "", "ồ"), + (0x1ed3, "Ồ", ""), + (0x1ed4, "", "ổ"), + (0x1ed5, "Ổ", ""), + (0x1ed6, "", "ỗ"), + (0x1ed7, "Ỗ", ""), + (0x1ed8, "", "ộ"), + (0x1ed9, "Ộ", ""), + (0x1eda, "", "ớ"), + (0x1edb, "Ớ", ""), + (0x1edc, "", "ờ"), + (0x1edd, "Ờ", ""), + (0x1ede, "", "ở"), + (0x1edf, "Ở", ""), + (0x1ee0, "", "ỡ"), + (0x1ee1, "Ỡ", ""), + (0x1ee2, "", "ợ"), + (0x1ee3, "Ợ", ""), + (0x1ee4, "", "ụ"), + (0x1ee5, "Ụ", ""), + (0x1ee6, "", "ủ"), + (0x1ee7, "Ủ", ""), + (0x1ee8, "", "ứ"), + (0x1ee9, "Ứ", ""), + (0x1eea, "", "ừ"), + (0x1eeb, "Ừ", ""), + (0x1eec, "", "ử"), + (0x1eed, "Ử", ""), + (0x1eee, "", "ữ"), + (0x1eef, "Ữ", ""), + (0x1ef0, "", "ự"), + (0x1ef1, "Ự", ""), + (0x1ef2, "", "ỳ"), + (0x1ef3, "Ỳ", ""), + (0x1ef4, "", "ỵ"), + (0x1ef5, "Ỵ", ""), + (0x1ef6, "", "ỷ"), + (0x1ef7, "Ỷ", ""), + (0x1ef8, "", "ỹ"), + (0x1ef9, "Ỹ", ""), + (0x1efa, "", "ỻ"), + (0x1efb, "Ỻ", ""), + (0x1efc, "", "ỽ"), + (0x1efd, "Ỽ", ""), + (0x1efe, "", "ỿ"), + (0x1eff, "Ỿ", ""), + (0x1f00, "Ἀ", ""), + (0x1f01, "Ἁ", ""), + (0x1f02, "Ἂ", ""), + (0x1f03, "Ἃ", ""), + (0x1f04, "Ἄ", ""), + (0x1f05, "Ἅ", ""), + (0x1f06, "Ἆ", ""), + (0x1f07, "Ἇ", ""), + (0x1f08, "", "ἀ"), + (0x1f09, "", "ἁ"), + (0x1f0a, "", "ἂ"), + (0x1f0b, "", "ἃ"), + (0x1f0c, "", "ἄ"), + (0x1f0d, "", "ἅ"), + (0x1f0e, "", "ἆ"), + (0x1f0f, "", "ἇ"), + (0x1f10, "Ἐ", ""), + (0x1f11, "Ἑ", ""), + (0x1f12, "Ἒ", ""), + (0x1f13, "Ἓ", ""), + (0x1f14, "Ἔ", ""), + (0x1f15, "Ἕ", ""), + (0x1f18, "", "ἐ"), + (0x1f19, "", "ἑ"), + (0x1f1a, "", "ἒ"), + (0x1f1b, "", "ἓ"), + (0x1f1c, "", "ἔ"), + (0x1f1d, "", "ἕ"), + (0x1f20, "Ἠ", ""), + (0x1f21, "Ἡ", ""), + (0x1f22, "Ἢ", ""), + (0x1f23, "Ἣ", ""), + (0x1f24, "Ἤ", ""), + (0x1f25, "Ἥ", ""), + (0x1f26, "Ἦ", ""), + (0x1f27, "Ἧ", ""), + (0x1f28, "", "ἠ"), + (0x1f29, "", "ἡ"), + (0x1f2a, "", "ἢ"), + (0x1f2b, "", "ἣ"), + (0x1f2c, "", "ἤ"), + (0x1f2d, "", "ἥ"), + (0x1f2e, "", "ἦ"), + (0x1f2f, "", "ἧ"), + (0x1f30, "Ἰ", ""), + (0x1f31, "Ἱ", ""), + (0x1f32, "Ἲ", ""), + (0x1f33, "Ἳ", ""), + (0x1f34, "Ἴ", ""), + (0x1f35, "Ἵ", ""), + (0x1f36, "Ἶ", ""), + (0x1f37, "Ἷ", ""), + (0x1f38, "", "ἰ"), + (0x1f39, "", "ἱ"), + (0x1f3a, "", "ἲ"), + (0x1f3b, "", "ἳ"), + (0x1f3c, "", "ἴ"), + (0x1f3d, "", "ἵ"), + (0x1f3e, "", "ἶ"), + (0x1f3f, "", "ἷ"), + (0x1f40, "Ὀ", ""), + (0x1f41, "Ὁ", ""), + (0x1f42, "Ὂ", ""), + (0x1f43, "Ὃ", ""), + (0x1f44, "Ὄ", ""), + (0x1f45, "Ὅ", ""), + (0x1f48, "", "ὀ"), + (0x1f49, "", "ὁ"), + (0x1f4a, "", "ὂ"), + (0x1f4b, "", "ὃ"), + (0x1f4c, "", "ὄ"), + (0x1f4d, "", "ὅ"), + (0x1f50, "Υ̓", ""), + (0x1f51, "Ὑ", ""), + (0x1f52, "Υ̓̀", ""), + (0x1f53, "Ὓ", ""), + (0x1f54, "Υ̓́", ""), + (0x1f55, "Ὕ", ""), + (0x1f56, "Υ̓͂", ""), + (0x1f57, "Ὗ", ""), + (0x1f59, "", "ὑ"), + (0x1f5b, "", "ὓ"), + (0x1f5d, "", "ὕ"), + (0x1f5f, "", "ὗ"), + (0x1f60, "Ὠ", ""), + (0x1f61, "Ὡ", ""), + (0x1f62, "Ὢ", ""), + (0x1f63, "Ὣ", ""), + (0x1f64, "Ὤ", ""), + (0x1f65, "Ὥ", ""), + (0x1f66, "Ὦ", ""), + (0x1f67, "Ὧ", ""), + (0x1f68, "", "ὠ"), + (0x1f69, "", "ὡ"), + (0x1f6a, "", "ὢ"), + (0x1f6b, "", "ὣ"), + (0x1f6c, "", "ὤ"), + (0x1f6d, "", "ὥ"), + (0x1f6e, "", "ὦ"), + (0x1f6f, "", "ὧ"), + (0x1f70, "Ὰ", ""), + (0x1f71, "Ά", ""), + (0x1f72, "Ὲ", ""), + (0x1f73, "Έ", ""), + (0x1f74, "Ὴ", ""), + (0x1f75, "Ή", ""), + (0x1f76, "Ὶ", ""), + (0x1f77, "Ί", ""), + (0x1f78, "Ὸ", ""), + (0x1f79, "Ό", ""), + (0x1f7a, "Ὺ", ""), + (0x1f7b, "Ύ", ""), + (0x1f7c, "Ὼ", ""), + (0x1f7d, "Ώ", ""), + (0x1f80, "ᾈ", ""), + (0x1f81, "ᾉ", ""), + (0x1f82, "ᾊ", ""), + (0x1f83, "ᾋ", ""), + (0x1f84, "ᾌ", ""), + (0x1f85, "ᾍ", ""), + (0x1f86, "ᾎ", ""), + (0x1f87, "ᾏ", ""), + (0x1f88, "", "ᾀ"), + (0x1f89, "", "ᾁ"), + (0x1f8a, "", "ᾂ"), + (0x1f8b, "", "ᾃ"), + (0x1f8c, "", "ᾄ"), + (0x1f8d, "", "ᾅ"), + (0x1f8e, "", "ᾆ"), + (0x1f8f, "", "ᾇ"), + (0x1f90, "ᾘ", ""), + (0x1f91, "ᾙ", ""), + (0x1f92, "ᾚ", ""), + (0x1f93, "ᾛ", ""), + (0x1f94, "ᾜ", ""), + (0x1f95, "ᾝ", ""), + (0x1f96, "ᾞ", ""), + (0x1f97, "ᾟ", ""), + (0x1f98, "", "ᾐ"), + (0x1f99, "", "ᾑ"), + (0x1f9a, "", "ᾒ"), + (0x1f9b, "", "ᾓ"), + (0x1f9c, "", "ᾔ"), + (0x1f9d, "", "ᾕ"), + (0x1f9e, "", "ᾖ"), + (0x1f9f, "", "ᾗ"), + (0x1fa0, "ᾨ", ""), + (0x1fa1, "ᾩ", ""), + (0x1fa2, "ᾪ", ""), + (0x1fa3, "ᾫ", ""), + (0x1fa4, "ᾬ", ""), + (0x1fa5, "ᾭ", ""), + (0x1fa6, "ᾮ", ""), + (0x1fa7, "ᾯ", ""), + (0x1fa8, "", "ᾠ"), + (0x1fa9, "", "ᾡ"), + (0x1faa, "", "ᾢ"), + (0x1fab, "", "ᾣ"), + (0x1fac, "", "ᾤ"), + (0x1fad, "", "ᾥ"), + (0x1fae, "", "ᾦ"), + (0x1faf, "", "ᾧ"), + (0x1fb0, "Ᾰ", ""), + (0x1fb1, "Ᾱ", ""), + (0x1fb2, "Ὰͅ", ""), + (0x1fb3, "ᾼ", ""), + (0x1fb4, "Άͅ", ""), + (0x1fb6, "Α͂", ""), + (0x1fb7, "ᾼ͂", ""), + (0x1fb8, "", "ᾰ"), + (0x1fb9, "", "ᾱ"), + (0x1fba, "", "ὰ"), + (0x1fbb, "", "ά"), + (0x1fbc, "", "ᾳ"), + (0x1fbe, "Ι", ""), + (0x1fc2, "Ὴͅ", ""), + (0x1fc3, "ῌ", ""), + (0x1fc4, "Ήͅ", ""), + (0x1fc6, "Η͂", ""), + (0x1fc7, "ῌ͂", ""), + (0x1fc8, "", "ὲ"), + (0x1fc9, "", "έ"), + (0x1fca, "", "ὴ"), + (0x1fcb, "", "ή"), + (0x1fcc, "", "ῃ"), + (0x1fd0, "Ῐ", ""), + (0x1fd1, "Ῑ", ""), + (0x1fd2, "Ϊ̀", ""), + (0x1fd3, "Ϊ́", ""), + (0x1fd6, "Ι͂", ""), + (0x1fd7, "Ϊ͂", ""), + (0x1fd8, "", "ῐ"), + (0x1fd9, "", "ῑ"), + (0x1fda, "", "ὶ"), + (0x1fdb, "", "ί"), + (0x1fe0, "Ῠ", ""), + (0x1fe1, "Ῡ", ""), + (0x1fe2, "Ϋ̀", ""), + (0x1fe3, "Ϋ́", ""), + (0x1fe4, "Ρ̓", ""), + (0x1fe5, "Ῥ", ""), + (0x1fe6, "Υ͂", ""), + (0x1fe7, "Ϋ͂", ""), + (0x1fe8, "", "ῠ"), + (0x1fe9, "", "ῡ"), + (0x1fea, "", "ὺ"), + (0x1feb, "", "ύ"), + (0x1fec, "", "ῥ"), + (0x1ff2, "Ὼͅ", ""), + (0x1ff3, "ῼ", ""), + (0x1ff4, "Ώͅ", ""), + (0x1ff6, "Ω͂", ""), + (0x1ff7, "ῼ͂", ""), + (0x1ff8, "", "ὸ"), + (0x1ff9, "", "ό"), + (0x1ffa, "", "ὼ"), + (0x1ffb, "", "ώ"), + (0x1ffc, "", "ῳ"), + (0x2126, "", "ω"), + (0x212a, "", "k"), + (0x212b, "", "å"), + (0x2132, "", "ⅎ"), + (0x214e, "Ⅎ", ""), + (0x2160, "", "ⅰ"), + (0x2161, "", "ⅱ"), + (0x2162, "", "ⅲ"), + (0x2163, "", "ⅳ"), + (0x2164, "", "ⅴ"), + (0x2165, "", "ⅵ"), + (0x2166, "", "ⅶ"), + (0x2167, "", "ⅷ"), + (0x2168, "", "ⅸ"), + (0x2169, "", "ⅹ"), + (0x216a, "", "ⅺ"), + (0x216b, "", "ⅻ"), + (0x216c, "", "ⅼ"), + (0x216d, "", "ⅽ"), + (0x216e, "", "ⅾ"), + (0x216f, "", "ⅿ"), + (0x2170, "Ⅰ", ""), + (0x2171, "Ⅱ", ""), + (0x2172, "Ⅲ", ""), + (0x2173, "Ⅳ", ""), + (0x2174, "Ⅴ", ""), + (0x2175, "Ⅵ", ""), + (0x2176, "Ⅶ", ""), + (0x2177, "Ⅷ", ""), + (0x2178, "Ⅸ", ""), + (0x2179, "Ⅹ", ""), + (0x217a, "Ⅺ", ""), + (0x217b, "Ⅻ", ""), + (0x217c, "Ⅼ", ""), + (0x217d, "Ⅽ", ""), + (0x217e, "Ⅾ", ""), + (0x217f, "Ⅿ", ""), + (0x2183, "", "ↄ"), + (0x2184, "Ↄ", ""), + (0x24b6, "", "ⓐ"), + (0x24b7, "", "ⓑ"), + (0x24b8, "", "ⓒ"), + (0x24b9, "", "ⓓ"), + (0x24ba, "", "ⓔ"), + (0x24bb, "", "ⓕ"), + (0x24bc, "", "ⓖ"), + (0x24bd, "", "ⓗ"), + (0x24be, "", "ⓘ"), + (0x24bf, "", "ⓙ"), + (0x24c0, "", "ⓚ"), + (0x24c1, "", "ⓛ"), + (0x24c2, "", "ⓜ"), + (0x24c3, "", "ⓝ"), + (0x24c4, "", "ⓞ"), + (0x24c5, "", "ⓟ"), + (0x24c6, "", "ⓠ"), + (0x24c7, "", "ⓡ"), + (0x24c8, "", "ⓢ"), + (0x24c9, "", "ⓣ"), + (0x24ca, "", "ⓤ"), + (0x24cb, "", "ⓥ"), + (0x24cc, "", "ⓦ"), + (0x24cd, "", "ⓧ"), + (0x24ce, "", "ⓨ"), + (0x24cf, "", "ⓩ"), + (0x24d0, "Ⓐ", ""), + (0x24d1, "Ⓑ", ""), + (0x24d2, "Ⓒ", ""), + (0x24d3, "Ⓓ", ""), + (0x24d4, "Ⓔ", ""), + (0x24d5, "Ⓕ", ""), + (0x24d6, "Ⓖ", ""), + (0x24d7, "Ⓗ", ""), + (0x24d8, "Ⓘ", ""), + (0x24d9, "Ⓙ", ""), + (0x24da, "Ⓚ", ""), + (0x24db, "Ⓛ", ""), + (0x24dc, "Ⓜ", ""), + (0x24dd, "Ⓝ", ""), + (0x24de, "Ⓞ", ""), + (0x24df, "Ⓟ", ""), + (0x24e0, "Ⓠ", ""), + (0x24e1, "Ⓡ", ""), + (0x24e2, "Ⓢ", ""), + (0x24e3, "Ⓣ", ""), + (0x24e4, "Ⓤ", ""), + (0x24e5, "Ⓥ", ""), + (0x24e6, "Ⓦ", ""), + (0x24e7, "Ⓧ", ""), + (0x24e8, "Ⓨ", ""), + (0x24e9, "Ⓩ", ""), + (0x2c00, "", "ⰰ"), + (0x2c01, "", "ⰱ"), + (0x2c02, "", "ⰲ"), + (0x2c03, "", "ⰳ"), + (0x2c04, "", "ⰴ"), + (0x2c05, "", "ⰵ"), + (0x2c06, "", "ⰶ"), + (0x2c07, "", "ⰷ"), + (0x2c08, "", "ⰸ"), + (0x2c09, "", "ⰹ"), + (0x2c0a, "", "ⰺ"), + (0x2c0b, "", "ⰻ"), + (0x2c0c, "", "ⰼ"), + (0x2c0d, "", "ⰽ"), + (0x2c0e, "", "ⰾ"), + (0x2c0f, "", "ⰿ"), + (0x2c10, "", "ⱀ"), + (0x2c11, "", "ⱁ"), + (0x2c12, "", "ⱂ"), + (0x2c13, "", "ⱃ"), + (0x2c14, "", "ⱄ"), + (0x2c15, "", "ⱅ"), + (0x2c16, "", "ⱆ"), + (0x2c17, "", "ⱇ"), + (0x2c18, "", "ⱈ"), + (0x2c19, "", "ⱉ"), + (0x2c1a, "", "ⱊ"), + (0x2c1b, "", "ⱋ"), + (0x2c1c, "", "ⱌ"), + (0x2c1d, "", "ⱍ"), + (0x2c1e, "", "ⱎ"), + (0x2c1f, "", "ⱏ"), + (0x2c20, "", "ⱐ"), + (0x2c21, "", "ⱑ"), + (0x2c22, "", "ⱒ"), + (0x2c23, "", "ⱓ"), + (0x2c24, "", "ⱔ"), + (0x2c25, "", "ⱕ"), + (0x2c26, "", "ⱖ"), + (0x2c27, "", "ⱗ"), + (0x2c28, "", "ⱘ"), + (0x2c29, "", "ⱙ"), + (0x2c2a, "", "ⱚ"), + (0x2c2b, "", "ⱛ"), + (0x2c2c, "", "ⱜ"), + (0x2c2d, "", "ⱝ"), + (0x2c2e, "", "ⱞ"), + (0x2c2f, "", "ⱟ"), + (0x2c30, "Ⰰ", ""), + (0x2c31, "Ⰱ", ""), + (0x2c32, "Ⰲ", ""), + (0x2c33, "Ⰳ", ""), + (0x2c34, "Ⰴ", ""), + (0x2c35, "Ⰵ", ""), + (0x2c36, "Ⰶ", ""), + (0x2c37, "Ⰷ", ""), + (0x2c38, "Ⰸ", ""), + (0x2c39, "Ⰹ", ""), + (0x2c3a, "Ⰺ", ""), + (0x2c3b, "Ⰻ", ""), + (0x2c3c, "Ⰼ", ""), + (0x2c3d, "Ⰽ", ""), + (0x2c3e, "Ⰾ", ""), + (0x2c3f, "Ⰿ", ""), + (0x2c40, "Ⱀ", ""), + (0x2c41, "Ⱁ", ""), + (0x2c42, "Ⱂ", ""), + (0x2c43, "Ⱃ", ""), + (0x2c44, "Ⱄ", ""), + (0x2c45, "Ⱅ", ""), + (0x2c46, "Ⱆ", ""), + (0x2c47, "Ⱇ", ""), + (0x2c48, "Ⱈ", ""), + (0x2c49, "Ⱉ", ""), + (0x2c4a, "Ⱊ", ""), + (0x2c4b, "Ⱋ", ""), + (0x2c4c, "Ⱌ", ""), + (0x2c4d, "Ⱍ", ""), + (0x2c4e, "Ⱎ", ""), + (0x2c4f, "Ⱏ", ""), + (0x2c50, "Ⱐ", ""), + (0x2c51, "Ⱑ", ""), + (0x2c52, "Ⱒ", ""), + (0x2c53, "Ⱓ", ""), + (0x2c54, "Ⱔ", ""), + (0x2c55, "Ⱕ", ""), + (0x2c56, "Ⱖ", ""), + (0x2c57, "Ⱗ", ""), + (0x2c58, "Ⱘ", ""), + (0x2c59, "Ⱙ", ""), + (0x2c5a, "Ⱚ", ""), + (0x2c5b, "Ⱛ", ""), + (0x2c5c, "Ⱜ", ""), + (0x2c5d, "Ⱝ", ""), + (0x2c5e, "Ⱞ", ""), + (0x2c5f, "Ⱟ", ""), + (0x2c60, "", "ⱡ"), + (0x2c61, "Ⱡ", ""), + (0x2c62, "", "ɫ"), + (0x2c63, "", "ᵽ"), + (0x2c64, "", "ɽ"), + (0x2c65, "Ⱥ", ""), + (0x2c66, "Ⱦ", ""), + (0x2c67, "", "ⱨ"), + (0x2c68, "Ⱨ", ""), + (0x2c69, "", "ⱪ"), + (0x2c6a, "Ⱪ", ""), + (0x2c6b, "", "ⱬ"), + (0x2c6c, "Ⱬ", ""), + (0x2c6d, "", "ɑ"), + (0x2c6e, "", "ɱ"), + (0x2c6f, "", "ɐ"), + (0x2c70, "", "ɒ"), + (0x2c72, "", "ⱳ"), + (0x2c73, "Ⱳ", ""), + (0x2c75, "", "ⱶ"), + (0x2c76, "Ⱶ", ""), + (0x2c7e, "", "ȿ"), + (0x2c7f, "", "ɀ"), + (0x2c80, "", "ⲁ"), + (0x2c81, "Ⲁ", ""), + (0x2c82, "", "ⲃ"), + (0x2c83, "Ⲃ", ""), + (0x2c84, "", "ⲅ"), + (0x2c85, "Ⲅ", ""), + (0x2c86, "", "ⲇ"), + (0x2c87, "Ⲇ", ""), + (0x2c88, "", "ⲉ"), + (0x2c89, "Ⲉ", ""), + (0x2c8a, "", "ⲋ"), + (0x2c8b, "Ⲋ", ""), + (0x2c8c, "", "ⲍ"), + (0x2c8d, "Ⲍ", ""), + (0x2c8e, "", "ⲏ"), + (0x2c8f, "Ⲏ", ""), + (0x2c90, "", "ⲑ"), + (0x2c91, "Ⲑ", ""), + (0x2c92, "", "ⲓ"), + (0x2c93, "Ⲓ", ""), + (0x2c94, "", "ⲕ"), + (0x2c95, "Ⲕ", ""), + (0x2c96, "", "ⲗ"), + (0x2c97, "Ⲗ", ""), + (0x2c98, "", "ⲙ"), + (0x2c99, "Ⲙ", ""), + (0x2c9a, "", "ⲛ"), + (0x2c9b, "Ⲛ", ""), + (0x2c9c, "", "ⲝ"), + (0x2c9d, "Ⲝ", ""), + (0x2c9e, "", "ⲟ"), + (0x2c9f, "Ⲟ", ""), + (0x2ca0, "", "ⲡ"), + (0x2ca1, "Ⲡ", ""), + (0x2ca2, "", "ⲣ"), + (0x2ca3, "Ⲣ", ""), + (0x2ca4, "", "ⲥ"), + (0x2ca5, "Ⲥ", ""), + (0x2ca6, "", "ⲧ"), + (0x2ca7, "Ⲧ", ""), + (0x2ca8, "", "ⲩ"), + (0x2ca9, "Ⲩ", ""), + (0x2caa, "", "ⲫ"), + (0x2cab, "Ⲫ", ""), + (0x2cac, "", "ⲭ"), + (0x2cad, "Ⲭ", ""), + (0x2cae, "", "ⲯ"), + (0x2caf, "Ⲯ", ""), + (0x2cb0, "", "ⲱ"), + (0x2cb1, "Ⲱ", ""), + (0x2cb2, "", "ⲳ"), + (0x2cb3, "Ⲳ", ""), + (0x2cb4, "", "ⲵ"), + (0x2cb5, "Ⲵ", ""), + (0x2cb6, "", "ⲷ"), + (0x2cb7, "Ⲷ", ""), + (0x2cb8, "", "ⲹ"), + (0x2cb9, "Ⲹ", ""), + (0x2cba, "", "ⲻ"), + (0x2cbb, "Ⲻ", ""), + (0x2cbc, "", "ⲽ"), + (0x2cbd, "Ⲽ", ""), + (0x2cbe, "", "ⲿ"), + (0x2cbf, "Ⲿ", ""), + (0x2cc0, "", "ⳁ"), + (0x2cc1, "Ⳁ", ""), + (0x2cc2, "", "ⳃ"), + (0x2cc3, "Ⳃ", ""), + (0x2cc4, "", "ⳅ"), + (0x2cc5, "Ⳅ", ""), + (0x2cc6, "", "ⳇ"), + (0x2cc7, "Ⳇ", ""), + (0x2cc8, "", "ⳉ"), + (0x2cc9, "Ⳉ", ""), + (0x2cca, "", "ⳋ"), + (0x2ccb, "Ⳋ", ""), + (0x2ccc, "", "ⳍ"), + (0x2ccd, "Ⳍ", ""), + (0x2cce, "", "ⳏ"), + (0x2ccf, "Ⳏ", ""), + (0x2cd0, "", "ⳑ"), + (0x2cd1, "Ⳑ", ""), + (0x2cd2, "", "ⳓ"), + (0x2cd3, "Ⳓ", ""), + (0x2cd4, "", "ⳕ"), + (0x2cd5, "Ⳕ", ""), + (0x2cd6, "", "ⳗ"), + (0x2cd7, "Ⳗ", ""), + (0x2cd8, "", "ⳙ"), + (0x2cd9, "Ⳙ", ""), + (0x2cda, "", "ⳛ"), + (0x2cdb, "Ⳛ", ""), + (0x2cdc, "", "ⳝ"), + (0x2cdd, "Ⳝ", ""), + (0x2cde, "", "ⳟ"), + (0x2cdf, "Ⳟ", ""), + (0x2ce0, "", "ⳡ"), + (0x2ce1, "Ⳡ", ""), + (0x2ce2, "", "ⳣ"), + (0x2ce3, "Ⳣ", ""), + (0x2ceb, "", "ⳬ"), + (0x2cec, "Ⳬ", ""), + (0x2ced, "", "ⳮ"), + (0x2cee, "Ⳮ", ""), + (0x2cf2, "", "ⳳ"), + (0x2cf3, "Ⳳ", ""), + (0x2d00, "Ⴀ", ""), + (0x2d01, "Ⴁ", ""), + (0x2d02, "Ⴂ", ""), + (0x2d03, "Ⴃ", ""), + (0x2d04, "Ⴄ", ""), + (0x2d05, "Ⴅ", ""), + (0x2d06, "Ⴆ", ""), + (0x2d07, "Ⴇ", ""), + (0x2d08, "Ⴈ", ""), + (0x2d09, "Ⴉ", ""), + (0x2d0a, "Ⴊ", ""), + (0x2d0b, "Ⴋ", ""), + (0x2d0c, "Ⴌ", ""), + (0x2d0d, "Ⴍ", ""), + (0x2d0e, "Ⴎ", ""), + (0x2d0f, "Ⴏ", ""), + (0x2d10, "Ⴐ", ""), + (0x2d11, "Ⴑ", ""), + (0x2d12, "Ⴒ", ""), + (0x2d13, "Ⴓ", ""), + (0x2d14, "Ⴔ", ""), + (0x2d15, "Ⴕ", ""), + (0x2d16, "Ⴖ", ""), + (0x2d17, "Ⴗ", ""), + (0x2d18, "Ⴘ", ""), + (0x2d19, "Ⴙ", ""), + (0x2d1a, "Ⴚ", ""), + (0x2d1b, "Ⴛ", ""), + (0x2d1c, "Ⴜ", ""), + (0x2d1d, "Ⴝ", ""), + (0x2d1e, "Ⴞ", ""), + (0x2d1f, "Ⴟ", ""), + (0x2d20, "Ⴠ", ""), + (0x2d21, "Ⴡ", ""), + (0x2d22, "Ⴢ", ""), + (0x2d23, "Ⴣ", ""), + (0x2d24, "Ⴤ", ""), + (0x2d25, "Ⴥ", ""), + (0x2d27, "Ⴧ", ""), + (0x2d2d, "Ⴭ", ""), + (0xa640, "", "ꙁ"), + (0xa641, "Ꙁ", ""), + (0xa642, "", "ꙃ"), + (0xa643, "Ꙃ", ""), + (0xa644, "", "ꙅ"), + (0xa645, "Ꙅ", ""), + (0xa646, "", "ꙇ"), + (0xa647, "Ꙇ", ""), + (0xa648, "", "ꙉ"), + (0xa649, "Ꙉ", ""), + (0xa64a, "", "ꙋ"), + (0xa64b, "Ꙋ", ""), + (0xa64c, "", "ꙍ"), + (0xa64d, "Ꙍ", ""), + (0xa64e, "", "ꙏ"), + (0xa64f, "Ꙏ", ""), + (0xa650, "", "ꙑ"), + (0xa651, "Ꙑ", ""), + (0xa652, "", "ꙓ"), + (0xa653, "Ꙓ", ""), + (0xa654, "", "ꙕ"), + (0xa655, "Ꙕ", ""), + (0xa656, "", "ꙗ"), + (0xa657, "Ꙗ", ""), + (0xa658, "", "ꙙ"), + (0xa659, "Ꙙ", ""), + (0xa65a, "", "ꙛ"), + (0xa65b, "Ꙛ", ""), + (0xa65c, "", "ꙝ"), + (0xa65d, "Ꙝ", ""), + (0xa65e, "", "ꙟ"), + (0xa65f, "Ꙟ", ""), + (0xa660, "", "ꙡ"), + (0xa661, "Ꙡ", ""), + (0xa662, "", "ꙣ"), + (0xa663, "Ꙣ", ""), + (0xa664, "", "ꙥ"), + (0xa665, "Ꙥ", ""), + (0xa666, "", "ꙧ"), + (0xa667, "Ꙧ", ""), + (0xa668, "", "ꙩ"), + (0xa669, "Ꙩ", ""), + (0xa66a, "", "ꙫ"), + (0xa66b, "Ꙫ", ""), + (0xa66c, "", "ꙭ"), + (0xa66d, "Ꙭ", ""), + (0xa680, "", "ꚁ"), + (0xa681, "Ꚁ", ""), + (0xa682, "", "ꚃ"), + (0xa683, "Ꚃ", ""), + (0xa684, "", "ꚅ"), + (0xa685, "Ꚅ", ""), + (0xa686, "", "ꚇ"), + (0xa687, "Ꚇ", ""), + (0xa688, "", "ꚉ"), + (0xa689, "Ꚉ", ""), + (0xa68a, "", "ꚋ"), + (0xa68b, "Ꚋ", ""), + (0xa68c, "", "ꚍ"), + (0xa68d, "Ꚍ", ""), + (0xa68e, "", "ꚏ"), + (0xa68f, "Ꚏ", ""), + (0xa690, "", "ꚑ"), + (0xa691, "Ꚑ", ""), + (0xa692, "", "ꚓ"), + (0xa693, "Ꚓ", ""), + (0xa694, "", "ꚕ"), + (0xa695, "Ꚕ", ""), + (0xa696, "", "ꚗ"), + (0xa697, "Ꚗ", ""), + (0xa698, "", "ꚙ"), + (0xa699, "Ꚙ", ""), + (0xa69a, "", "ꚛ"), + (0xa69b, "Ꚛ", ""), + (0xa722, "", "ꜣ"), + (0xa723, "Ꜣ", ""), + (0xa724, "", "ꜥ"), + (0xa725, "Ꜥ", ""), + (0xa726, "", "ꜧ"), + (0xa727, "Ꜧ", ""), + (0xa728, "", "ꜩ"), + (0xa729, "Ꜩ", ""), + (0xa72a, "", "ꜫ"), + (0xa72b, "Ꜫ", ""), + (0xa72c, "", "ꜭ"), + (0xa72d, "Ꜭ", ""), + (0xa72e, "", "ꜯ"), + (0xa72f, "Ꜯ", ""), + (0xa732, "", "ꜳ"), + (0xa733, "Ꜳ", ""), + (0xa734, "", "ꜵ"), + (0xa735, "Ꜵ", ""), + (0xa736, "", "ꜷ"), + (0xa737, "Ꜷ", ""), + (0xa738, "", "ꜹ"), + (0xa739, "Ꜹ", ""), + (0xa73a, "", "ꜻ"), + (0xa73b, "Ꜻ", ""), + (0xa73c, "", "ꜽ"), + (0xa73d, "Ꜽ", ""), + (0xa73e, "", "ꜿ"), + (0xa73f, "Ꜿ", ""), + (0xa740, "", "ꝁ"), + (0xa741, "Ꝁ", ""), + (0xa742, "", "ꝃ"), + (0xa743, "Ꝃ", ""), + (0xa744, "", "ꝅ"), + (0xa745, "Ꝅ", ""), + (0xa746, "", "ꝇ"), + (0xa747, "Ꝇ", ""), + (0xa748, "", "ꝉ"), + (0xa749, "Ꝉ", ""), + (0xa74a, "", "ꝋ"), + (0xa74b, "Ꝋ", ""), + (0xa74c, "", "ꝍ"), + (0xa74d, "Ꝍ", ""), + (0xa74e, "", "ꝏ"), + (0xa74f, "Ꝏ", ""), + (0xa750, "", "ꝑ"), + (0xa751, "Ꝑ", ""), + (0xa752, "", "ꝓ"), + (0xa753, "Ꝓ", ""), + (0xa754, "", "ꝕ"), + (0xa755, "Ꝕ", ""), + (0xa756, "", "ꝗ"), + (0xa757, "Ꝗ", ""), + (0xa758, "", "ꝙ"), + (0xa759, "Ꝙ", ""), + (0xa75a, "", "ꝛ"), + (0xa75b, "Ꝛ", ""), + (0xa75c, "", "ꝝ"), + (0xa75d, "Ꝝ", ""), + (0xa75e, "", "ꝟ"), + (0xa75f, "Ꝟ", ""), + (0xa760, "", "ꝡ"), + (0xa761, "Ꝡ", ""), + (0xa762, "", "ꝣ"), + (0xa763, "Ꝣ", ""), + (0xa764, "", "ꝥ"), + (0xa765, "Ꝥ", ""), + (0xa766, "", "ꝧ"), + (0xa767, "Ꝧ", ""), + (0xa768, "", "ꝩ"), + (0xa769, "Ꝩ", ""), + (0xa76a, "", "ꝫ"), + (0xa76b, "Ꝫ", ""), + (0xa76c, "", "ꝭ"), + (0xa76d, "Ꝭ", ""), + (0xa76e, "", "ꝯ"), + (0xa76f, "Ꝯ", ""), + (0xa779, "", "ꝺ"), + (0xa77a, "Ꝺ", ""), + (0xa77b, "", "ꝼ"), + (0xa77c, "Ꝼ", ""), + (0xa77d, "", "ᵹ"), + (0xa77e, "", "ꝿ"), + (0xa77f, "Ꝿ", ""), + (0xa780, "", "ꞁ"), + (0xa781, "Ꞁ", ""), + (0xa782, "", "ꞃ"), + (0xa783, "Ꞃ", ""), + (0xa784, "", "ꞅ"), + (0xa785, "Ꞅ", ""), + (0xa786, "", "ꞇ"), + (0xa787, "Ꞇ", ""), + (0xa78b, "", "ꞌ"), + (0xa78c, "Ꞌ", ""), + (0xa78d, "", "ɥ"), + (0xa790, "", "ꞑ"), + (0xa791, "Ꞑ", ""), + (0xa792, "", "ꞓ"), + (0xa793, "Ꞓ", ""), + (0xa794, "Ꞔ", ""), + (0xa796, "", "ꞗ"), + (0xa797, "Ꞗ", ""), + (0xa798, "", "ꞙ"), + (0xa799, "Ꞙ", ""), + (0xa79a, "", "ꞛ"), + (0xa79b, "Ꞛ", ""), + (0xa79c, "", "ꞝ"), + (0xa79d, "Ꞝ", ""), + (0xa79e, "", "ꞟ"), + (0xa79f, "Ꞟ", ""), + (0xa7a0, "", "ꞡ"), + (0xa7a1, "Ꞡ", ""), + (0xa7a2, "", "ꞣ"), + (0xa7a3, "Ꞣ", ""), + (0xa7a4, "", "ꞥ"), + (0xa7a5, "Ꞥ", ""), + (0xa7a6, "", "ꞧ"), + (0xa7a7, "Ꞧ", ""), + (0xa7a8, "", "ꞩ"), + (0xa7a9, "Ꞩ", ""), + (0xa7aa, "", "ɦ"), + (0xa7ab, "", "ɜ"), + (0xa7ac, "", "ɡ"), + (0xa7ad, "", "ɬ"), + (0xa7ae, "", "ɪ"), + (0xa7b0, "", "ʞ"), + (0xa7b1, "", "ʇ"), + (0xa7b2, "", "ʝ"), + (0xa7b3, "", "ꭓ"), + (0xa7b4, "", "ꞵ"), + (0xa7b5, "Ꞵ", ""), + (0xa7b6, "", "ꞷ"), + (0xa7b7, "Ꞷ", ""), + (0xa7b8, "", "ꞹ"), + (0xa7b9, "Ꞹ", ""), + (0xa7ba, "", "ꞻ"), + (0xa7bb, "Ꞻ", ""), + (0xa7bc, "", "ꞽ"), + (0xa7bd, "Ꞽ", ""), + (0xa7be, "", "ꞿ"), + (0xa7bf, "Ꞿ", ""), + (0xa7c0, "", "ꟁ"), + (0xa7c1, "Ꟁ", ""), + (0xa7c2, "", "ꟃ"), + (0xa7c3, "Ꟃ", ""), + (0xa7c4, "", "ꞔ"), + (0xa7c5, "", "ʂ"), + (0xa7c6, "", "ᶎ"), + (0xa7c7, "", "ꟈ"), + (0xa7c8, "Ꟈ", ""), + (0xa7c9, "", "ꟊ"), + (0xa7ca, "Ꟊ", ""), + (0xa7d0, "", "ꟑ"), + (0xa7d1, "Ꟑ", ""), + (0xa7d6, "", "ꟗ"), + (0xa7d7, "Ꟗ", ""), + (0xa7d8, "", "ꟙ"), + (0xa7d9, "Ꟙ", ""), + (0xa7f5, "", "ꟶ"), + (0xa7f6, "Ꟶ", ""), + (0xab53, "Ꭓ", ""), + (0xab70, "Ꭰ", ""), + (0xab71, "Ꭱ", ""), + (0xab72, "Ꭲ", ""), + (0xab73, "Ꭳ", ""), + (0xab74, "Ꭴ", ""), + (0xab75, "Ꭵ", ""), + (0xab76, "Ꭶ", ""), + (0xab77, "Ꭷ", ""), + (0xab78, "Ꭸ", ""), + (0xab79, "Ꭹ", ""), + (0xab7a, "Ꭺ", ""), + (0xab7b, "Ꭻ", ""), + (0xab7c, "Ꭼ", ""), + (0xab7d, "Ꭽ", ""), + (0xab7e, "Ꭾ", ""), + (0xab7f, "Ꭿ", ""), + (0xab80, "Ꮀ", ""), + (0xab81, "Ꮁ", ""), + (0xab82, "Ꮂ", ""), + (0xab83, "Ꮃ", ""), + (0xab84, "Ꮄ", ""), + (0xab85, "Ꮅ", ""), + (0xab86, "Ꮆ", ""), + (0xab87, "Ꮇ", ""), + (0xab88, "Ꮈ", ""), + (0xab89, "Ꮉ", ""), + (0xab8a, "Ꮊ", ""), + (0xab8b, "Ꮋ", ""), + (0xab8c, "Ꮌ", ""), + (0xab8d, "Ꮍ", ""), + (0xab8e, "Ꮎ", ""), + (0xab8f, "Ꮏ", ""), + (0xab90, "Ꮐ", ""), + (0xab91, "Ꮑ", ""), + (0xab92, "Ꮒ", ""), + (0xab93, "Ꮓ", ""), + (0xab94, "Ꮔ", ""), + (0xab95, "Ꮕ", ""), + (0xab96, "Ꮖ", ""), + (0xab97, "Ꮗ", ""), + (0xab98, "Ꮘ", ""), + (0xab99, "Ꮙ", ""), + (0xab9a, "Ꮚ", ""), + (0xab9b, "Ꮛ", ""), + (0xab9c, "Ꮜ", ""), + (0xab9d, "Ꮝ", ""), + (0xab9e, "Ꮞ", ""), + (0xab9f, "Ꮟ", ""), + (0xaba0, "Ꮠ", ""), + (0xaba1, "Ꮡ", ""), + (0xaba2, "Ꮢ", ""), + (0xaba3, "Ꮣ", ""), + (0xaba4, "Ꮤ", ""), + (0xaba5, "Ꮥ", ""), + (0xaba6, "Ꮦ", ""), + (0xaba7, "Ꮧ", ""), + (0xaba8, "Ꮨ", ""), + (0xaba9, "Ꮩ", ""), + (0xabaa, "Ꮪ", ""), + (0xabab, "Ꮫ", ""), + (0xabac, "Ꮬ", ""), + (0xabad, "Ꮭ", ""), + (0xabae, "Ꮮ", ""), + (0xabaf, "Ꮯ", ""), + (0xabb0, "Ꮰ", ""), + (0xabb1, "Ꮱ", ""), + (0xabb2, "Ꮲ", ""), + (0xabb3, "Ꮳ", ""), + (0xabb4, "Ꮴ", ""), + (0xabb5, "Ꮵ", ""), + (0xabb6, "Ꮶ", ""), + (0xabb7, "Ꮷ", ""), + (0xabb8, "Ꮸ", ""), + (0xabb9, "Ꮹ", ""), + (0xabba, "Ꮺ", ""), + (0xabbb, "Ꮻ", ""), + (0xabbc, "Ꮼ", ""), + (0xabbd, "Ꮽ", ""), + (0xabbe, "Ꮾ", ""), + (0xabbf, "Ꮿ", ""), + (0xfb00, "Ff", ""), + (0xfb01, "Fi", ""), + (0xfb02, "Fl", ""), + (0xfb03, "Ffi", ""), + (0xfb04, "Ffl", ""), + (0xfb05, "St", ""), + (0xfb06, "St", ""), + (0xfb13, "Մն", ""), + (0xfb14, "Մե", ""), + (0xfb15, "Մի", ""), + (0xfb16, "Վն", ""), + (0xfb17, "Մխ", ""), + (0xff21, "", "a"), + (0xff22, "", "b"), + (0xff23, "", "c"), + (0xff24, "", "d"), + (0xff25, "", "e"), + (0xff26, "", "f"), + (0xff27, "", "g"), + (0xff28, "", "h"), + (0xff29, "", "i"), + (0xff2a, "", "j"), + (0xff2b, "", "k"), + (0xff2c, "", "l"), + (0xff2d, "", "m"), + (0xff2e, "", "n"), + (0xff2f, "", "o"), + (0xff30, "", "p"), + (0xff31, "", "q"), + (0xff32, "", "r"), + (0xff33, "", "s"), + (0xff34, "", "t"), + (0xff35, "", "u"), + (0xff36, "", "v"), + (0xff37, "", "w"), + (0xff38, "", "x"), + (0xff39, "", "y"), + (0xff3a, "", "z"), + (0xff41, "A", ""), + (0xff42, "B", ""), + (0xff43, "C", ""), + (0xff44, "D", ""), + (0xff45, "E", ""), + (0xff46, "F", ""), + (0xff47, "G", ""), + (0xff48, "H", ""), + (0xff49, "I", ""), + (0xff4a, "J", ""), + (0xff4b, "K", ""), + (0xff4c, "L", ""), + (0xff4d, "M", ""), + (0xff4e, "N", ""), + (0xff4f, "O", ""), + (0xff50, "P", ""), + (0xff51, "Q", ""), + (0xff52, "R", ""), + (0xff53, "S", ""), + (0xff54, "T", ""), + (0xff55, "U", ""), + (0xff56, "V", ""), + (0xff57, "W", ""), + (0xff58, "X", ""), + (0xff59, "Y", ""), + (0xff5a, "Z", ""), + (0x10400, "", "𐐨"), + (0x10401, "", "𐐩"), + (0x10402, "", "𐐪"), + (0x10403, "", "𐐫"), + (0x10404, "", "𐐬"), + (0x10405, "", "𐐭"), + (0x10406, "", "𐐮"), + (0x10407, "", "𐐯"), + (0x10408, "", "𐐰"), + (0x10409, "", "𐐱"), + (0x1040a, "", "𐐲"), + (0x1040b, "", "𐐳"), + (0x1040c, "", "𐐴"), + (0x1040d, "", "𐐵"), + (0x1040e, "", "𐐶"), + (0x1040f, "", "𐐷"), + (0x10410, "", "𐐸"), + (0x10411, "", "𐐹"), + (0x10412, "", "𐐺"), + (0x10413, "", "𐐻"), + (0x10414, "", "𐐼"), + (0x10415, "", "𐐽"), + (0x10416, "", "𐐾"), + (0x10417, "", "𐐿"), + (0x10418, "", "𐑀"), + (0x10419, "", "𐑁"), + (0x1041a, "", "𐑂"), + (0x1041b, "", "𐑃"), + (0x1041c, "", "𐑄"), + (0x1041d, "", "𐑅"), + (0x1041e, "", "𐑆"), + (0x1041f, "", "𐑇"), + (0x10420, "", "𐑈"), + (0x10421, "", "𐑉"), + (0x10422, "", "𐑊"), + (0x10423, "", "𐑋"), + (0x10424, "", "𐑌"), + (0x10425, "", "𐑍"), + (0x10426, "", "𐑎"), + (0x10427, "", "𐑏"), + (0x10428, "𐐀", ""), + (0x10429, "𐐁", ""), + (0x1042a, "𐐂", ""), + (0x1042b, "𐐃", ""), + (0x1042c, "𐐄", ""), + (0x1042d, "𐐅", ""), + (0x1042e, "𐐆", ""), + (0x1042f, "𐐇", ""), + (0x10430, "𐐈", ""), + (0x10431, "𐐉", ""), + (0x10432, "𐐊", ""), + (0x10433, "𐐋", ""), + (0x10434, "𐐌", ""), + (0x10435, "𐐍", ""), + (0x10436, "𐐎", ""), + (0x10437, "𐐏", ""), + (0x10438, "𐐐", ""), + (0x10439, "𐐑", ""), + (0x1043a, "𐐒", ""), + (0x1043b, "𐐓", ""), + (0x1043c, "𐐔", ""), + (0x1043d, "𐐕", ""), + (0x1043e, "𐐖", ""), + (0x1043f, "𐐗", ""), + (0x10440, "𐐘", ""), + (0x10441, "𐐙", ""), + (0x10442, "𐐚", ""), + (0x10443, "𐐛", ""), + (0x10444, "𐐜", ""), + (0x10445, "𐐝", ""), + (0x10446, "𐐞", ""), + (0x10447, "𐐟", ""), + (0x10448, "𐐠", ""), + (0x10449, "𐐡", ""), + (0x1044a, "𐐢", ""), + (0x1044b, "𐐣", ""), + (0x1044c, "𐐤", ""), + (0x1044d, "𐐥", ""), + (0x1044e, "𐐦", ""), + (0x1044f, "𐐧", ""), + (0x104b0, "", "𐓘"), + (0x104b1, "", "𐓙"), + (0x104b2, "", "𐓚"), + (0x104b3, "", "𐓛"), + (0x104b4, "", "𐓜"), + (0x104b5, "", "𐓝"), + (0x104b6, "", "𐓞"), + (0x104b7, "", "𐓟"), + (0x104b8, "", "𐓠"), + (0x104b9, "", "𐓡"), + (0x104ba, "", "𐓢"), + (0x104bb, "", "𐓣"), + (0x104bc, "", "𐓤"), + (0x104bd, "", "𐓥"), + (0x104be, "", "𐓦"), + (0x104bf, "", "𐓧"), + (0x104c0, "", "𐓨"), + (0x104c1, "", "𐓩"), + (0x104c2, "", "𐓪"), + (0x104c3, "", "𐓫"), + (0x104c4, "", "𐓬"), + (0x104c5, "", "𐓭"), + (0x104c6, "", "𐓮"), + (0x104c7, "", "𐓯"), + (0x104c8, "", "𐓰"), + (0x104c9, "", "𐓱"), + (0x104ca, "", "𐓲"), + (0x104cb, "", "𐓳"), + (0x104cc, "", "𐓴"), + (0x104cd, "", "𐓵"), + (0x104ce, "", "𐓶"), + (0x104cf, "", "𐓷"), + (0x104d0, "", "𐓸"), + (0x104d1, "", "𐓹"), + (0x104d2, "", "𐓺"), + (0x104d3, "", "𐓻"), + (0x104d8, "𐒰", ""), + (0x104d9, "𐒱", ""), + (0x104da, "𐒲", ""), + (0x104db, "𐒳", ""), + (0x104dc, "𐒴", ""), + (0x104dd, "𐒵", ""), + (0x104de, "𐒶", ""), + (0x104df, "𐒷", ""), + (0x104e0, "𐒸", ""), + (0x104e1, "𐒹", ""), + (0x104e2, "𐒺", ""), + (0x104e3, "𐒻", ""), + (0x104e4, "𐒼", ""), + (0x104e5, "𐒽", ""), + (0x104e6, "𐒾", ""), + (0x104e7, "𐒿", ""), + (0x104e8, "𐓀", ""), + (0x104e9, "𐓁", ""), + (0x104ea, "𐓂", ""), + (0x104eb, "𐓃", ""), + (0x104ec, "𐓄", ""), + (0x104ed, "𐓅", ""), + (0x104ee, "𐓆", ""), + (0x104ef, "𐓇", ""), + (0x104f0, "𐓈", ""), + (0x104f1, "𐓉", ""), + (0x104f2, "𐓊", ""), + (0x104f3, "𐓋", ""), + (0x104f4, "𐓌", ""), + (0x104f5, "𐓍", ""), + (0x104f6, "𐓎", ""), + (0x104f7, "𐓏", ""), + (0x104f8, "𐓐", ""), + (0x104f9, "𐓑", ""), + (0x104fa, "𐓒", ""), + (0x104fb, "𐓓", ""), + (0x10570, "", "𐖗"), + (0x10571, "", "𐖘"), + (0x10572, "", "𐖙"), + (0x10573, "", "𐖚"), + (0x10574, "", "𐖛"), + (0x10575, "", "𐖜"), + (0x10576, "", "𐖝"), + (0x10577, "", "𐖞"), + (0x10578, "", "𐖟"), + (0x10579, "", "𐖠"), + (0x1057a, "", "𐖡"), + (0x1057c, "", "𐖣"), + (0x1057d, "", "𐖤"), + (0x1057e, "", "𐖥"), + (0x1057f, "", "𐖦"), + (0x10580, "", "𐖧"), + (0x10581, "", "𐖨"), + (0x10582, "", "𐖩"), + (0x10583, "", "𐖪"), + (0x10584, "", "𐖫"), + (0x10585, "", "𐖬"), + (0x10586, "", "𐖭"), + (0x10587, "", "𐖮"), + (0x10588, "", "𐖯"), + (0x10589, "", "𐖰"), + (0x1058a, "", "𐖱"), + (0x1058c, "", "𐖳"), + (0x1058d, "", "𐖴"), + (0x1058e, "", "𐖵"), + (0x1058f, "", "𐖶"), + (0x10590, "", "𐖷"), + (0x10591, "", "𐖸"), + (0x10592, "", "𐖹"), + (0x10594, "", "𐖻"), + (0x10595, "", "𐖼"), + (0x10597, "𐕰", ""), + (0x10598, "𐕱", ""), + (0x10599, "𐕲", ""), + (0x1059a, "𐕳", ""), + (0x1059b, "𐕴", ""), + (0x1059c, "𐕵", ""), + (0x1059d, "𐕶", ""), + (0x1059e, "𐕷", ""), + (0x1059f, "𐕸", ""), + (0x105a0, "𐕹", ""), + (0x105a1, "𐕺", ""), + (0x105a3, "𐕼", ""), + (0x105a4, "𐕽", ""), + (0x105a5, "𐕾", ""), + (0x105a6, "𐕿", ""), + (0x105a7, "𐖀", ""), + (0x105a8, "𐖁", ""), + (0x105a9, "𐖂", ""), + (0x105aa, "𐖃", ""), + (0x105ab, "𐖄", ""), + (0x105ac, "𐖅", ""), + (0x105ad, "𐖆", ""), + (0x105ae, "𐖇", ""), + (0x105af, "𐖈", ""), + (0x105b0, "𐖉", ""), + (0x105b1, "𐖊", ""), + (0x105b3, "𐖌", ""), + (0x105b4, "𐖍", ""), + (0x105b5, "𐖎", ""), + (0x105b6, "𐖏", ""), + (0x105b7, "𐖐", ""), + (0x105b8, "𐖑", ""), + (0x105b9, "𐖒", ""), + (0x105bb, "𐖔", ""), + (0x105bc, "𐖕", ""), + (0x10c80, "", "𐳀"), + (0x10c81, "", "𐳁"), + (0x10c82, "", "𐳂"), + (0x10c83, "", "𐳃"), + (0x10c84, "", "𐳄"), + (0x10c85, "", "𐳅"), + (0x10c86, "", "𐳆"), + (0x10c87, "", "𐳇"), + (0x10c88, "", "𐳈"), + (0x10c89, "", "𐳉"), + (0x10c8a, "", "𐳊"), + (0x10c8b, "", "𐳋"), + (0x10c8c, "", "𐳌"), + (0x10c8d, "", "𐳍"), + (0x10c8e, "", "𐳎"), + (0x10c8f, "", "𐳏"), + (0x10c90, "", "𐳐"), + (0x10c91, "", "𐳑"), + (0x10c92, "", "𐳒"), + (0x10c93, "", "𐳓"), + (0x10c94, "", "𐳔"), + (0x10c95, "", "𐳕"), + (0x10c96, "", "𐳖"), + (0x10c97, "", "𐳗"), + (0x10c98, "", "𐳘"), + (0x10c99, "", "𐳙"), + (0x10c9a, "", "𐳚"), + (0x10c9b, "", "𐳛"), + (0x10c9c, "", "𐳜"), + (0x10c9d, "", "𐳝"), + (0x10c9e, "", "𐳞"), + (0x10c9f, "", "𐳟"), + (0x10ca0, "", "𐳠"), + (0x10ca1, "", "𐳡"), + (0x10ca2, "", "𐳢"), + (0x10ca3, "", "𐳣"), + (0x10ca4, "", "𐳤"), + (0x10ca5, "", "𐳥"), + (0x10ca6, "", "𐳦"), + (0x10ca7, "", "𐳧"), + (0x10ca8, "", "𐳨"), + (0x10ca9, "", "𐳩"), + (0x10caa, "", "𐳪"), + (0x10cab, "", "𐳫"), + (0x10cac, "", "𐳬"), + (0x10cad, "", "𐳭"), + (0x10cae, "", "𐳮"), + (0x10caf, "", "𐳯"), + (0x10cb0, "", "𐳰"), + (0x10cb1, "", "𐳱"), + (0x10cb2, "", "𐳲"), + (0x10cc0, "𐲀", ""), + (0x10cc1, "𐲁", ""), + (0x10cc2, "𐲂", ""), + (0x10cc3, "𐲃", ""), + (0x10cc4, "𐲄", ""), + (0x10cc5, "𐲅", ""), + (0x10cc6, "𐲆", ""), + (0x10cc7, "𐲇", ""), + (0x10cc8, "𐲈", ""), + (0x10cc9, "𐲉", ""), + (0x10cca, "𐲊", ""), + (0x10ccb, "𐲋", ""), + (0x10ccc, "𐲌", ""), + (0x10ccd, "𐲍", ""), + (0x10cce, "𐲎", ""), + (0x10ccf, "𐲏", ""), + (0x10cd0, "𐲐", ""), + (0x10cd1, "𐲑", ""), + (0x10cd2, "𐲒", ""), + (0x10cd3, "𐲓", ""), + (0x10cd4, "𐲔", ""), + (0x10cd5, "𐲕", ""), + (0x10cd6, "𐲖", ""), + (0x10cd7, "𐲗", ""), + (0x10cd8, "𐲘", ""), + (0x10cd9, "𐲙", ""), + (0x10cda, "𐲚", ""), + (0x10cdb, "𐲛", ""), + (0x10cdc, "𐲜", ""), + (0x10cdd, "𐲝", ""), + (0x10cde, "𐲞", ""), + (0x10cdf, "𐲟", ""), + (0x10ce0, "𐲠", ""), + (0x10ce1, "𐲡", ""), + (0x10ce2, "𐲢", ""), + (0x10ce3, "𐲣", ""), + (0x10ce4, "𐲤", ""), + (0x10ce5, "𐲥", ""), + (0x10ce6, "𐲦", ""), + (0x10ce7, "𐲧", ""), + (0x10ce8, "𐲨", ""), + (0x10ce9, "𐲩", ""), + (0x10cea, "𐲪", ""), + (0x10ceb, "𐲫", ""), + (0x10cec, "𐲬", ""), + (0x10ced, "𐲭", ""), + (0x10cee, "𐲮", ""), + (0x10cef, "𐲯", ""), + (0x10cf0, "𐲰", ""), + (0x10cf1, "𐲱", ""), + (0x10cf2, "𐲲", ""), + (0x118a0, "", "𑣀"), + (0x118a1, "", "𑣁"), + (0x118a2, "", "𑣂"), + (0x118a3, "", "𑣃"), + (0x118a4, "", "𑣄"), + (0x118a5, "", "𑣅"), + (0x118a6, "", "𑣆"), + (0x118a7, "", "𑣇"), + (0x118a8, "", "𑣈"), + (0x118a9, "", "𑣉"), + (0x118aa, "", "𑣊"), + (0x118ab, "", "𑣋"), + (0x118ac, "", "𑣌"), + (0x118ad, "", "𑣍"), + (0x118ae, "", "𑣎"), + (0x118af, "", "𑣏"), + (0x118b0, "", "𑣐"), + (0x118b1, "", "𑣑"), + (0x118b2, "", "𑣒"), + (0x118b3, "", "𑣓"), + (0x118b4, "", "𑣔"), + (0x118b5, "", "𑣕"), + (0x118b6, "", "𑣖"), + (0x118b7, "", "𑣗"), + (0x118b8, "", "𑣘"), + (0x118b9, "", "𑣙"), + (0x118ba, "", "𑣚"), + (0x118bb, "", "𑣛"), + (0x118bc, "", "𑣜"), + (0x118bd, "", "𑣝"), + (0x118be, "", "𑣞"), + (0x118bf, "", "𑣟"), + (0x118c0, "𑢠", ""), + (0x118c1, "𑢡", ""), + (0x118c2, "𑢢", ""), + (0x118c3, "𑢣", ""), + (0x118c4, "𑢤", ""), + (0x118c5, "𑢥", ""), + (0x118c6, "𑢦", ""), + (0x118c7, "𑢧", ""), + (0x118c8, "𑢨", ""), + (0x118c9, "𑢩", ""), + (0x118ca, "𑢪", ""), + (0x118cb, "𑢫", ""), + (0x118cc, "𑢬", ""), + (0x118cd, "𑢭", ""), + (0x118ce, "𑢮", ""), + (0x118cf, "𑢯", ""), + (0x118d0, "𑢰", ""), + (0x118d1, "𑢱", ""), + (0x118d2, "𑢲", ""), + (0x118d3, "𑢳", ""), + (0x118d4, "𑢴", ""), + (0x118d5, "𑢵", ""), + (0x118d6, "𑢶", ""), + (0x118d7, "𑢷", ""), + (0x118d8, "𑢸", ""), + (0x118d9, "𑢹", ""), + (0x118da, "𑢺", ""), + (0x118db, "𑢻", ""), + (0x118dc, "𑢼", ""), + (0x118dd, "𑢽", ""), + (0x118de, "𑢾", ""), + (0x118df, "𑢿", ""), + (0x16e40, "", "𖹠"), + (0x16e41, "", "𖹡"), + (0x16e42, "", "𖹢"), + (0x16e43, "", "𖹣"), + (0x16e44, "", "𖹤"), + (0x16e45, "", "𖹥"), + (0x16e46, "", "𖹦"), + (0x16e47, "", "𖹧"), + (0x16e48, "", "𖹨"), + (0x16e49, "", "𖹩"), + (0x16e4a, "", "𖹪"), + (0x16e4b, "", "𖹫"), + (0x16e4c, "", "𖹬"), + (0x16e4d, "", "𖹭"), + (0x16e4e, "", "𖹮"), + (0x16e4f, "", "𖹯"), + (0x16e50, "", "𖹰"), + (0x16e51, "", "𖹱"), + (0x16e52, "", "𖹲"), + (0x16e53, "", "𖹳"), + (0x16e54, "", "𖹴"), + (0x16e55, "", "𖹵"), + (0x16e56, "", "𖹶"), + (0x16e57, "", "𖹷"), + (0x16e58, "", "𖹸"), + (0x16e59, "", "𖹹"), + (0x16e5a, "", "𖹺"), + (0x16e5b, "", "𖹻"), + (0x16e5c, "", "𖹼"), + (0x16e5d, "", "𖹽"), + (0x16e5e, "", "𖹾"), + (0x16e5f, "", "𖹿"), + (0x16e60, "𖹀", ""), + (0x16e61, "𖹁", ""), + (0x16e62, "𖹂", ""), + (0x16e63, "𖹃", ""), + (0x16e64, "𖹄", ""), + (0x16e65, "𖹅", ""), + (0x16e66, "𖹆", ""), + (0x16e67, "𖹇", ""), + (0x16e68, "𖹈", ""), + (0x16e69, "𖹉", ""), + (0x16e6a, "𖹊", ""), + (0x16e6b, "𖹋", ""), + (0x16e6c, "𖹌", ""), + (0x16e6d, "𖹍", ""), + (0x16e6e, "𖹎", ""), + (0x16e6f, "𖹏", ""), + (0x16e70, "𖹐", ""), + (0x16e71, "𖹑", ""), + (0x16e72, "𖹒", ""), + (0x16e73, "𖹓", ""), + (0x16e74, "𖹔", ""), + (0x16e75, "𖹕", ""), + (0x16e76, "𖹖", ""), + (0x16e77, "𖹗", ""), + (0x16e78, "𖹘", ""), + (0x16e79, "𖹙", ""), + (0x16e7a, "𖹚", ""), + (0x16e7b, "𖹛", ""), + (0x16e7c, "𖹜", ""), + (0x16e7d, "𖹝", ""), + (0x16e7e, "𖹞", ""), + (0x16e7f, "𖹟", ""), + (0x1e900, "", "𞤢"), + (0x1e901, "", "𞤣"), + (0x1e902, "", "𞤤"), + (0x1e903, "", "𞤥"), + (0x1e904, "", "𞤦"), + (0x1e905, "", "𞤧"), + (0x1e906, "", "𞤨"), + (0x1e907, "", "𞤩"), + (0x1e908, "", "𞤪"), + (0x1e909, "", "𞤫"), + (0x1e90a, "", "𞤬"), + (0x1e90b, "", "𞤭"), + (0x1e90c, "", "𞤮"), + (0x1e90d, "", "𞤯"), + (0x1e90e, "", "𞤰"), + (0x1e90f, "", "𞤱"), + (0x1e910, "", "𞤲"), + (0x1e911, "", "𞤳"), + (0x1e912, "", "𞤴"), + (0x1e913, "", "𞤵"), + (0x1e914, "", "𞤶"), + (0x1e915, "", "𞤷"), + (0x1e916, "", "𞤸"), + (0x1e917, "", "𞤹"), + (0x1e918, "", "𞤺"), + (0x1e919, "", "𞤻"), + (0x1e91a, "", "𞤼"), + (0x1e91b, "", "𞤽"), + (0x1e91c, "", "𞤾"), + (0x1e91d, "", "𞤿"), + (0x1e91e, "", "𞥀"), + (0x1e91f, "", "𞥁"), + (0x1e920, "", "𞥂"), + (0x1e921, "", "𞥃"), + (0x1e922, "𞤀", ""), + (0x1e923, "𞤁", ""), + (0x1e924, "𞤂", ""), + (0x1e925, "𞤃", ""), + (0x1e926, "𞤄", ""), + (0x1e927, "𞤅", ""), + (0x1e928, "𞤆", ""), + (0x1e929, "𞤇", ""), + (0x1e92a, "𞤈", ""), + (0x1e92b, "𞤉", ""), + (0x1e92c, "𞤊", ""), + (0x1e92d, "𞤋", ""), + (0x1e92e, "𞤌", ""), + (0x1e92f, "𞤍", ""), + (0x1e930, "𞤎", ""), + (0x1e931, "𞤏", ""), + (0x1e932, "𞤐", ""), + (0x1e933, "𞤑", ""), + (0x1e934, "𞤒", ""), + (0x1e935, "𞤓", ""), + (0x1e936, "𞤔", ""), + (0x1e937, "𞤕", ""), + (0x1e938, "𞤖", ""), + (0x1e939, "𞤗", ""), + (0x1e93a, "𞤘", ""), + (0x1e93b, "𞤙", ""), + (0x1e93c, "𞤚", ""), + (0x1e93d, "𞤛", ""), + (0x1e93e, "𞤜", ""), + (0x1e93f, "𞤝", ""), + (0x1e940, "𞤞", ""), + (0x1e941, "𞤟", ""), + (0x1e942, "𞤠", ""), + (0x1e943, "𞤡", ""), +]; diff --git a/rust_backend/crates/core/src/metadata/musicbrainz/casing.rs b/rust_backend/crates/core/src/metadata/musicbrainz/casing.rs new file mode 100644 index 00000000..b4e792f6 --- /dev/null +++ b/rust_backend/crates/core/src/metadata/musicbrainz/casing.rs @@ -0,0 +1,93 @@ +// English title-case control flow follows golang.org/x/text/cases (Go Authors). +// See rust_backend/NOTICE for the BSD license. +use super::case_data::{MAPPINGS, RANGES}; + +fn flags(ch: char) -> u8 { + let code = ch as u32; + let index = RANGES.partition_point(|(_, end, _)| *end < code); + RANGES + .get(index) + .filter(|(start, _, _)| *start <= code) + .map_or(4, |(_, _, flags)| *flags) +} + +fn mapped(output: &mut String, ch: char, first: bool) { + if let Ok(index) = MAPPINGS.binary_search_by_key(&(ch as u32), |(code, _, _)| *code) { + let (_, title, lower) = MAPPINGS[index]; + let mapping = if first { title } else { lower }; + if !mapping.is_empty() { + output.push_str(mapping); + return; + } + } + output.push(ch); +} + +pub(super) fn title(value: &str) -> String { + let chars: Vec<_> = value.chars().collect(); + let mut output = String::with_capacity(value.len()); + let mut mid_word = false; + let mut index = 0; + while let Some(&ch) = chars.get(index) { + let properties = flags(ch); + index += 1; + if properties & 1 != 0 { + if !mid_word { + mapped(&mut output, ch, true); + mid_word = true; + } else if ch == 'Σ' { + // Go caps this lookahead at 30 ignorables plus one following + // rune. Ignorables are copied, even if they are also cased. + let sigma = output.len(); + output.push('ς'); + let mut was_mid = false; + for _ in 0..31 { + let Some(&next) = chars.get(index) else { break }; + let next_flags = flags(next); + if next_flags & 2 == 0 { + if next_flags & 1 != 0 { + output.replace_range(sigma..sigma + 'ς'.len_utf8(), "σ"); + } + break; + } + let is_mid = next_flags & 8 != 0; + if (was_mid && is_mid) || next_flags & 4 != 0 { + mid_word = false; + } + was_mid = is_mid; + output.push(next); + index += 1; + } + } else { + mapped(&mut output, ch, false); + } + } else { + output.push(ch); + if properties & 4 != 0 { + mid_word = false; + } + } + if properties & 8 != 0 && chars.get(index).is_some_and(|next| flags(*next) & 8 != 0) { + mid_word = false; + } + } + output +} + +#[cfg(test)] +mod tests { + use super::title; + + #[test] + fn sigma_context_does_not_depend_on_transform_buffer_boundaries() { + // Go's transform.String can lose mid-word state at its 128-byte + // buffer boundary while looking beyond sigma. Preserve the same + // contextual casing regardless of the length of an earlier prefix. + for size in 0..300 { + let prefix = "a|".repeat(size); + let value = format!("{prefix}ΟΣ\u{1aff1}A'ß_\u{1aff1}dz|"); + let expected = format!("{}Οσ\u{1aff1}A'ß_\u{1aff1}Dz|", "A|".repeat(size)); + assert_eq!(title(&value), expected, "prefix length {size}"); + } + } +} diff --git a/rust_backend/crates/core/src/metadata/reenrich.rs b/rust_backend/crates/core/src/metadata/reenrich.rs new file mode 100644 index 00000000..0ca649e1 --- /dev/null +++ b/rust_backend/crates/core/src/metadata/reenrich.rs @@ -0,0 +1,466 @@ +//! Re-enrichment input, candidate policy and preview metadata. No file access. + +use crate::lyrics::json::go_deserialize; +use crate::matching::lowercase; +use crate::resolver::{artists_match, titles_match, track_identity_title, track_titles_match}; +use serde::Serialize; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[derive(Clone, Default, Serialize)] +pub struct Request { + pub file_path: String, + pub cover_url: String, + pub cover_max_dimension: isize, + pub embed_lyrics: bool, + pub lyrics_mode: String, + pub artist_tag_mode: String, + pub spotify_id: String, + pub track_name: String, + pub artist_name: String, + pub album_name: String, + pub album_artist: String, + pub track_number: isize, + pub disc_number: isize, + pub total_tracks: isize, + pub total_discs: isize, + pub release_date: String, + pub isrc: String, + pub genre: String, + pub label: String, + pub copyright: String, + pub composer: String, + pub duration_ms: i64, + pub search_online: bool, + pub update_fields: Option<Vec<Option<String>>>, + pub preview_only: bool, + pub replace_release_metadata: bool, +} +go_deserialize!(Request { + "file_path" => file_path, "cover_url" => cover_url, + "cover_max_dimension" => cover_max_dimension, "embed_lyrics" => embed_lyrics, + "lyrics_mode" => lyrics_mode, "artist_tag_mode" => artist_tag_mode, + "spotify_id" => spotify_id, "track_name" => track_name, "artist_name" => artist_name, + "album_name" => album_name, "album_artist" => album_artist, + "track_number" => track_number, "disc_number" => disc_number, + "total_tracks" => total_tracks, "total_discs" => total_discs, + "release_date" => release_date, "isrc" => isrc, "genre" => genre, + "label" => label, "copyright" => copyright, "composer" => composer, + "duration_ms" => duration_ms, "search_online" => search_online, + "update_fields" => update_fields, "preview_only" => preview_only, + "replace_release_metadata" => replace_release_metadata, +}); + +pub fn text<'a>(value: &'a Value, key: &str) -> &'a str { + value[key].as_str().unwrap_or_default() +} + +fn number(value: &Value, key: &str) -> i64 { + value[key].as_i64().unwrap_or_default() +} + +pub fn placeholder(value: &str) -> bool { + matches!( + lowercase(value.trim()).as_str(), + "" | "unknown" | "unknown artist" | "unknown title" | "unknown album" + ) +} + +impl Request { + pub fn parse(raw: &str) -> Result<Self, serde_json::Error> { + serde_json::from_str(&crate::text::json_surrogates(raw)) + } + + pub fn selected(&self, group: &str, tag: &str) -> bool { + selected(&self.update_fields, group, tag) + } + + pub fn any_selected(&self, group: &str, tags: &[&str]) -> bool { + tags.iter().any(|tag| self.selected(group, tag)) + } + + /// Canonical keys shared by native FLAC enrichment and the FFmpeg plan. + /// Empty values preserve existing tags, unlike an explicit editor clear. + pub fn write_metadata(&self, lyrics: &str) -> BTreeMap<String, String> { + let mut fields = BTreeMap::new(); + for (group, tag, key, value) in [ + ("basic_tags", "track_name", "TITLE", &self.track_name), + ("basic_tags", "artist_name", "ARTIST", &self.artist_name), + ("basic_tags", "album_name", "ALBUM", &self.album_name), + ( + "basic_tags", + "album_artist", + "ALBUMARTIST", + &self.album_artist, + ), + ("release_info", "release_date", "DATE", &self.release_date), + ("release_info", "isrc", "ISRC", &self.isrc), + ("extra", "genre", "GENRE", &self.genre), + ("extra", "label", "ORGANIZATION", &self.label), + ("extra", "copyright", "COPYRIGHT", &self.copyright), + ("extra", "composer", "COMPOSER", &self.composer), + ] { + if self.selected(group, tag) && !value.is_empty() { + fields.insert(key.into(), value.clone()); + } + } + for (tag, total_tag, key, number, total) in [ + ( + "track_number", + "total_tracks", + "TRACKNUMBER", + self.track_number, + self.total_tracks, + ), + ( + "disc_number", + "total_discs", + "DISCNUMBER", + self.disc_number, + self.total_discs, + ), + ] { + if self.any_selected("track_info", &[tag, total_tag]) && number > 0 { + fields.insert( + key.into(), + if total > 0 { + format!("{number}/{total}") + } else { + number.to_string() + }, + ); + } + } + if self.selected("lyrics", "lyrics") + && !self.lyrics_mode.trim().eq_ignore_ascii_case("external") + && !lyrics.is_empty() + { + fields.insert("LYRICS".into(), lyrics.into()); + fields.insert("UNSYNCEDLYRICS".into(), lyrics.into()); + } + fields + } + + pub fn write_external_lrc(&self, lyrics: &str) -> bool { + self.embed_lyrics + && self.selected("lyrics", "lyrics") + && !lyrics.trim().is_empty() + && matches!( + self.lyrics_mode.trim().to_ascii_lowercase().as_str(), + "external" | "both" + ) + } + + pub fn query(&self) -> String { + let mut parts: Vec<_> = [&self.track_name, &self.artist_name] + .into_iter() + .filter(|value| !placeholder(value)) + .map(|value| value.trim()) + .collect(); + if parts.is_empty() && !placeholder(&self.album_name) { + parts.push(self.album_name.trim()); + } + parts.join(" ").trim().into() + } + + pub fn apply(&mut self, track: &Value) { + let same_release = self.replace_release_metadata + || placeholder(&self.album_name) + || text(track, "album_name").trim().is_empty() + || titles_match(&self.album_name, text(track, "album_name")); + for (key, prefix) in [ + ("spotify_id", ""), + ("deezer_id", "deezer:"), + ("qobuz_id", "qobuz:"), + ("tidal_id", "tidal:"), + ("id", ""), + ] { + if !text(track, key).is_empty() { + self.spotify_id = format!("{prefix}{}", text(track, key)); + break; + } + } + let selected = |group, tag| selected(&self.update_fields, group, tag); + for (group, tag, source, target, release_only) in [ + ( + "basic_tags", + "track_name", + "name", + &mut self.track_name, + false, + ), + ( + "basic_tags", + "artist_name", + "artists", + &mut self.artist_name, + false, + ), + ( + "basic_tags", + "album_name", + "album_name", + &mut self.album_name, + true, + ), + ( + "basic_tags", + "album_artist", + "album_artist", + &mut self.album_artist, + true, + ), + ( + "release_info", + "release_date", + "release_date", + &mut self.release_date, + true, + ), + ("release_info", "isrc", "isrc", &mut self.isrc, false), + ("extra", "genre", "genre", &mut self.genre, false), + ("extra", "label", "label", &mut self.label, false), + ( + "extra", + "copyright", + "copyright", + &mut self.copyright, + false, + ), + ("extra", "composer", "composer", &mut self.composer, false), + ] { + let value = text(track, source); + if (!release_only || same_release) && selected(group, tag) && !value.is_empty() { + *target = value.into(); + } + } + for (tag, target) in [ + ("track_number", &mut self.track_number), + ("total_tracks", &mut self.total_tracks), + ("disc_number", &mut self.disc_number), + ("total_discs", &mut self.total_discs), + ] { + let value = number(track, tag); + if same_release && selected("track_info", tag) && value > 0 { + *target = value as isize; + } + } + if same_release + && selected("cover", "cover") + && let Some(cover) = [text(track, "cover_url"), text(track, "images")] + .into_iter() + .find(|value| !value.is_empty()) + { + self.cover_url = cover.into(); + } + if number(track, "duration_ms") > 0 { + self.duration_ms = number(track, "duration_ms"); + } + } + + pub fn result_metadata(&self) -> Value { + let mut result = json!({"spotify_id":self.spotify_id,"duration_ms":self.duration_ms}); + for (group, tag, key, value) in [ + ( + "basic_tags", + "track_name", + "track_name", + json!(self.track_name), + ), + ( + "basic_tags", + "artist_name", + "artist_name", + json!(self.artist_name), + ), + ( + "basic_tags", + "album_name", + "album_name", + json!(self.album_name), + ), + ( + "basic_tags", + "album_artist", + "album_artist", + json!(self.album_artist), + ), + ( + "track_info", + "track_number", + "track_number", + json!(self.track_number), + ), + ( + "track_info", + "total_tracks", + "total_tracks", + json!(self.total_tracks), + ), + ( + "track_info", + "disc_number", + "disc_number", + json!(self.disc_number), + ), + ( + "track_info", + "total_discs", + "total_discs", + json!(self.total_discs), + ), + ( + "release_info", + "release_date", + "release_date", + json!(self.release_date), + ), + ("release_info", "isrc", "isrc", json!(self.isrc)), + ("cover", "cover", "cover_url", json!(self.cover_url)), + ("extra", "genre", "genre", json!(self.genre)), + ("extra", "label", "label", json!(self.label)), + ("extra", "copyright", "copyright", json!(self.copyright)), + ("extra", "composer", "composer", json!(self.composer)), + ] { + if self.selected(group, tag) { + result[key] = value; + } + } + result + } + + /// ReEnrich passes no candidate album to trackMatchesRequest in Go. + pub fn verified(&self, track: &Value) -> bool { + let (title, artist, isrc) = ( + text(track, "name"), + text(track, "artists"), + text(track, "isrc"), + ); + let exact = !self.isrc.is_empty() + && !isrc.is_empty() + && self.isrc.trim().eq_ignore_ascii_case(isrc.trim()); + if !exact + && ((!self.artist_name.is_empty() + && !artist.is_empty() + && !artists_match(&self.artist_name, artist)) + || (!self.track_name.is_empty() + && !title.is_empty() + && !track_titles_match(&self.track_name, title))) + { + return false; + } + let (expected, duration) = (self.duration_ms / 1000, number(track, "duration_ms") / 1000); + if expected > 0 && duration > 0 && expected.abs_diff(duration) > 10 { + let identity = track_identity_title(&self.track_name); + let candidate_identity = track_identity_title(title); + let identity_matches = if !identity.is_empty() && !candidate_identity.is_empty() { + identity == candidate_identity + } else { + lowercase(self.track_name.trim()) == lowercase(title.trim()) + }; + return exact + && !self.track_name.is_empty() + && !title.is_empty() + && identity_matches + && !self.artist_name.is_empty() + && !artist.is_empty() + && artists_match(&self.artist_name, artist) + && !(duration <= 35 && expected > 45); + } + true + } + + pub fn select<'a>(&self, tracks: &'a [Value]) -> Option<&'a Value> { + let isrc = self.isrc.trim(); + let album = self.album_name.trim(); + let title = if placeholder(&self.track_name) { + "" + } else { + &self.track_name + }; + let artist = if placeholder(&self.artist_name) { + "" + } else { + &self.artist_name + }; + let mut best = None; + let mut best_score = i32::MIN; + for track in tracks { + let exact = !isrc.is_empty() && isrc.eq_ignore_ascii_case(text(track, "isrc").trim()); + let title_matches = !title.is_empty() + && !text(track, "name").is_empty() + && titles_match(title, text(track, "name")); + let artist_matches = !artist.is_empty() + && !text(track, "artists").is_empty() + && artists_match(artist, text(track, "artists")); + let album_matches = !album.is_empty() + && !text(track, "album_name").is_empty() + && titles_match(album, text(track, "album_name")); + let verified = self.verified(track); + if !exact + && ((!title.is_empty() && !title_matches) + || (!artist.is_empty() && !artist_matches) + || (title.is_empty() + && artist.is_empty() + && ((!album.is_empty() && !album_matches) + || (album.is_empty() && !verified)))) + { + continue; + } + let mut score = i32::from(verified) * 2000 + + i32::from(exact) * 10000 + + i32::from(title_matches) * 400 + + i32::from(artist_matches) * 320; + if !album.is_empty() && !text(track, "album_name").is_empty() { + if album_matches { + score += 120; + } else if lowercase(album).contains(&lowercase(text(track, "album_name"))) + || lowercase(text(track, "album_name")).contains(&lowercase(album)) + { + score += 50; + } + } + if self.duration_ms > 0 + && number(track, "duration_ms") > 0 + && (self.duration_ms / 1000).abs_diff(number(track, "duration_ms") / 1000) <= 10 + { + score += 80; + } + score += i32::from(!text(track, "release_date").is_empty()) * 70 + + i32::from(number(track, "track_number") > 0) * 20 + + i32::from(number(track, "disc_number") > 0) * 10 + + i32::from(!text(track, "isrc").is_empty()) * 40; + if score > best_score { + best_score = score; + best = Some(track); + } + } + best + } +} + +fn selected(fields: &Option<Vec<Option<String>>>, group: &str, tag: &str) -> bool { + fields.as_ref().is_none_or(|fields| { + fields.is_empty() + || fields.iter().any(|field| { + field + .as_deref() + .is_some_and(|field| field == group || field == tag) + }) + }) +} + +pub fn from_catalog(track: &super::TrackMetadata) -> Value { + let mut result = serde_json::to_value(track).expect("catalog metadata serialization"); + result["id"] = json!(track.spotify_id); + result["deezer_id"] = json!( + track + .spotify_id + .strip_prefix("deezer:") + .unwrap_or(&track.spotify_id) + .trim() + ); + result["cover_url"] = json!(track.images); + result["provider_id"] = json!("deezer"); + result +} diff --git a/rust_backend/crates/core/src/metadata/share.rs b/rust_backend/crates/core/src/metadata/share.rs new file mode 100644 index 00000000..f69c190a --- /dev/null +++ b/rust_backend/crates/core/src/metadata/share.rs @@ -0,0 +1,271 @@ +//! Collection matching and share links from already-decoded extension tracks. + +use crate::lyrics::json::go_deserialize; +use crate::lyrics::matching::{normalize_loose_artist, normalize_title}; +use crate::matching::lowercase; +use serde::Serialize; +use serde_json::Value; + +#[derive(Default)] +pub struct Request { + pub name: String, + pub artists: String, + pub kind: String, + pub source_extension_id: String, +} +go_deserialize!(Request { + "name" => name, "artists" => artists, "type" => kind, + "source_extension_id" => source_extension_id, +}); + +impl Request { + pub fn parse(raw: &str) -> Result<Self, serde_json::Error> { + let mut request: Self = serde_json::from_str(&crate::text::json_surrogates(raw))?; + request.name = request.name.trim().into(); + request.artists = request.artists.trim().into(); + request.source_extension_id = request.source_extension_id.trim().into(); + request.kind = lowercase(request.kind.trim()); + if request.kind.is_empty() { + request.kind = "album".into(); + } + Ok(request) + } + + pub fn query(&self) -> String { + if self.artists.is_empty() { + self.name.clone() + } else { + format!("{} {}", self.name, self.artists) + } + } + + pub fn cache_key(&self, providers: &[Provider]) -> String { + let mut identities: Vec<_> = providers + .iter() + .map(|provider| { + [ + provider.id.trim(), + provider.display_name.trim(), + provider.source_dir.trim(), + ] + .join("\x1f") + }) + .collect(); + identities.sort(); + [ + normalize_title(&self.kind), + normalize_title(&self.name), + normalize_loose_artist(&self.artists), + self.source_extension_id.clone(), + identities.join("\x1e"), + ] + .join("\x1d") + } +} + +pub struct Provider { + pub id: String, + pub display_name: String, + pub source_dir: String, + pub capabilities: Value, +} + +#[derive(Serialize)] +pub struct ShareResult { + pub extension_id: String, + pub display_name: String, + pub found: bool, + #[serde(skip_serializing_if = "String::is_empty")] + pub url: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub item_name: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub item_artists: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub error: String, +} + +impl ShareResult { + pub fn new(provider: &Provider) -> Self { + Self { + extension_id: provider.id.clone(), + display_name: if provider.display_name.is_empty() { + provider.id.clone() + } else { + provider.display_name.clone() + }, + found: false, + url: String::new(), + item_name: String::new(), + item_artists: String::new(), + error: String::new(), + } + } + + pub fn cacheable(&self) -> bool { + let error = lowercase(self.error.trim()); + self.found + || error.is_empty() + || matches!(error.as_str(), "no results" | "unsupported collection type") + || error.ends_with(" not found") + || error.contains("found without shareable link") + } +} + +fn text<'a>(value: &'a Value, key: &str) -> &'a str { + value[key].as_str().unwrap_or_default() +} + +fn is_collection(track: &Value, kind: &str) -> bool { + // Go's simple folding also accepts long s in the fixed ASCII kind "artist". + text(track, "item_type") + .trim() + .chars() + .map(|ch| { + if ch == 'ſ' { + 's' + } else { + ch.to_ascii_lowercase() + } + }) + .eq(kind.chars()) +} + +fn name<'a>(track: &'a Value, kind: &str) -> &'a str { + text( + track, + if is_collection(track, kind) { + "name" + } else if kind == "album" { + "album_name" + } else { + "artists" + }, + ) +} + +fn share_url(provider: &Provider, track: &Value, kind: &str) -> String { + let direct = |value: &str| { + let value = value.trim(); + (value.starts_with("http://") || value.starts_with("https://")).then_some(value.to_owned()) + }; + if is_collection(track, kind) + && let Some(url) = direct(text(track, "external_urls")) + { + return url; + } + let field = if kind == "album" { + "album_url" + } else { + "artist_url" + }; + if let Some(url) = direct(text(track, field)) { + return url; + } + if let Some(links) = track["external_links"].as_object() { + for (key, value) in links { + if lowercase(key).contains(kind) + && let Some(url) = direct(value.as_str().unwrap_or_default()) + { + return url; + } + } + } + let id_field = if kind == "album" { + "album_id" + } else { + "artist_id" + }; + let id = [ + text(track, id_field), + if is_collection(track, kind) { + text(track, "id") + } else { + "" + }, + if kind == "album" { + text(track, "album_url") + } else { + "" + }, + ] + .into_iter() + .map(str::trim) + .find(|id| !id.is_empty()) + .unwrap_or_default(); + if id.is_empty() { + return String::new(); + } + let id = id + .split_once(':') + .filter(|(prefix, suffix)| !prefix.is_empty() && !suffix.is_empty()) + .map_or(id, |(_, suffix)| suffix); + provider.capabilities["shareUrlTemplates"][kind] + .as_str() + .unwrap_or_default() + .trim() + .replace("{id}", id) +} + +pub fn select(provider: &Provider, request: &Request, tracks: &[Value]) -> ShareResult { + let mut result = ShareResult::new(provider); + if tracks.is_empty() { + result.error = "no results".into(); + return result; + } + let kind = request.kind.as_str(); + if !matches!(kind, "album" | "artist") { + result.error = "unsupported collection type".into(); + return result; + } + let normalize = if kind == "album" { + normalize_title + } else { + normalize_loose_artist + }; + let target = normalize(&request.name); + let artists = normalize_loose_artist(&request.artists); + let mut best = None; + let mut best_score = 0; + for track in tracks { + let candidate = normalize(name(track, kind)); + let mut score = if is_collection(track, kind) { 25 } else { 0 }; + if candidate == target { + score += 100; + } else if !candidate.is_empty() + && !target.is_empty() + && (candidate.contains(&target) || target.contains(&candidate)) + { + score += if kind == "album" { 50 } else { 60 }; + } + if kind == "album" && !artists.is_empty() { + let candidate_artists = normalize_loose_artist(&format!( + "{} {}", + text(track, "artists"), + text(track, "album_artist") + )); + if candidate_artists.contains(&artists) || artists.contains(&candidate_artists) { + score += 30; + } + } + if score > best_score { + best_score = score; + best = Some(track); + } + } + let Some(best) = best.filter(|_| best_score >= if kind == "album" { 50 } else { 60 }) else { + result.error = format!("{kind} not found"); + return result; + }; + result.url = share_url(provider, best, kind); + if result.url.is_empty() { + result.error = format!("{kind} found without shareable link"); + return result; + } + result.found = true; + result.item_name = name(best, kind).into(); + if kind == "album" { + result.item_artists = text(best, "artists").into(); + } + result +} diff --git a/rust_backend/crates/core/src/progress/mod.rs b/rust_backend/crates/core/src/progress/mod.rs new file mode 100644 index 00000000..0b3b7359 --- /dev/null +++ b/rust_backend/crates/core/src/progress/mod.rs @@ -0,0 +1,429 @@ +//! Go-compatible sparse download progress, shared by every native consumer. + +mod reporter; +mod subscription; + +pub use reporter::{ItemProgressWriter, ItemTransferProgressReporter}; +pub use subscription::ProgressSubscription; + +use serde::Serialize; +use std::collections::BTreeMap; +use std::fmt; +use std::sync::{Arc, Condvar, Mutex}; + +pub const UPDATE_BYTES: i64 = 128 * 1024; + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ItemProgress { + pub item_id: String, + pub bytes_total: i64, + pub bytes_received: i64, + pub progress: f64, + pub speed_mbps: f64, + pub is_downloading: bool, + pub status: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub stage: String, + #[serde(skip)] + revision: i64, +} + +#[derive(PartialEq)] +struct BridgeState { + bytes_bucket: i64, + total: i64, + percent: i64, + speed_tenths: i64, + downloading: bool, + status: String, + stage: String, +} + +impl ItemProgress { + fn bridge(&self) -> BridgeState { + let progress = if self.progress.is_nan() || self.progress <= 0.0 { + 0.0 + } else { + self.progress.min(1.0) + }; + let speed = if self.speed_mbps.is_nan() || self.speed_mbps <= 0.0 { + 0.0 + } else { + self.speed_mbps + }; + BridgeState { + bytes_bucket: self.bytes_received / UPDATE_BYTES, + total: self.bytes_total, + percent: (progress * 100.0).round() as i64, + speed_tenths: rounded_speed(speed * 10.0), + downloading: self.is_downloading, + status: self.status.clone(), + stage: self.stage.clone(), + } + } + + fn finite(&self) -> bool { + self.progress.is_finite() && self.speed_mbps.is_finite() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProgressError { + Closed, + SubscriptionClosed, +} + +impl fmt::Display for ProgressError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Closed => "download progress registry closed", + Self::SubscriptionClosed => "download progress subscription closed", + }) + } +} + +impl std::error::Error for ProgressError {} + +struct State { + items: BTreeMap<String, ItemProgress>, + removed: BTreeMap<String, i64>, + seq: i64, + reset: i64, + dirty: bool, + cached: String, + closed: bool, +} + +impl Default for State { + fn default() -> Self { + Self { + items: BTreeMap::new(), + removed: BTreeMap::new(), + seq: 0, + reset: 0, + dirty: true, + cached: "{\"items\":{}}".into(), + closed: false, + } + } +} + +impl State { + fn check(&self) -> Result<(), ProgressError> { + if self.closed { + Err(ProgressError::Closed) + } else { + Ok(()) + } + } + + fn next(&mut self, changed: &Condvar) -> i64 { + self.seq = self.seq.wrapping_add(1); + self.dirty = true; + changed.notify_all(); + self.seq + } + + fn delta(&self, since: i64) -> String { + if since >= self.seq { + return String::new(); + } + #[derive(Serialize)] + struct Delta<'a> { + seq: i64, + #[serde(skip_serializing_if = "std::ops::Not::not")] + reset: bool, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + items: BTreeMap<&'a String, &'a ItemProgress>, + #[serde(skip_serializing_if = "Vec::is_empty")] + removed: Vec<&'a String>, + } + let reset = since <= 0 || since < self.reset; + let items: BTreeMap<_, _> = self + .items + .iter() + .filter(|(_, item)| reset || item.revision > since) + .collect(); + // serde_json represents non-finite floats as null; encoding/json fails + // the whole document instead. Preserve Go's empty/fallback responses. + if items.values().any(|item| !item.finite()) { + return String::new(); + } + let removed = if reset { + Vec::new() + } else { + self.removed + .iter() + .filter(|(_, revision)| **revision > since) + .map(|(id, _)| id) + .collect() + }; + serde_json::to_string(&Delta { + seq: self.seq, + reset, + items, + removed, + }) + .unwrap_or_default() + } +} + +#[derive(Default)] +struct Shared { + state: Mutex<State>, + changed: Condvar, +} + +#[derive(Default)] +pub struct ProgressRegistry { + shared: Arc<Shared>, +} + +impl ProgressRegistry { + pub fn new() -> Self { + Self::default() + } + + pub fn start(&self, id: &str) -> Result<(), ProgressError> { + let mut state = self.shared.state.lock().expect("progress state lock"); + state.check()?; + let revision = state.next(&self.shared.changed); + state.items.insert( + id.to_owned(), + ItemProgress { + item_id: id.to_owned(), + bytes_total: 0, + bytes_received: 0, + progress: 0.0, + speed_mbps: 0.0, + is_downloading: false, + status: "preparing".into(), + stage: String::new(), + revision, + }, + ); + state.removed.remove(id); + Ok(()) + } + + fn update( + &self, + id: &str, + update: impl FnOnce(&mut ItemProgress), + ) -> Result<(), ProgressError> { + let mut state = self.shared.state.lock().expect("progress state lock"); + state.check()?; + if let Some(item) = state.items.get_mut(id) { + let before = item.bridge(); + update(item); + if item.bridge() != before { + let revision = state.next(&self.shared.changed); + state.items.get_mut(id).expect("existing progress").revision = revision; + } + } + Ok(()) + } + + pub fn preparing(&self, id: &str, stage: &str) -> Result<(), ProgressError> { + self.update(id, |item| { + item.progress = 0.0; + item.bytes_received = 0; + item.bytes_total = 0; + item.speed_mbps = 0.0; + item.is_downloading = true; + item.status = "preparing".into(); + item.stage = stage.to_owned(); + }) + } + + pub fn downloading(&self, id: &str) -> Result<(), ProgressError> { + self.update(id, mark_downloading) + } + + pub fn set_total(&self, id: &str, total: i64) -> Result<(), ProgressError> { + self.update(id, |item| item.bytes_total = total) + } + + pub fn set_received(&self, id: &str, received: i64) -> Result<(), ProgressError> { + self.update(id, |item| set_received(item, received)) + } + + pub fn set_received_with_speed( + &self, + id: &str, + received: i64, + speed: f64, + ) -> Result<(), ProgressError> { + self.update(id, |item| { + item.speed_mbps = speed; + set_received(item, received); + }) + } + + pub fn set_progress( + &self, + id: &str, + progress: f64, + received: i64, + total: i64, + ) -> Result<(), ProgressError> { + self.update(id, |item| { + let has_bytes = received > 0 || total > 0; + item.progress = if item.status != "preparing" || has_bytes || progress >= 1.0 { + progress + } else { + 0.0 + }; + if received > 0 { + item.bytes_received = received; + } + if total > 0 { + item.bytes_total = total; + } + if has_bytes || progress >= 1.0 || item.status == "downloading" { + mark_downloading(item); + } + }) + } + + pub fn finalizing(&self, id: &str) -> Result<(), ProgressError> { + self.update(id, |item| { + item.progress = 1.0; + item.status = "finalizing".into(); + item.stage.clear(); + }) + } + + pub fn complete(&self, id: &str) -> Result<(), ProgressError> { + self.update(id, |item| { + item.progress = 1.0; + item.is_downloading = false; + item.status = "completed".into(); + item.stage.clear(); + }) + } + + pub fn remove(&self, id: &str) -> Result<(), ProgressError> { + let mut state = self.shared.state.lock().expect("progress state lock"); + state.check()?; + if state.items.remove(id).is_some() { + let seq = state.next(&self.shared.changed); + state.removed.insert(id.to_owned(), seq); + if state.removed.len() > 512 { + let mut revisions: Vec<_> = state.removed.values().copied().collect(); + revisions.sort_unstable(); + let cutoff = revisions[revisions.len() / 2]; + state.removed.retain(|_, revision| *revision > cutoff); + state.reset = state.reset.max(cutoff); + } + } + // Even removing a missing ID invalidates the snapshot cache in Go. + state.dirty = true; + Ok(()) + } + + pub fn clear(&self) -> Result<(), ProgressError> { + let mut state = self.shared.state.lock().expect("progress state lock"); + state.check()?; + state.items.clear(); + state.removed.clear(); + state.reset = state.next(&self.shared.changed); + Ok(()) + } + + pub fn item(&self, id: &str) -> Result<String, ProgressError> { + let state = self.shared.state.lock().expect("progress state lock"); + state.check()?; + Ok(match state.items.get(id) { + Some(item) if item.finite() => serde_json::to_string(item).unwrap_or_default(), + Some(_) => String::new(), + None => "{}".into(), + }) + } + + pub fn snapshot(&self) -> Result<String, ProgressError> { + let mut state = self.shared.state.lock().expect("progress state lock"); + state.check()?; + if state.dirty { + if state.items.values().any(|item| !item.finite()) { + return Ok("{\"items\":{}}".into()); + } + #[derive(Serialize)] + struct Snapshot<'a> { + items: &'a BTreeMap<String, ItemProgress>, + } + state.cached = serde_json::to_string(&Snapshot { + items: &state.items, + }) + .expect("finite progress JSON"); + state.dirty = false; + } + Ok(state.cached.clone()) + } + + pub fn delta(&self, since: i64) -> Result<String, ProgressError> { + let state = self.shared.state.lock().expect("progress state lock"); + state.check()?; + Ok(state.delta(since)) + } + + pub fn subscribe(&self) -> Result<ProgressSubscription, ProgressError> { + self.shared + .state + .lock() + .expect("progress state lock") + .check()?; + Ok(ProgressSubscription::new(Arc::clone(&self.shared))) + } + + pub fn wait_delta(&self, since: i64, timeout_ms: i64) -> Result<String, ProgressError> { + self.subscribe()?.wait_delta(since, timeout_ms) + } + + pub fn shutdown(&self) { + let mut state = self.shared.state.lock().expect("progress state lock"); + state.closed = true; + state.items.clear(); + state.removed.clear(); + state.cached.clear(); + self.shared.changed.notify_all(); + } +} + +impl Drop for ProgressRegistry { + fn drop(&mut self) { + self.shutdown(); + } +} + +fn mark_downloading(item: &mut ItemProgress) { + item.is_downloading = true; + item.status = "downloading".into(); + item.stage.clear(); +} + +fn set_received(item: &mut ItemProgress, received: i64) { + item.bytes_received = received; + if item.bytes_total > 0 { + item.progress = received as f64 / item.bytes_total as f64; + } + if received > 0 { + mark_downloading(item); + } +} + +fn rounded_speed(value: f64) -> i64 { + let value = value.round(); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + if value >= 9_223_372_036_854_775_808.0 { + return i64::MIN; + } + #[cfg(target_arch = "arm")] + if value >= 18_446_744_073_709_551_616.0 || !value.is_finite() { + // Go ARM32's int64 conversion delegates out-of-range values to _d2v. + return (u64::from(value as u32) << 32) as i64; + } else if value >= 9_223_372_036_854_775_808.0 { + return value as u64 as i64; + } + value as i64 +} diff --git a/rust_backend/crates/core/src/progress/reporter.rs b/rust_backend/crates/core/src/progress/reporter.rs new file mode 100644 index 00000000..5f8f37a1 --- /dev/null +++ b/rust_backend/crates/core/src/progress/reporter.rs @@ -0,0 +1,162 @@ +use super::{ProgressRegistry, UPDATE_BYTES}; +use crate::cancellation::CancellationRegistry; +use std::io::{self, Write}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +pub struct ItemTransferProgressReporter { + progress: Arc<ProgressRegistry>, + id: String, + state: Mutex<ReportState>, +} + +struct ReportState { + received: i64, + total: i64, + at: Instant, +} + +impl ItemTransferProgressReporter { + pub fn new(progress: Arc<ProgressRegistry>, id: &str, received: i64, total: i64) -> Self { + Self { + progress, + id: id.to_owned(), + state: Mutex::new(ReportState { + received, + total, + at: Instant::now(), + }), + } + } + + pub fn report(&self, received: i64, total: i64) { + self.report_at(received, total, Instant::now()); + } + + fn report_at(&self, received: i64, total: i64, now: Instant) { + if self.id.is_empty() { + return; + } + let mut state = self.state.lock().expect("progress reporter lock"); + let delta = received.wrapping_sub(state.received); + if (0..UPDATE_BYTES).contains(&delta) + && total == state.total + && now.saturating_duration_since(state.at) < Duration::from_millis(250) + { + return; + } + *state = ReportState { + received, + total, + at: now, + }; + if total > 0 { + let _ = self.progress.set_progress( + &self.id, + received as f64 / total as f64, + received, + total, + ); + } else { + let _ = self.progress.set_received(&self.id, received); + } + } +} + +/// Adapter for media writers. Network transfer reporters deliberately do not +/// synthesize speed: the existing Go writer owns that separate behavior. +pub struct ItemProgressWriter<W> { + writer: W, + id: String, + progress: Arc<ProgressRegistry>, + cancellation: Arc<CancellationRegistry>, + current: i64, + last_reported: i64, + last_time: Instant, + last_bytes: i64, +} + +impl<W> ItemProgressWriter<W> { + pub fn new( + writer: W, + id: &str, + progress: Arc<ProgressRegistry>, + cancellation: Arc<CancellationRegistry>, + ) -> Self { + Self { + writer, + id: id.to_owned(), + progress, + cancellation, + current: 0, + last_reported: 0, + last_time: Instant::now(), + last_bytes: 0, + } + } + + pub fn into_inner(self) -> W { + self.writer + } +} + +impl<W: Write> Write for ItemProgressWriter<W> { + fn write(&mut self, bytes: &[u8]) -> io::Result<usize> { + if !self.id.is_empty() { + match self.cancellation.is_cancelled(&self.id) { + Ok(true) => return Err(io::Error::other("download cancelled")), + Err(error) => return Err(io::Error::other(error)), + Ok(false) => {} + } + } + let count = self.writer.write(bytes)?; + self.current = self.current.wrapping_add(count as i64); + if self.last_reported == 0 || self.current.wrapping_sub(self.last_reported) >= UPDATE_BYTES + { + let now = Instant::now(); + let elapsed = now.saturating_duration_since(self.last_time).as_secs_f64(); + let speed = if elapsed > 0.0 { + self.current.wrapping_sub(self.last_bytes) as f64 / (1024.0 * 1024.0) / elapsed + } else { + 0.0 + }; + let _ = self + .progress + .set_received_with_speed(&self.id, self.current, speed); + self.last_reported = self.current; + self.last_bytes = self.current; + self.last_time = now; + } + Ok(count) + } + + fn flush(&mut self) -> io::Result<()> { + self.writer.flush() + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn coalescing_flushes_threshold_interval_total_changes_and_rollbacks() { + let progress = Arc::new(ProgressRegistry::new()); + progress.start("example").unwrap(); + let reporter = + ItemTransferProgressReporter::new(Arc::clone(&progress), "example", 0, 1 << 20); + let now = reporter.state.lock().unwrap().at; + let received = || { + serde_json::from_str::<serde_json::Value>(&progress.item("example").unwrap()).unwrap()["bytes_received"].as_i64().unwrap() + }; + reporter.report_at(65536, 1 << 20, now); + assert_eq!(received(), 0); + reporter.report_at(UPDATE_BYTES, 1 << 20, now); + assert_eq!(received(), UPDATE_BYTES); + reporter.report_at(UPDATE_BYTES + 1, 1 << 20, now + Duration::from_millis(250)); + assert_eq!(received(), UPDATE_BYTES + 1); + reporter.report_at(UPDATE_BYTES + 2, 2 << 20, now + Duration::from_millis(250)); + assert_eq!(received(), UPDATE_BYTES + 2); + reporter.report_at(1, 2 << 20, now + Duration::from_millis(250)); + assert_eq!(received(), 1); + } +} diff --git a/rust_backend/crates/core/src/progress/subscription.rs b/rust_backend/crates/core/src/progress/subscription.rs new file mode 100644 index 00000000..4d431c27 --- /dev/null +++ b/rust_backend/crates/core/src/progress/subscription.rs @@ -0,0 +1,61 @@ +use super::{ProgressError, Shared}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +pub struct ProgressSubscription { + shared: Arc<Shared>, + closed: AtomicBool, +} + +impl ProgressSubscription { + pub(super) fn new(shared: Arc<Shared>) -> Self { + Self { + shared, + closed: AtomicBool::new(false), + } + } + + /// No event is consumed: independent UI/background listeners use their own + /// sequence, and every waiter sees the same revision. Close wakes only this + /// subscription semantically; other waiters continue against their deadline. + pub fn wait_delta(&self, since: i64, timeout_ms: i64) -> Result<String, ProgressError> { + let timeout = Duration::from_millis(if timeout_ms <= 0 { + 15_000 + } else { + timeout_ms.min(60_000) + } as u64); + let started = Instant::now(); + let mut state = self.shared.state.lock().expect("progress state lock"); + loop { + state.check()?; + if self.closed.load(Ordering::Acquire) { + return Err(ProgressError::SubscriptionClosed); + } + if since < state.seq { + return Ok(state.delta(since)); + } + let remaining = timeout.saturating_sub(started.elapsed()); + if remaining.is_zero() { + return Ok(String::new()); + } + (state, _) = self + .shared + .changed + .wait_timeout(state, remaining) + .expect("progress wait lock"); + } + } + + pub fn close(&self) { + let _state = self.shared.state.lock().expect("progress state lock"); + self.closed.store(true, Ordering::Release); + self.shared.changed.notify_all(); + } +} + +impl Drop for ProgressSubscription { + fn drop(&mut self) { + self.close(); + } +} diff --git a/rust_backend/crates/core/src/resolver.rs b/rust_backend/crates/core/src/resolver.rs new file mode 100644 index 00000000..6ff3c5bd --- /dev/null +++ b/rust_backend/crates/core/src/resolver.rs @@ -0,0 +1,339 @@ +//! Typed platform-resolver payloads and metadata matching shared with Go. + +use crate::lyrics::json::{Update, go_deserialize}; +use crate::lyrics::matching::normalize_loose_artist; +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::BTreeMap; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct Metadata { + pub title: String, + pub artist: String, +} + +/// Go reuses a non-nil map across repeated JSON fields, but resets it on null. +#[derive(Clone, Debug)] +pub struct Map<T>(pub BTreeMap<String, T>); + +impl<T> Default for Map<T> { + fn default() -> Self { + Self(BTreeMap::new()) + } +} + +impl<'de, T: Deserialize<'de>> Update<'de> for Map<T> { + fn update<D: Deserializer<'de>>(&mut self, decoder: D) -> Result<(), D::Error> { + match Option::<BTreeMap<String, T>>::deserialize(decoder)? { + Some(values) => self.0.extend(values), + None => self.0.clear(), + } + Ok(()) + } +} + +#[derive(Clone, Debug, Default)] +pub struct Link { + pub url: String, +} +go_deserialize!(Link { "url" => url, }); + +#[derive(Clone, Debug, Default)] +pub struct Entity { + pub title: String, + pub artist: String, +} +go_deserialize!(Entity { "title" => title, "artistname" => artist, }); + +#[derive(Clone, Debug, Default)] +pub struct UniTune { + pub entity_id: String, + pub links: Map<Link>, + pub entities: Map<Entity>, +} +go_deserialize!(UniTune { + "entityuniqueid" => entity_id, + "linksbyplatform" => links, + "entitiesbyuniqueid" => entities, +}); + +#[derive(Clone, Debug, Default)] +pub struct Artist { + pub name: String, +} +go_deserialize!(Artist { "name" => name, }); + +#[derive(Clone, Debug, Default)] +pub struct Recording { + pub id: String, + pub score: isize, + pub title: String, + pub artists: Option<Vec<Artist>>, +} +go_deserialize!(Recording { + "id" => id, "score" => score, "title" => title, "artist-credit" => artists, +}); + +#[derive(Clone, Debug, Default)] +pub struct Recordings { + pub recordings: Option<Vec<Recording>>, +} +go_deserialize!(Recordings { "recordings" => recordings, }); + +#[derive(Clone, Debug, Default)] +pub struct Resource { + pub resource: String, +} +go_deserialize!(Resource { "resource" => resource, }); + +#[derive(Clone, Debug, Default)] +pub struct Relation { + pub url: Resource, +} +go_deserialize!(Relation { "url" => url, }); + +#[derive(Clone, Debug, Default)] +pub struct Relations { + pub relations: Option<Vec<Relation>>, +} +go_deserialize!(Relations { "relations" => relations, }); + +#[derive(Clone, Debug, Default)] +pub struct Created { + pub full_url: String, + pub title: String, + pub artist: String, +} +go_deserialize!(Created { "full_url" => full_url, "title" => title, "artist" => artist, }); + +#[derive(Clone, Debug, Default)] +pub struct PageData { + pub title: String, + pub artist: String, + pub services: Map<Option<Link>>, +} +go_deserialize!(PageData { "title" => title, "artist" => artist, "services" => services, }); + +#[derive(Clone, Debug, Default)] +pub struct Page { + pub data: PageData, +} +go_deserialize!(Page { "data" => data, }); + +/// The platform resolver uses Go's broader metadata artist matcher, rather +/// than the stricter primary-artist equality required for lyrics searches. +pub fn artists_match(expected: &str, found: &str) -> bool { + let first = normalize_loose_artist(expected); + let second = normalize_loose_artist(found); + if first.contains(&second) || second.contains(&first) { + return true; + } + let split = |value: &str| { + let mut value = crate::matching::lowercase(value); + for separator in [ + " feat. ", " feat ", " ft. ", " ft ", " & ", " and ", ",", ";", " x ", + ] { + value = value.replace(separator, "|"); + } + value + .split('|') + .map(normalize_loose_artist) + .filter(|v| !v.is_empty()) + .collect::<Vec<_>>() + }; + for first in split(expected) { + for second in split(found) { + if first.contains(&second) || second.contains(&first) { + return true; + } + let mut first: Vec<_> = first.split_whitespace().collect(); + let mut second: Vec<_> = second.split_whitespace().collect(); + first.sort_unstable(); + second.sort_unstable(); + if first == second { + return true; + } + } + } + let latin = |value: &str| { + !value.chars().any(|ch| { + matches!(ch as u32, + 0x4e00..=0x9fff | 0x3040..=0x309f | 0x30a0..=0x30ff | + 0xac00..=0xd7af | 0x0600..=0x06ff | 0x0400..=0x04ff) + }) + }; + latin(expected) != latin(found) +} + +/// Album/title matching used by Go's metadata enrichment policy. +pub fn titles_match(expected: &str, found: &str) -> bool { + use crate::lyrics::matching::normalize_title; + use crate::matching::lowercase; + let (expected, found) = (lowercase(expected.trim()), lowercase(found.trim())); + let contains = |a: &str, b: &str| a.contains(b) || b.contains(a); + if contains(&expected, &found) { + return true; + } + let clean = |value: &str| { + let mut value = value.to_owned(); + let versions = [ + "remaster", + "remastered", + "deluxe", + "bonus", + "single", + "album version", + "radio edit", + "original mix", + "extended", + "club mix", + "remix", + "live", + "acoustic", + "demo", + ]; + for (open, close) in [('(', ')'), ('[', ']')] { + while let (Some(start), Some(end)) = (value.rfind(open), value.rfind(close)) { + if end <= start + || !versions + .iter() + .any(|word| value[start + 1..end].contains(word)) + { + break; + } + value = format!("{}{}", value[..start].trim(), &value[end + 1..]); + } + } + for suffix in [ + " - remaster", + " - remastered", + " - single version", + " - radio edit", + " - live", + " - acoustic", + " - demo", + " - remix", + ] { + if value.ends_with(suffix) { + value.truncate(value.len() - suffix.len()); + } + } + while value.contains(" ") { + value = value.replace(" ", " "); + } + value.trim().to_owned() + }; + let (a, b) = (clean(&expected), clean(&found)); + if a == b || (!a.is_empty() && !b.is_empty() && contains(&a, &b)) { + return true; + } + let core = |value: &str| { + let end = [value.find('('), value.find('['), value.find(" - ")] + .into_iter() + .flatten() + .filter(|index| *index > 0) + .min() + .unwrap_or(value.len()); + value[..end].trim().to_owned() + }; + let (a, b) = (core(&expected), core(&found)); + if !a.is_empty() && a == b { + return true; + } + let (a, b) = (normalize_title(&expected), normalize_title(&found)); + if !a.is_empty() && !b.is_empty() && contains(&a, &b) { + return true; + } + let symbols = |value: &str| { + use unicode_general_category::{GeneralCategory::*, get_general_category}; + value + .chars() + .filter(|ch| { + !ch.is_alphanumeric() + && !ch.is_whitespace() + && !matches!( + get_general_category(*ch), + ConnectorPunctuation + | DashPunctuation + | OpenPunctuation + | ClosePunctuation + | InitialPunctuation + | FinalPunctuation + | OtherPunctuation + | NonspacingMark + | SpacingMark + | EnclosingMark + ) + }) + .collect::<String>() + }; + if !expected.chars().any(char::is_alphanumeric) || !found.chars().any(char::is_alphanumeric) { + let a = symbols(&expected); + return !a.is_empty() && a == symbols(&found); + } + false +} + +pub fn track_identity_title(value: &str) -> String { + crate::lyrics::matching::normalize_title(&track_title_without_annotations(value)) +} + +fn track_title_without_annotations(value: &str) -> std::borrow::Cow<'_, str> { + static ANNOTATION: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| { + regex::Regex::new(r#"(?i)[(\[][\t\n\f\r ]*(?:(?:feat\.?|ft\.?|featuring)[\t\n\f\r ]+[^)\]]+|from[\t\n\f\r ]+["“][^)\]]+["”][\t\n\f\r ]*)[)\]]"#).unwrap() + }); + ANNOTATION.replace_all(value, " ") +} + +pub fn track_titles_match(expected: &str, found: &str) -> bool { + let (expected, found) = ( + track_title_without_annotations(expected), + track_title_without_annotations(found), + ); + let (a, b) = ( + crate::lyrics::matching::normalize_title(&expected), + crate::lyrics::matching::normalize_title(&found), + ); + if !a.is_empty() && a == b { + return true; + } + if a.split_whitespace() + .chain(b.split_whitespace()) + .any(|word| { + matches!( + word, + "mix" + | "remix" + | "live" + | "acoustic" + | "demo" + | "instrumental" + | "karaoke" + | "edit" + | "extended" + | "slowed" + | "sped" + ) + }) + { + return false; + } + titles_match(&expected, &found) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repeated_maps_merge_but_null_resets_and_nested_structs_retain_fields() { + let payload: UniTune = crate::lyrics::decode_document(br#"{"linksByPlatform":{"a":{"url":"first"}},"LINKSBYPLATFORM":{"b":{"URL":"second","url":null}},"unused":1e400}"#).unwrap(); + assert_eq!(payload.links.0.len(), 2); + assert_eq!(payload.links.0["b"].url, "second"); + let payload: Page = crate::lyrics::decode_response(br#"{"data":{"title":"Title","services":{"a":{"url":"old"}}},"data":{"services":null},"data":{"artist":"Artist","services":{"b":null}}}; ignored"#).unwrap(); + assert_eq!(payload.data.title, "Title"); + assert_eq!(payload.data.artist, "Artist"); + assert_eq!(payload.data.services.0.len(), 1); + assert!(payload.data.services.0["b"].is_none()); + } +} diff --git a/rust_backend/crates/core/src/tags.rs b/rust_backend/crates/core/src/tags.rs new file mode 100644 index 00000000..223169e4 --- /dev/null +++ b/rust_backend/crates/core/src/tags.rs @@ -0,0 +1,241 @@ +//! Audio tags used by metadata, lyrics and duplicate detection. Readers are +//! supplied by the native owner so descriptor paths need no temporary copy. + +mod comments; +mod containers; +mod file; +mod id3; +mod library; +mod mp4; +mod write; +use crate::text::utf8; +pub(crate) use containers::{ogg_packets, ogg_stream_is_opus}; +pub use file::{file_metadata_extension, read_file_metadata}; +pub use library::{ + library_extension, library_id, library_metadata, read_library_metadata, + read_library_metadata_with_cover, +}; +use serde::Serialize; +use std::io::{self, BufReader, Read, Seek, SeekFrom}; +pub use write::{ + embed_flac_metadata, rewrite_ac4_config, rewrite_ac4_metadata, rewrite_audio_tags, + rewrite_m4a_freeform, +}; + +#[derive(Clone, Debug)] +pub struct CoverArt { + pub data: Vec<u8>, + pub mime: String, +} + +/// Extract the embedded bytes without decoding the image or reading audio into +/// memory. Format selection/hints and output publication belong to the owner. +pub fn extract_cover( + reader: &mut (impl Read + Seek), + format: &str, + check: &dyn Fn() -> Result<(), String>, +) -> Result<CoverArt, String> { + check()?; + reader + .seek(SeekFrom::Start(0)) + .map_err(|error| error.to_string())?; + let mut observed = file::ObservedReader { + file: reader, + check, + failure: None, + }; + let mut reader = BufReader::new(&mut observed); + let result = match format { + "flac" => containers::flac_cover(&mut reader), + "m4a" | "aac" => mp4::cover(&mut reader), + "mp3" => id3::cover(&mut reader), + "ogg" | "opus" => containers::ogg_cover(&mut reader), + "wav" => containers::riff_cover(&mut reader, false), + "aiff" | "aif" | "aifc" => containers::riff_cover(&mut reader, true), + _ => Err("unsupported audio format for cover extraction".into()), + }; + if let Some(error) = observed.failure { + return Err(error); + } + check()?; + result +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize)] +pub struct AudioMetadata { + pub title: String, + pub artist: String, + pub album: String, + pub album_artist: String, + pub genre: String, + pub year: String, + pub date: String, + pub track_number: i64, + pub total_tracks: i64, + pub disc_number: i64, + pub total_discs: i64, + pub isrc: String, + pub lyrics: String, + pub label: String, + pub copyright: String, + pub composer: String, + pub comment: String, + pub album_type: String, + pub explicit: bool, + pub upc: String, + pub replay_gain_track_gain: String, + pub replay_gain_track_peak: String, + pub replay_gain_album_gain: String, + pub replay_gain_album_peak: String, +} + +struct CheckedReader<'a, R> { + reader: R, + check: &'a dyn Fn() -> Result<(), String>, +} + +impl<R: Read> Read for CheckedReader<'_, R> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> { + (self.check)().map_err(io::Error::other)?; + self.reader.read(buffer) + } +} + +impl<R: Seek> Seek for CheckedReader<'_, R> { + fn seek(&mut self, position: SeekFrom) -> io::Result<u64> { + (self.check)().map_err(io::Error::other)?; + self.reader.seek(position) + } +} + +struct TagReader<'a, R> { + reader: BufReader<CheckedReader<'a, R>>, + position: u64, +} + +impl<R: Read> Read for TagReader<'_, R> { + fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> { + (self.reader.get_ref().check)().map_err(io::Error::other)?; + let count = self.reader.read(bytes)?; + self.position += count as u64; + Ok(count) + } +} + +impl<R: Read + Seek> Seek for TagReader<'_, R> { + fn seek(&mut self, offset: SeekFrom) -> io::Result<u64> { + (self.reader.get_ref().check)().map_err(io::Error::other)?; + let target = match offset { + SeekFrom::Start(position) => position, + SeekFrom::Current(delta) => self + .position + .checked_add_signed(delta) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid tag seek"))?, + SeekFrom::End(_) => { + self.position = self.reader.seek(offset)?; + return Ok(self.position); + } + }; + if let Ok(delta) = i64::try_from(i128::from(target) - i128::from(self.position)) { + self.reader.seek_relative(delta)?; + } else { + self.reader.seek(SeekFrom::Start(target))?; + } + self.position = target; + Ok(target) + } + + fn stream_position(&mut self) -> io::Result<u64> { + (self.reader.get_ref().check)().map_err(io::Error::other)?; + Ok(self.position) + } +} + +pub fn read_audio_tags( + reader: &mut (impl Read + Seek), + format: &str, + check: &dyn Fn() -> Result<(), String>, +) -> Result<AudioMetadata, String> { + read_tags(reader, format, check, None) +} + +fn read_tags( + reader: &mut (impl Read + Seek), + format: &str, + check: &dyn Fn() -> Result<(), String>, + cover: Option<&mut Option<CoverArt>>, +) -> Result<AudioMetadata, String> { + check()?; + reader + .seek(SeekFrom::Start(0)) + .map_err(|error| error.to_string())?; + let mut reader = TagReader { + reader: BufReader::new(CheckedReader { reader, check }), + position: 0, + }; + let result = match format { + "flac" => match cover { + Some(cover) => containers::flac_with_cover(&mut reader, cover), + None => containers::flac(&mut reader), + }, + "m4a" | "mp4" | "aac" => mp4::read(&mut reader), + "mp3" => id3::read(&mut reader), + "ogg" | "opus" => containers::ogg(&mut reader), + "ape" | "wv" | "mpc" => containers::ape(&mut reader), + "wav" => containers::riff(&mut reader, false), + "aiff" | "aif" | "aifc" => containers::riff(&mut reader, true), + _ => Err("unsupported audio tag format".into()), + }; + check()?; + result +} + +fn exact(reader: &mut impl Read, bytes: &mut [u8]) -> Result<(), String> { + let mut offset = 0; + while offset < bytes.len() { + match reader + .read(&mut bytes[offset..]) + .map_err(|error| error.to_string())? + { + 0 => return Err(if offset == 0 { "EOF" } else { "unexpected EOF" }.into()), + count => offset += count, + } + } + Ok(()) +} + +fn bytes(reader: &mut impl Read, length: usize) -> Result<Vec<u8>, String> { + let mut bytes = vec![0; length]; + exact(reader, &mut bytes)?; + Ok(bytes) +} + +fn seek(reader: &mut impl Seek, position: SeekFrom) -> Result<u64, String> { + reader.seek(position).map_err(|error| error.to_string()) +} + +fn number(value: &str) -> i64 { + match value.trim().parse::<isize>() { + Ok(value) => value as i64, + Err(error) => match error.kind() { + std::num::IntErrorKind::PosOverflow => isize::MAX as i64, + std::num::IntErrorKind::NegOverflow => isize::MIN as i64, + _ => 0, + }, + } +} + +fn pair(value: &str) -> (i64, i64) { + let value = value.trim(); + match value.find('/').filter(|index| *index > 0) { + Some(index) => (number(&value[..index]), number(&value[index + 1..])), + None => (number(value), 0), + } +} + +fn truthy(value: &str) -> bool { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "explicit" + ) +} diff --git a/rust_backend/crates/core/src/tags/comments.rs b/rust_backend/crates/core/src/tags/comments.rs new file mode 100644 index 00000000..081acb46 --- /dev/null +++ b/rust_backend/crates/core/src/tags/comments.rs @@ -0,0 +1,242 @@ +use super::{AudioMetadata, pair, truthy, utf8}; +use crate::matching::{lowercase, uppercase}; + +pub(super) fn parse(data: &[u8], max_comment: usize) -> Vec<(String, String)> { + let Some(length) = data.get(..4) else { + return vec![]; + }; + let vendor = u32::from_le_bytes(length.try_into().unwrap()) as usize; + let Some(end) = vendor.checked_add(8) else { + return vec![]; + }; + let Some(count) = data.get(end - 4..end) else { + return vec![]; + }; + let count = u32::from_le_bytes(count.try_into().unwrap()); + let mut offset = end; + let mut items = Vec::new(); + let count = if max_comment == usize::MAX { + count + } else { + count.min(100) + }; + for _ in 0..count { + let Some(length) = data.get(offset..offset.saturating_add(4)) else { + break; + }; + let length = u32::from_le_bytes(length.try_into().unwrap()) as usize; + offset += 4; + let Some(value) = data.get(offset..offset.saturating_add(length)) else { + break; + }; + offset += length; + if length <= max_comment + && let Some(index) = value.iter().position(|value| *value == b'=') + { + items.push((utf8(&value[..index]), utf8(&value[index + 1..]))); + } + } + items +} + +fn joined<'a>(items: impl Iterator<Item = &'a str>) -> String { + let mut seen = std::collections::BTreeSet::new(); + items + .map(str::trim) + .filter(|value| !value.is_empty() && seen.insert(lowercase(value))) + .collect::<Vec<_>>() + .join(", ") +} + +pub(super) fn flac(metadata: &mut AudioMetadata, items: &[(String, String)]) { + let all = |name: &'static str| { + items + .iter() + .filter(move |(key, _)| key.len() == name.len() && uppercase(key) == name) + .map(|(_, value)| value.as_str()) + }; + let get = |name| all(name).next().unwrap_or("").to_owned(); + metadata.title = get("TITLE"); + metadata.artist = joined(all("ARTIST")); + metadata.album = get("ALBUM"); + metadata.album_artist = ["ALBUMARTIST", "ALBUM ARTIST", "ALBUM_ARTIST"] + .into_iter() + .map(|name| joined(all(name))) + .find(|value| !value.is_empty()) + .unwrap_or_default(); + metadata.date = get("DATE"); + if metadata.date.is_empty() { + metadata.date = get("YEAR"); + } + metadata.isrc = get("ISRC"); + metadata.lyrics = ["LYRICS", "UNSYNCEDLYRICS", "SYNCEDLYRICS"] + .into_iter() + .map(get) + .find(|value| !value.trim().is_empty()) + .unwrap_or_default(); + for (names, target, total) in [ + ( + ["TRACKNUMBER", "TRACK"], + &mut metadata.track_number, + &mut metadata.total_tracks, + ), + ( + ["DISCNUMBER", "DISC"], + &mut metadata.disc_number, + &mut metadata.total_discs, + ), + ] { + for (index, name) in names.into_iter().enumerate() { + if index == 1 && *target != 0 { + break; + } + let value = get(name); + if !value.is_empty() { + (*target, *total) = pair(&value); + } + } + } + metadata.genre = get("GENRE"); + metadata.label = ["ORGANIZATION", "LABEL", "PUBLISHER"] + .into_iter() + .map(get) + .find(|value| !value.is_empty()) + .unwrap_or_default(); + metadata.copyright = get("COPYRIGHT"); + metadata.composer = get("COMPOSER"); + metadata.comment = get("COMMENT"); + metadata.explicit = truthy(&get("ITUNESADVISORY")); + metadata.album_type = get("RELEASETYPE"); + if metadata.album_type.is_empty() && truthy(&get("COMPILATION")) { + metadata.album_type = "compilation".into(); + } + metadata.upc = get("BARCODE"); + if metadata.upc.is_empty() { + metadata.upc = get("UPC"); + } + metadata.replay_gain_track_gain = get("REPLAYGAIN_TRACK_GAIN"); + metadata.replay_gain_track_peak = get("REPLAYGAIN_TRACK_PEAK"); + metadata.replay_gain_album_gain = get("REPLAYGAIN_ALBUM_GAIN"); + metadata.replay_gain_album_peak = get("REPLAYGAIN_ALBUM_PEAK"); +} + +pub(super) fn apply(metadata: &mut AudioMetadata, key: &str, value: &str, ape: bool) { + let key = uppercase(if ape { key.trim() } else { key }); + let value = if ape { value.trim() } else { value }; + if ape && value.is_empty() { + return; + } + match key.as_str() { + "TITLE" => metadata.title = value.into(), + "ARTIST" => metadata.artist = value.into(), + "ALBUM" => metadata.album = value.into(), + "ALBUMARTIST" | "ALBUM ARTIST" => metadata.album_artist = value.into(), + "ALBUM_ARTIST" if !ape => metadata.album_artist = value.into(), + "DATE" | "YEAR" => { + if ape { + if key == "YEAR" { + metadata.year = value.into(); + } else { + metadata.date = value.into(); + } + } else { + metadata.date = value.into(); + if value.len() >= 4 { + metadata.year = utf8(&value.as_bytes()[..4]); + } + } + } + "GENRE" => metadata.genre = value.into(), + "TRACK" | "TRACKNUMBER" => (metadata.track_number, metadata.total_tracks) = pair(value), + "DISC" | "DISCNUMBER" => (metadata.disc_number, metadata.total_discs) = pair(value), + "ISRC" => metadata.isrc = value.into(), + "COMPOSER" => metadata.composer = value.into(), + "COMMENT" => metadata.comment = value.into(), + "DESCRIPTION" if !ape => metadata.comment = value.into(), + "LYRICS" | "UNSYNCEDLYRICS" | "SYNCEDLYRICS" if metadata.lyrics.is_empty() => { + metadata.lyrics = value.into() + } + "LABEL" | "PUBLISHER" => metadata.label = value.into(), + "ORGANIZATION" if !ape => metadata.label = value.into(), + "COPYRIGHT" => metadata.copyright = value.into(), + "ITUNESADVISORY" => metadata.explicit = truthy(value), + "RELEASETYPE" => metadata.album_type = value.into(), + "BARCODE" | "UPC" => metadata.upc = value.into(), + "COMPILATION" if truthy(value) && metadata.album_type.is_empty() => { + metadata.album_type = "compilation".into() + } + "REPLAYGAIN_TRACK_GAIN" => metadata.replay_gain_track_gain = value.into(), + "REPLAYGAIN_TRACK_PEAK" => metadata.replay_gain_track_peak = value.into(), + "REPLAYGAIN_ALBUM_GAIN" => metadata.replay_gain_album_gain = value.into(), + "REPLAYGAIN_ALBUM_PEAK" => metadata.replay_gain_album_peak = value.into(), + "R128_TRACK_GAIN" | "R128_ALBUM_GAIN" if !ape => { + let target = if key == "R128_TRACK_GAIN" { + &mut metadata.replay_gain_track_gain + } else { + &mut metadata.replay_gain_album_gain + }; + if target.is_empty() + && let Ok(value) = value.trim().parse::<i16>() + { + *target = format!("{:.2} dB", f64::from(value) / 256.0 + 5.0); + } + } + _ => {} + } +} + +pub(super) fn ape_key(key: &str) -> bool { + matches!( + uppercase(key.trim()).as_str(), + "TITLE" + | "ARTIST" + | "ALBUM" + | "ALBUMARTIST" + | "ALBUM ARTIST" + | "GENRE" + | "YEAR" + | "DATE" + | "TRACK" + | "TRACKNUMBER" + | "DISC" + | "DISCNUMBER" + | "ISRC" + | "LYRICS" + | "UNSYNCEDLYRICS" + | "SYNCEDLYRICS" + | "LABEL" + | "PUBLISHER" + | "COPYRIGHT" + | "COMPOSER" + | "COMMENT" + | "ITUNESADVISORY" + | "RELEASETYPE" + | "BARCODE" + | "UPC" + | "COMPILATION" + | "REPLAYGAIN_TRACK_GAIN" + | "REPLAYGAIN_TRACK_PEAK" + | "REPLAYGAIN_ALBUM_GAIN" + | "REPLAYGAIN_ALBUM_PEAK" + ) +} + +pub(super) fn ogg(items: &[(String, String)]) -> AudioMetadata { + let mut metadata = AudioMetadata::default(); + let mut artists = Vec::new(); + let mut album_artists = Vec::new(); + for (key, value) in items { + match uppercase(key).as_str() { + "ARTIST" => artists.push(value.as_str()), + "ALBUMARTIST" | "ALBUM_ARTIST" | "ALBUM ARTIST" => album_artists.push(value.as_str()), + _ => apply(&mut metadata, key, value, false), + } + } + if !artists.is_empty() { + metadata.artist = joined(artists.into_iter()); + } + if !album_artists.is_empty() { + metadata.album_artist = joined(album_artists.into_iter()); + } + metadata +} diff --git a/rust_backend/crates/core/src/tags/containers.rs b/rust_backend/crates/core/src/tags/containers.rs new file mode 100644 index 00000000..874fb7ce --- /dev/null +++ b/rust_backend/crates/core/src/tags/containers.rs @@ -0,0 +1,552 @@ +use super::{AudioMetadata, CoverArt, bytes, comments, exact, id3, seek, utf8}; +use base64::Engine; +use std::collections::BTreeMap; +use std::io::{Cursor, Read, Seek, SeekFrom}; + +pub(super) fn flac(reader: &mut (impl Read + Seek)) -> Result<AudioMetadata, String> { + flac_inner(reader, None).map_err(|error| format!("failed to parse FLAC file: {error}")) +} + +pub(super) fn flac_cover(reader: &mut (impl Read + Seek)) -> Result<CoverArt, String> { + let mut cover = None; + flac_with_cover(reader, &mut cover)?; + cover.ok_or_else(|| "no cover art found in file".into()) +} + +pub(super) fn flac_with_cover( + reader: &mut (impl Read + Seek), + cover: &mut Option<CoverArt>, +) -> Result<AudioMetadata, String> { + let metadata = flac_inner(reader, Some(cover)) + .map_err(|error| format!("failed to parse FLAC file: {error}"))?; + if let Some(cover) = cover { + cover.mime = if cover.data.len() > 8 && &cover.data[1..4] == b"PNG" { + "image/png" + } else { + "image/jpeg" + } + .into(); + } + Ok(metadata) +} + +fn flac_inner( + reader: &mut (impl Read + Seek), + mut cover: Option<&mut Option<CoverArt>>, +) -> Result<AudioMetadata, String> { + let mut signature = [0; 4]; + exact(reader, &mut signature)?; + if &signature != b"fLaC" { + return Err("fLaC head incorrect".into()); + } + let end = seek(reader, SeekFrom::End(0))?; + seek(reader, SeekFrom::Start(4))?; + let mut metadata = AudioMetadata::default(); + let mut found = false; + let mut front_cover = false; + loop { + let mut header = [0; 4]; + exact(reader, &mut header)?; + let length = u32::from_be_bytes([0, header[1], header[2], header[3]]); + let position = reader + .stream_position() + .map_err(|error| error.to_string())?; + if position + u64::from(length) > end { + return Err(if position == end { + "EOF" + } else { + "unexpected EOF" + } + .into()); + } + if header[0] & 0x7f == 4 && !found { + let data = bytes(reader, length as usize)?; + // FLAC uses the first valid Vorbis block; later blocks do not merge. + if valid_comments(&data) { + comments::flac(&mut metadata, &comments::parse(&data, usize::MAX)); + found = true; + } + } else if header[0] & 0x7f == 6 && !front_cover && cover.is_some() { + let data = bytes(reader, length as usize)?; + if let Some((kind, picture)) = picture_block(&data, false) + && let Some(cover) = cover.as_deref_mut() + && (kind == 3 || cover.is_none()) + { + front_cover = kind == 3; + *cover = Some(picture); + } + } else { + seek(reader, SeekFrom::Current(i64::from(length)))?; + } + if header[0] & 0x80 != 0 { + let mut sync = [0; 2]; + exact(reader, &mut sync)?; + if sync[0] != 0xff || sync[1] >> 2 != 0x3e { + return Err("frames do not begin with sync code".into()); + } + return Ok(metadata); + } + } +} + +fn picture_block(data: &[u8], ogg_limits: bool) -> Option<(u32, CoverArt)> { + let parse = || -> Result<(u32, CoverArt), String> { + let mut input = Cursor::new(data); + let kind = u32::from_be_bytes(bytes(&mut input, 4)?.try_into().unwrap()); + let mime_length = u32::from_be_bytes(bytes(&mut input, 4)?.try_into().unwrap()) as usize; + if mime_length > data.len().saturating_sub(8) || ogg_limits && mime_length > 256 { + return Err("invalid picture MIME length".into()); + } + let mime = utf8(&bytes(&mut input, mime_length)?); + let description = u32::from_be_bytes(bytes(&mut input, 4)?.try_into().unwrap()); + if ogg_limits && description > 10000 { + return Err("invalid picture description length".into()); + } + seek(&mut input, SeekFrom::Current(i64::from(description) + 16))?; + let length = u32::from_be_bytes(bytes(&mut input, 4)?.try_into().unwrap()) as usize; + if length == 0 + || length > data.len().saturating_sub(input.position() as usize) + || ogg_limits && length > 10_000_000 + { + return Err("invalid picture data length".into()); + } + Ok(( + kind, + CoverArt { + data: bytes(&mut input, length)?, + mime, + }, + )) + }; + parse().ok() +} + +fn valid_comments(data: &[u8]) -> bool { + let Some(vendor) = data.get(..4) else { + return false; + }; + let Some(mut offset) = (u32::from_le_bytes(vendor.try_into().unwrap()) as usize).checked_add(8) + else { + return false; + }; + let Some(count) = data.get(offset - 4..offset) else { + return false; + }; + for _ in 0..u32::from_le_bytes(count.try_into().unwrap()) { + let Some(length) = data.get(offset..offset.saturating_add(4)) else { + return false; + }; + let length = u32::from_le_bytes(length.try_into().unwrap()) as usize; + offset = offset.saturating_add(4).saturating_add(length); + // bytes.Reader.Read reports EOF even for a zero-length comment when + // its length field exhausted the block; Go skips that invalid block. + if offset > data.len() || length == 0 && offset == data.len() { + return false; + } + } + true +} + +#[derive(Clone)] +pub(crate) struct OggPage { + pub(crate) header: [u8; 27], + pub(crate) segments: Vec<u8>, + pub(crate) data: Vec<u8>, +} + +impl OggPage { + pub(crate) fn read(reader: &mut impl Read) -> Result<Option<Self>, String> { + let mut header = [0; 27]; + let mut offset = 0; + while offset < header.len() { + let count = reader + .read(&mut header[offset..]) + .map_err(|error| error.to_string())?; + if count == 0 { + return if offset == 0 { + Ok(None) + } else { + Err("unexpected EOF".into()) + }; + } + offset += count; + } + if &header[..4] != b"OggS" { + return Err("not an Ogg page".into()); + } + let segments = bytes(reader, header[26] as usize)?; + let data = bytes(reader, segments.iter().map(|n| *n as usize).sum())?; + Ok(Some(Self { + header, + segments, + data, + })) + } + + pub(crate) fn len(&self) -> usize { + 27 + self.segments.len() + self.data.len() + } +} + +pub(crate) fn ogg_packets( + reader: &mut impl Read, + max_packets: usize, + max_pages: usize, +) -> Result<Vec<Vec<u8>>, String> { + let mut packets = Vec::new(); + let mut current = Vec::new(); + let mut skip_packet = false; + for _ in 0..max_pages { + let page = OggPage::read(reader).and_then(|page| page.ok_or_else(|| "EOF".into())); + let OggPage { + header, + segments, + data, + } = match page { + Ok(page) => page, + Err(error) if packets.is_empty() => return Err(error), + Err(_) => break, + }; + if header[5] & 1 == 0 && !current.is_empty() { + current.clear(); + skip_packet = false; + } + let mut offset = 0; + for length in segments { + let length = length as usize; + if !skip_packet && current.len() + length > 10 * 1024 * 1024 { + current.clear(); + skip_packet = true; + } + if !skip_packet { + current.extend_from_slice(&data[offset..offset + length]); + } + offset += length; + if length < 255 { + if !current.is_empty() { + packets.push(std::mem::take(&mut current)); + } + skip_packet = false; + if packets.len() >= max_packets { + break; + } + } + } + if packets.len() >= max_packets { + break; + } + } + Ok(packets) +} + +pub(crate) fn ogg_stream_is_opus(packets: &[Vec<u8>]) -> Option<bool> { + packets.iter().find_map(|packet| { + if packet.starts_with(b"OpusHead") { + Some(true) + } else if packet.len() > 7 && packet.starts_with(b"\x01vorbis") { + Some(false) + } else { + None + } + }) +} + +pub(super) fn ogg(reader: &mut impl Read) -> Result<AudioMetadata, String> { + let packets = ogg_packets(reader, 30, 80)?; + let stream = ogg_stream_is_opus(&packets); + let packet = packets.iter().find_map(|packet| { + if stream != Some(false) && packet.len() > 8 && packet.starts_with(b"OpusTags") { + Some(&packet[8..]) + } else if stream != Some(true) && packet.len() > 7 && packet.starts_with(b"\x03vorbis") { + Some(&packet[7..]) + } else { + None + } + }); + let metadata = packet + .map(|data| comments::ogg(&comments::parse(data, 512 * 1024))) + .unwrap_or_default(); + if metadata.title.is_empty() + && metadata.artist.is_empty() + && metadata.replay_gain_track_gain.is_empty() + && metadata.replay_gain_album_gain.is_empty() + { + Err("no Vorbis comments found".into()) + } else { + Ok(metadata) + } +} + +pub(super) fn ogg_cover(reader: &mut impl Read) -> Result<CoverArt, String> { + let packets = ogg_packets(reader, 30, 80)?; + let stream = ogg_stream_is_opus(&packets); + for packet in &packets { + let comments = if stream != Some(false) && packet.starts_with(b"OpusTags") { + &packet[8..] + } else if stream != Some(true) && packet.starts_with(b"\x03vorbis") { + &packet[7..] + } else { + continue; + }; + for (key, value) in comments::parse(comments, 10_000_000) { + if !key.eq_ignore_ascii_case("METADATA_BLOCK_PICTURE") { + continue; + } + let value: Vec<_> = value + .bytes() + .filter(|byte| !matches!(byte, b' ' | b'\t' | b'\r' | b'\n')) + .collect(); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&value) + .or_else(|_| base64::engine::general_purpose::STANDARD_NO_PAD.decode(&value)); + if let Ok(decoded) = decoded + && let Some((_, cover)) = picture_block(&decoded, true) + { + return Ok(cover); + } + } + } + Err("no cover art found".into()) +} + +pub(super) fn ape(reader: &mut (impl Read + Seek)) -> Result<AudioMetadata, String> { + let end = seek(reader, SeekFrom::End(0))?; + if end < 32 { + return Err("file too small for APE tag".into()); + } + for offset in [Some(end - 32), end.checked_sub(161).map(|_| end - 160)] + .into_iter() + .flatten() + { + if let Ok(metadata) = ape_at(reader, offset) { + return Ok(metadata); + } + } + Err("no APEv2 tag found".into()) +} + +pub(crate) struct ApeFooter { + pub(crate) version: u32, + pub(crate) size: u32, + pub(crate) count: u32, + pub(crate) flags: u32, +} + +impl ApeFooter { + pub(crate) fn read( + reader: &mut (impl Read + Seek), + offset: u64, + ) -> Result<Option<Self>, String> { + seek(reader, SeekFrom::Start(offset))?; + let data = bytes(reader, 32)?; + let integer = |index| u32::from_le_bytes(data[index..index + 4].try_into().unwrap()); + if &data[..8] != b"APETAGEX" || integer(20) & (1 << 29) != 0 { + return Ok(None); + } + Ok(Some(Self { + version: integer(8), + size: integer(12), + count: integer(16), + flags: integer(20), + })) + } + + pub(crate) fn items_start(&self, offset: u64) -> Result<u64, String> { + if !matches!(self.version, 1000 | 2000) || self.size < 32 || self.count > 1000 { + return Err("invalid APE footer".into()); + } + offset + .checked_sub(u64::from(self.size - 32)) + .ok_or_else(|| "invalid APE items offset".into()) + } +} + +fn ape_at(reader: &mut (impl Read + Seek), footer: u64) -> Result<AudioMetadata, String> { + let header = ApeFooter::read(reader, footer)?.ok_or("invalid APE footer")?; + let start = header.items_start(footer)?; + seek(reader, SeekFrom::Start(start))?; + let mut remaining = footer - start; + let mut metadata = AudioMetadata::default(); + for _ in 0..header.count { + if remaining < 8 { + break; + } + let header = bytes(reader, 8)?; + remaining -= 8; + let length = u64::from(u32::from_le_bytes(header[..4].try_into().unwrap())); + let mut key = Vec::new(); + let mut terminated = false; + while remaining > 0 { + let mut value = [0]; + exact(reader, &mut value)?; + remaining -= 1; + if value[0] == 0 { + terminated = true; + break; + } + // Unknown overlong keys cannot match any supported metadata field. + if key.len() <= 1024 { + key.push(value[0]); + } + } + if !terminated || length > remaining { + break; + } + if length <= 32 * 1024 * 1024 && key.len() <= 1024 && comments::ape_key(&utf8(&key)) { + let value = bytes(reader, length as usize)?; + comments::apply(&mut metadata, &utf8(&key), &utf8(&value), true); + } else { + seek(reader, SeekFrom::Current(length as i64))?; + } + remaining -= length; + } + Ok(metadata) +} + +pub(super) fn riff(reader: &mut (impl Read + Seek), aiff: bool) -> Result<AudioMetadata, String> { + let (tag, info) = riff_chunks(reader, aiff)?; + riff_metadata(&tag, &info, aiff) +} + +pub(super) fn riff_cover(reader: &mut (impl Read + Seek), aiff: bool) -> Result<CoverArt, String> { + let (tag, _) = riff_chunks(reader, aiff).map_err(|_| "no embedded cover")?; + let (data, mime) = super::write::embedded_cover(&tag).ok_or("no embedded cover")?; + Ok(CoverArt { data, mime }) +} + +fn riff_chunks( + reader: &mut (impl Read + Seek), + aiff: bool, +) -> Result<(Vec<u8>, BTreeMap<String, String>), String> { + let header = bytes(reader, 12)?; + if if aiff { + &header[..4] != b"FORM" || !matches!(&header[8..], b"AIFF" | b"AIFC") + } else { + &header[..4] != b"RIFF" || &header[8..] != b"WAVE" + } { + return Err(if aiff { + "not an AIFF file" + } else { + "not a WAVE file" + } + .into()); + } + let mut tag = Vec::new(); + let mut info = BTreeMap::<String, String>::new(); + loop { + let mut header = [0; 8]; + if exact(reader, &mut header).is_err() { + break; + } + let length = if aiff { + u32::from_be_bytes(header[4..].try_into().unwrap()) + } else { + u32::from_le_bytes(header[4..].try_into().unwrap()) + }; + let wanted = matches!(&header[..4], b"ID3 " | b"id3 ") + || if aiff { + matches!(&header[..4], b"NAME" | b"AUTH" | b"ANNO" | b"(c) ") + } else { + &header[..4] == b"LIST" + }; + if wanted && length > 0 && length <= 16 * 1024 * 1024 { + if let Ok(data) = bytes(reader, length as usize) { + if matches!(&header[..4], b"ID3 " | b"id3 ") { + tag = data; + } else if aiff { + info.insert( + utf8(&header[..4]), + utf8(&data).trim().trim_end_matches('\0').into(), + ); + } else { + riff_info(&data, &mut info); + } + } + seek(reader, SeekFrom::Current(i64::from(length & 1)))?; + } else { + seek( + reader, + SeekFrom::Current(i64::from(length) + i64::from(length & 1)), + )?; + } + } + Ok((tag, info)) +} + +fn riff_metadata( + tag: &[u8], + info: &BTreeMap<String, String>, + aiff: bool, +) -> Result<AudioMetadata, String> { + if let Ok(metadata) = id3::embedded(tag) + && has_tags(&metadata) + { + return Ok(metadata); + } + let get = |key| info.get(key).cloned().unwrap_or_default(); + let metadata = if aiff { + AudioMetadata { + title: get("NAME"), + artist: get("AUTH"), + comment: get("ANNO"), + copyright: get("(c) "), + ..AudioMetadata::default() + } + } else { + let date = get("ICRD"); + AudioMetadata { + title: get("INAM"), + artist: get("IART"), + album: get("IPRD"), + genre: id3::genre(&get("IGNR")), + year: if date.len() >= 4 { + utf8(&date.as_bytes()[..4]) + } else { + String::new() + }, + date, + comment: get("ICMT"), + copyright: get("ICOP"), + composer: get("IMUS"), + track_number: get("ITRK").trim().parse::<isize>().unwrap_or(0) as i64, + ..AudioMetadata::default() + } + }; + if has_tags(&metadata) { + Ok(metadata) + } else { + Err(if aiff { + "no AIFF tags found" + } else { + "no WAV tags found" + } + .into()) + } +} + +fn has_tags(metadata: &AudioMetadata) -> bool { + !metadata.title.is_empty() || !metadata.artist.is_empty() || !metadata.album.is_empty() +} + +fn riff_info(data: &[u8], info: &mut BTreeMap<String, String>) { + if !data.starts_with(b"INFO") { + return; + } + let mut offset: usize = 4; + while let Some(header) = data.get(offset..offset.saturating_add(8)) { + let length = u32::from_le_bytes(header[4..].try_into().unwrap()) as usize; + offset += 8; + if length == 0 { + break; + } + let Some(value) = data.get(offset..offset.saturating_add(length)) else { + break; + }; + info.insert( + utf8(&header[..4]), + utf8(value).trim_end_matches('\0').trim().into(), + ); + offset += length; + offset = offset.saturating_add(length & 1); + } +} diff --git a/rust_backend/crates/core/src/tags/file.rs b/rust_backend/crates/core/src/tags/file.rs new file mode 100644 index 00000000..06fe1952 --- /dev/null +++ b/rust_backend/crates/core/src/tags/file.rs @@ -0,0 +1,214 @@ +//! Application metadata JSON, preserving the distinct legacy tag-reader JSON. + +use super::{AudioMetadata, read_audio_tags}; +use crate::media::{mp3_quality, ogg_quality, probe_mp4_quality, probe_quality, riff_quality}; +use serde_json::{Map, Value}; +use std::io::{self, Read, Seek, SeekFrom}; + +/// Read the public ReadFileMetadataWithHint payload from an already-owned file. +/// The path remains its identity; a display hint supplies only a missing suffix. +pub fn read_file_metadata( + file: &mut (impl Read + Seek), + path: &str, + hint: &str, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Value, String> { + check()?; + let mut file = ObservedReader { + file, + check, + failure: None, + }; + let result = read_metadata(&mut file, path, hint); + // Parsers tolerate absent/corrupt tags, but must not turn an OS error or + // cancellation into success, even if the parser swallowed a read error. + if let Some(error) = file.failure { + return Err(error); + } + check()?; + result +} + +fn read_metadata(file: &mut (impl Read + Seek), path: &str, hint: &str) -> Result<Value, String> { + let extension = file_metadata_extension(path, hint)?; + let (mut format, mut codec) = match extension.as_str() { + ".flac" => ("flac", "flac"), + ".m4a" | ".mp4" | ".aac" => ("m4a", ""), + ".mp3" => ("mp3", "mp3"), + ".ogg" | ".opus" => ("opus", "opus"), + ".ape" => ("ape", "ape"), + ".wv" => ("wv", "wv"), + ".mpc" => ("mpc", "mpc"), + ".wav" => ("wav", "pcm"), + ".aiff" | ".aif" | ".aifc" => ("aiff", "pcm"), + _ => unreachable!("validated metadata extension"), + }; + let size = file.seek(SeekFrom::End(0)).map_err(|e| e.to_string())? as i64; + let mut metadata = read_audio_tags(file, &extension[1..], &|| Ok(())); + if format == "flac" && metadata.is_err() { + let fallback = read_audio_tags(file, "ogg", &|| Ok(())); + if fallback.is_ok() { + metadata = fallback; + format = "opus"; + codec = "opus"; + } else { + return Err(format!( + "failed to read metadata: {}", + metadata.unwrap_err() + )); + } + } + let mut result = match metadata { + Ok(metadata) => tag_fields(metadata, format != "flac")?, + Err(_) => { + let mut fields = tag_fields(AudioMetadata::default(), false)?; + fields.retain(|key, _| !key.starts_with("replaygain_")); + fields + } + }; + result.insert("format".into(), format.into()); + result.insert("audio_codec".into(), codec.into()); + result.insert("duration".into(), 0.into()); + file.seek(SeekFrom::Start(0)).map_err(|e| e.to_string())?; + let quality = match format { + "flac" => probe_quality(file, &|| Ok(())), + "m4a" => probe_mp4_quality(file, &|| Ok(())), + "mp3" => mp3_quality(file, size), + "opus" => ogg_quality(file, size, path), + "wav" | "aiff" => riff_quality(file, size, format == "aiff"), + _ => return Ok(result.into()), + }; + if let Ok(quality) = quality { + result.insert("sample_rate".into(), quality.sample_rate.into()); + result.insert("duration".into(), quality.duration.into()); + if format != "opus" { + result.insert("bit_depth".into(), quality.bit_depth.into()); + } + let bitrate = match format { + "flac" => { + if !quality.codec.is_empty() { + result.insert("audio_codec".into(), quality.codec.into()); + } + if size > 0 && quality.sample_rate > 0 && quality.total_samples > 0 { + Some( + (size as f64 * 8.0 + / (quality.total_samples as f64 / quality.sample_rate as f64) + / 1000.0) as i64, + ) + } else { + None + } + } + "m4a" => { + let mapped = match quality.codec.trim().to_lowercase().as_str() { + "flac" => "flac", + "alac" => "alac", + "eac3" | "ec-3" => "eac3", + "ac3" | "ac-3" => "ac3", + "ac4" | "ac-4" => "ac4", + "opus" => "opus", + _ => "m4a", + }; + result.insert("format".into(), mapped.into()); + result.insert("audio_codec".into(), quality.codec.into()); + if quality.bitrate > 0 && !matches!(mapped, "flac" | "alac") { + Some(quality.bitrate) + } else if quality.duration > 0 && size > 0 { + Some((size as f64 * 8.0 / quality.duration as f64 / 1000.0) as i64) + } else { + None + } + } + "mp3" | "opus" if quality.bitrate > 0 => Some(quality.bitrate / 1000), + _ => None, + }; + if let Some(bitrate) = bitrate { + result.insert("bitrate".into(), bitrate.into()); + } + } + Ok(result.into()) +} + +/// Validate before opening the native path: Go rejects unsupported suffixes +/// even when that path does not exist or cannot be opened. +pub fn file_metadata_extension(path: &str, hint: &str) -> Result<String, String> { + let path_ext = extension(path); + let suffix = if path_ext.is_empty() { + extension(hint) + } else { + path_ext + } + .to_lowercase(); + match suffix.as_str() { + ".flac" | ".m4a" | ".mp4" | ".aac" | ".mp3" | ".ogg" | ".opus" | ".ape" | ".wv" + | ".mpc" | ".wav" | ".aiff" | ".aif" | ".aifc" => Ok(suffix), + _ => Err(format!("unsupported file format: {path}")), + } +} + +fn tag_fields( + mut metadata: AudioMetadata, + year_fallback: bool, +) -> Result<Map<String, Value>, String> { + if year_fallback && metadata.date.is_empty() { + metadata.date = metadata.year.clone(); + } + let Value::Object(mut fields) = serde_json::to_value(metadata).map_err(|e| e.to_string())? + else { + unreachable!("AudioMetadata serializes as an object"); + }; + fields.remove("year"); + for suffix in ["track_gain", "track_peak", "album_gain", "album_peak"] { + if let Some(value) = fields.remove(&format!("replay_gain_{suffix}")) { + fields.insert(format!("replaygain_{suffix}"), value); + } + } + Ok(fields) +} + +// filepath.Ext includes a leading dotfile and a trailing dot, unlike Path::extension. +fn extension(path: &str) -> &str { + let name = path.rsplit('/').next().unwrap_or_default(); + name.rfind('.').map_or("", |index| &name[index..]) +} + +pub(super) struct ObservedReader<'a, R> { + pub(super) file: &'a mut R, + pub(super) check: &'a dyn Fn() -> Result<(), String>, + pub(super) failure: Option<String>, +} + +impl<R> ObservedReader<'_, R> { + fn check(&mut self) -> io::Result<()> { + if let Some(error) = &self.failure { + return Err(io::Error::other(error.clone())); + } + (self.check)().map_err(|error| { + self.failure = Some(error.clone()); + io::Error::other(error) + }) + } + + fn record<T>(&mut self, result: io::Result<T>) -> io::Result<T> { + if let Err(error) = &result { + self.failure = Some(error.to_string()); + } + result + } +} + +impl<R: Read> Read for ObservedReader<'_, R> { + fn read(&mut self, data: &mut [u8]) -> io::Result<usize> { + self.check()?; + let result = self.file.read(data); + self.record(result) + } +} + +impl<R: Seek> Seek for ObservedReader<'_, R> { + fn seek(&mut self, offset: SeekFrom) -> io::Result<u64> { + self.check()?; + let result = self.file.seek(offset); + self.record(result) + } +} diff --git a/rust_backend/crates/core/src/tags/id3.rs b/rust_backend/crates/core/src/tags/id3.rs new file mode 100644 index 00000000..f0560701 --- /dev/null +++ b/rust_backend/crates/core/src/tags/id3.rs @@ -0,0 +1,675 @@ +use super::{AudioMetadata, CoverArt, bytes, exact, pair, seek, truthy, utf8}; +use crate::matching::{lowercase, uppercase}; +use std::io::{self, Cursor, Read, Seek, SeekFrom}; + +// ID3v2.2/2.3 global unsynchronization also covers frame headers. Decode +// incrementally so skipping artwork never requires a tag-sized allocation. +struct Input<'a, R> { + source: &'a mut R, + remaining: u64, + unsync: bool, + after_ff: bool, +} + +impl<R: Read> Read for Input<'_, R> { + fn read(&mut self, output: &mut [u8]) -> io::Result<usize> { + if !self.unsync { + let length = output.len().min(self.remaining as usize); + let count = self.source.read(&mut output[..length])?; + self.remaining -= count as u64; + return Ok(count); + } + let mut count = 0; + while count < output.len() && self.remaining > 0 { + let mut value = [0]; + if self.source.read(&mut value)? == 0 { + break; + } + self.remaining -= 1; + if self.after_ff && value[0] == 0 { + self.after_ff = false; + continue; + } + self.after_ff = value[0] == 0xff; + output[count] = value[0]; + count += 1; + } + Ok(count) + } +} + +impl<R: Read + Seek> Input<'_, R> { + fn skip(&mut self, length: u64) -> Result<(), String> { + if !self.unsync { + seek(self.source, SeekFrom::Current(length as i64))?; + self.remaining = self.remaining.saturating_sub(length); + return Ok(()); + } + let mut left = length; + let mut buffer = [0; 8192]; + while left > 0 { + let count = left.min(buffer.len() as u64) as usize; + exact(self, &mut buffer[..count])?; + left -= count as u64; + } + Ok(()) + } +} + +pub(super) fn read(reader: &mut (impl Read + Seek)) -> Result<AudioMetadata, String> { + let mut metadata = v2(reader).unwrap_or_default(); + if (metadata.title.is_empty() || metadata.artist.is_empty()) + && let Ok(old) = v1(reader) + { + for (target, value) in [ + (&mut metadata.title, old.title), + (&mut metadata.artist, old.artist), + (&mut metadata.album, old.album), + (&mut metadata.year, old.year), + (&mut metadata.genre, old.genre), + ] { + if target.is_empty() { + *target = value; + } + } + } + if metadata.title.is_empty() && metadata.artist.is_empty() { + Err("no ID3 tags found".into()) + } else { + Ok(metadata) + } +} + +fn v2(reader: &mut (impl Read + Seek)) -> Result<AudioMetadata, String> { + v2_with_cover(reader, false).map(|(metadata, _)| metadata) +} + +pub(super) fn cover(reader: &mut (impl Read + Seek)) -> Result<CoverArt, String> { + v2_with_cover(reader, true)? + .1 + .ok_or_else(|| "no cover art found".into()) +} + +fn v2_with_cover( + reader: &mut (impl Read + Seek), + include_cover: bool, +) -> Result<(AudioMetadata, Option<CoverArt>), String> { + let header = bytes(reader, 10)?; + if &header[..3] != b"ID3" { + return Err("no ID3v2 header".into()); + } + let version = header[3]; + let flags = header[5]; + if !(2..=4).contains(&version) || header[6..].iter().any(|value| *value >= 128) { + return Err("invalid ID3 version or tag size".into()); + } + let mut size = syncsafe(&header[6..]); + let end = seek(reader, SeekFrom::End(0))?; + if size > end.saturating_sub(10) { + return Err("unexpected EOF".into()); + } + seek(reader, SeekFrom::Start(10))?; + let mut input = Input { + source: reader, + remaining: size, + unsync: version < 4 && flags & 0x80 != 0, + after_ff: false, + }; + if flags & 0x40 != 0 { + if version == 2 { + return Err("compressed ID3v2.2 tag unsupported".into()); + } + let extended = bytes(&mut input, 4)?; + let length = if version == 4 { + syncsafe(&extended).checked_sub(4) + } else { + Some(u64::from(u32::from_be_bytes(extended.try_into().unwrap()))) + } + .ok_or("invalid ID3 extended header")?; + if size < 4 || length > size - 4 { + return Err("invalid ID3 extended header".into()); + } + input.skip(length)?; + size -= length + 4; + } + let mut metadata = AudioMetadata::default(); + let mut cover = None; + // Go retains fields decoded before a malformed later frame. + let result = frames( + &mut input, + size, + version, + version == 4 && flags & 0x80 != 0, + &mut metadata, + include_cover.then_some(&mut cover), + ); + if include_cover && cover.is_none() { + result?; + } + Ok((metadata, cover)) +} + +pub(super) fn embedded(data: &[u8]) -> Result<AudioMetadata, String> { + if data.len() < 10 || &data[..3] != b"ID3" { + return Err("no ID3v2 header".into()); + } + let version = data[3]; + let flags = data[5]; + let size = syncsafe(&data[6..10]) as usize; + let size = if size == 0 || size > data.len() - 10 { + data.len() - 10 + } else { + size + }; + let mut body = &data[10..10 + size]; + if flags & 0x10 != 0 && body.len() >= 10 && body[body.len() - 10..].starts_with(b"3DI") { + body = &body[..body.len() - 10]; + } + if flags & 0x40 != 0 && body.len() >= 4 { + let size = match version { + 3 => u32::from_be_bytes(body[..4].try_into().unwrap()) as u64, + 4 => syncsafe(&body[..4]), + _ => 0, + }; + let skip = if size + 4 <= body.len() as u64 { + size + 4 + } else { + size + }; + if size > 0 && skip < body.len() as u64 { + body = &body[skip as usize..]; + } + } + let size = body.len() as u64; + let mut cursor = Cursor::new(body); + let mut input = Input { + source: &mut cursor, + remaining: size, + unsync: false, + after_ff: false, + }; + let mut metadata = AudioMetadata::default(); + let _ = frames( + &mut input, + size, + version, + flags & 0x80 != 0, + &mut metadata, + None, + ); + Ok(metadata) +} + +fn frames<R: Read + Seek>( + input: &mut Input<'_, R>, + mut remaining: u64, + version: u8, + unsync: bool, + metadata: &mut AudioMetadata, + mut cover: Option<&mut Option<CoverArt>>, +) -> Result<(), String> { + let (header_size, id_size) = if version == 2 { (6, 3) } else { (10, 4) }; + while remaining >= header_size as u64 { + let header = bytes(input, header_size)?; + remaining -= header_size as u64; + if header[0] == 0 || header.starts_with(b"3DI") { + break; + } + let size = match version { + 2 => u64::from(u32::from_be_bytes([0, header[3], header[4], header[5]])), + 4 => { + if header[4..8].iter().any(|value| *value >= 128) { + return Err("invalid ID3 frame size".into()); + } + syncsafe(&header[4..8]) + } + _ => u64::from(u32::from_be_bytes(header[4..8].try_into().unwrap())), + }; + if size == 0 || size > remaining { + return Err("invalid ID3 frame bounds".into()); + } + remaining -= size; + let id = &header[..id_size]; + let flags = if version == 2 { 0 } else { header[9] }; + let picture = matches!(id, b"APIC" | b"PIC"); + let wanted = if picture { + cover.as_ref().is_some_and(|cover| cover.is_none()) + } else { + id[0] == b'T' || matches!(id, b"COMM" | b"USLT" | b"ULT") + }; + let unsupported = + (version == 3 && flags & 0xc0 != 0) || (version == 4 && flags & 0x0c != 0); + if !wanted || unsupported || size > 32 * 1024 * 1024 { + input.skip(size)?; + continue; + } + let mut data = bytes(input, size as usize)?; + let mut offset = + usize::from((version == 3 && flags & 0x20 != 0) || (version == 4 && flags & 0x40 != 0)); + if version == 4 && flags & 1 != 0 { + offset += 4; + } + if offset > data.len() { + continue; + } + if unsync || version == 4 && flags & 2 != 0 { + let mut after_ff = false; + let mut write = offset; + for read in offset..data.len() { + let value = data[read]; + if after_ff && value == 0 { + after_ff = false; + continue; + } + after_ff = value == 0xff; + data[write] = value; + write += 1; + } + data.truncate(write); + } + if picture { + if let Some(cover) = cover.as_deref_mut() { + *cover = picture_frame(&data[offset..], version); + } + } else { + apply(metadata, version, id, &data[offset..]); + } + } + Ok(()) +} + +fn picture_frame(data: &[u8], version: u8) -> Option<CoverArt> { + let encoding = *data.first()?; + let (mime, start) = if version == 2 { + let format = data.get(1..4)?; + ( + if format == b"PNG" { + "image/png" + } else { + "image/jpeg" + } + .into(), + 5, + ) + } else { + let end = data.get(1..)?.iter().position(|byte| *byte == 0)? + 1; + (utf8(&data[1..end]), end + 2) + }; + let description = data.get(start..)?; + let skip = if matches!(encoding, 0 | 3) { + description.iter().position(|byte| *byte == 0)? + 1 + } else { + description.windows(2).position(|bytes| bytes == [0, 0])? + 2 + }; + let image = description.get(skip..)?; + (!image.is_empty()).then(|| CoverArt { + data: image.to_vec(), + mime, + }) +} + +fn apply(metadata: &mut AudioMetadata, version: u8, id: &[u8], data: &[u8]) { + let text = data + .split_first() + .map(|(encoding, body)| decode(*encoding, body)) + .unwrap_or_default(); + let value = text.split('\0').next().unwrap_or_default(); + let id = if version == 2 { + match id { + b"TT2" => b"TIT2".as_slice(), + b"TP1" => b"TPE1", + b"TP2" => b"TPE2", + b"TAL" => b"TALB", + b"TYE" => { + metadata.year = value.into(); + return; + } + b"TCO" => b"TCON", + b"TRK" => b"TRCK", + b"TPA" => b"TPOS", + b"TCM" => b"TCOM", + b"TPB" => b"TPUB", + b"TCR" => b"TCOP", + b"ULT" => b"USLT", + b"TXX" => b"TXXX", + _ => return, + } + } else { + id + }; + match id { + b"TIT2" => metadata.title = value.into(), + b"TPE1" => metadata.artist = value.into(), + b"TPE2" => metadata.album_artist = value.into(), + b"TALB" => metadata.album = value.into(), + b"TYER" | b"TDRC" => { + metadata.year = value.into(); + if value.len() >= 4 { + metadata.date = value.into(); + } + } + b"TCON" => metadata.genre = genre(value), + b"TRCK" => (metadata.track_number, metadata.total_tracks) = pair(value), + b"TPOS" => (metadata.disc_number, metadata.total_discs) = pair(value), + b"TSRC" => metadata.isrc = value.into(), + b"TCOM" => metadata.composer = value.into(), + b"TPUB" => metadata.label = value.into(), + b"TCOP" => metadata.copyright = value.into(), + b"TCMP" if truthy(value) && metadata.album_type.is_empty() => { + metadata.album_type = "compilation".into() + } + b"COMM" | b"USLT" => { + let value = language(data); + if !value.is_empty() { + if id == b"COMM" { + metadata.comment = value; + } else if metadata.lyrics.is_empty() { + metadata.lyrics = value; + } + } + } + b"TXXX" => { + let (description, value) = user_text(data); + if matches!( + lowercase(description.trim()).as_str(), + "lyrics" + | "lyric" + | "unsyncedlyrics" + | "unsynced lyrics" + | "syncedlyrics" + | "synced lyrics" + | "uslt" + | "sylt" + | "lrc" + ) && metadata.lyrics.is_empty() + && !value.is_empty() + { + metadata.lyrics = value.clone(); + } + match uppercase(&description).as_str() { + "ITUNESADVISORY" => metadata.explicit = truthy(&value), + "RELEASETYPE" => metadata.album_type = value, + "BARCODE" | "UPC" => metadata.upc = value, + "REPLAYGAIN_TRACK_GAIN" if version != 2 => metadata.replay_gain_track_gain = value, + "REPLAYGAIN_TRACK_PEAK" if version != 2 => metadata.replay_gain_track_peak = value, + "REPLAYGAIN_ALBUM_GAIN" if version != 2 => metadata.replay_gain_album_gain = value, + "REPLAYGAIN_ALBUM_PEAK" if version != 2 => metadata.replay_gain_album_peak = value, + _ => {} + } + } + _ => {} + } +} + +pub(crate) fn decode(encoding: u8, mut data: &[u8]) -> String { + if !matches!(encoding, 1 | 2) { + return utf8(data).trim_end_matches('\0').into(); + } + let mut little = false; + if encoding == 1 { + if data.starts_with(&[0xff, 0xfe]) { + little = true; + data = &data[2..]; + } else if data.starts_with(&[0xfe, 0xff]) { + data = &data[2..]; + } + } + // Preserve the legacy decoder's individual UTF-16 code-unit conversion. + data.as_chunks::<2>() + .0 + .iter() + .map(|pair| { + if little { + u16::from_le_bytes(*pair) + } else { + u16::from_be_bytes(*pair) + } + }) + .take_while(|value| *value != 0) + .map(|value| char::from_u32(u32::from(value)).unwrap_or(char::REPLACEMENT_CHARACTER)) + .collect() +} + +fn separator(data: &[u8], encoding: u8) -> Option<(usize, usize)> { + if matches!(encoding, 1 | 2) { + data.as_chunks::<2>() + .0 + .iter() + .position(|pair| *pair == [0, 0]) + .map(|index| (index * 2, 2)) + } else { + data.iter() + .position(|value| *value == 0) + .map(|index| (index, 1)) + } +} + +fn language(data: &[u8]) -> String { + if data.len() < 5 { + return String::new(); + } + let encoding = data[0]; + let rest = &data[4..]; + let text = match separator(rest, encoding) { + Some((index, size)) if matches!(encoding, 1 | 2) || index + size < rest.len() => { + &rest[index + size..] + } + _ if matches!(encoding, 1 | 2) => &[], + _ => rest, + }; + decode(encoding, text) +} + +pub(crate) fn user_text(data: &[u8]) -> (String, String) { + let Some((&encoding, body)) = data.split_first() else { + return Default::default(); + }; + let Some((index, size)) = separator(body, encoding) else { + return Default::default(); + }; + if index + size >= body.len() { + return Default::default(); + } + ( + decode(encoding, &body[..index]).trim().into(), + decode(encoding, &body[index + size..]).trim().into(), + ) +} + +fn v1(reader: &mut (impl Read + Seek)) -> Result<AudioMetadata, String> { + let end = seek(reader, SeekFrom::End(0))?; + if end < 128 { + return Err("no ID3v1 tag found".into()); + } + seek(reader, SeekFrom::Start(end - 128))?; + let data = bytes(reader, 128)?; + if !data.starts_with(b"TAG") { + return Err("no ID3v1 tag".into()); + } + let text = |start, end| { + utf8(&data[start..end]) + .trim_end_matches([' ', '\0']) + .to_owned() + }; + Ok(AudioMetadata { + title: text(3, 33), + artist: text(33, 63), + album: text(63, 93), + year: text(93, 97), + genre: GENRES.get(data[127] as usize).unwrap_or(&"").to_string(), + ..AudioMetadata::default() + }) +} + +fn syncsafe(data: &[u8]) -> u64 { + u64::from(data[0]) << 21 + | u64::from(data[1]) << 14 + | u64::from(data[2]) << 7 + | u64::from(data[3]) +} + +pub(super) fn genre(value: &str) -> String { + if value.starts_with('(') + && let Some(end) = value.find(')') + && let Ok(index) = value[1..end].parse::<usize>() + && let Some(genre) = GENRES.get(index) + { + return if end + 1 < value.len() { + value[end + 1..].into() + } else { + (*genre).into() + }; + } + value.into() +} + +const GENRES: &[&str] = &[ + "Blues", + "Classic Rock", + "Country", + "Dance", + "Disco", + "Funk", + "Grunge", + "Hip-Hop", + "Jazz", + "Metal", + "New Age", + "Oldies", + "Other", + "Pop", + "R&B", + "Rap", + "Reggae", + "Rock", + "Techno", + "Industrial", + "Alternative", + "Ska", + "Death Metal", + "Pranks", + "Soundtrack", + "Euro-Techno", + "Ambient", + "Trip-Hop", + "Vocal", + "Jazz+Funk", + "Fusion", + "Trance", + "Classical", + "Instrumental", + "Acid", + "House", + "Game", + "Sound Clip", + "Gospel", + "Noise", + "AlternRock", + "Bass", + "Soul", + "Punk", + "Space", + "Meditative", + "Instrumental Pop", + "Instrumental Rock", + "Ethnic", + "Gothic", + "Darkwave", + "Techno-Industrial", + "Electronic", + "Pop-Folk", + "Eurodance", + "Dream", + "Southern Rock", + "Comedy", + "Cult", + "Gangsta", + "Top 40", + "Christian Rap", + "Pop/Funk", + "Jungle", + "Native American", + "Cabaret", + "New Wave", + "Psychedelic", + "Rave", + "Showtunes", + "Trailer", + "Lo-Fi", + "Tribal", + "Acid Punk", + "Acid Jazz", + "Polka", + "Retro", + "Musical", + "Rock & Roll", + "Hard Rock", + "Folk", + "Folk-Rock", + "National Folk", + "Swing", + "Fast Fusion", + "Bebop", + "Latin", + "Revival", + "Celtic", + "Bluegrass", + "Avantgarde", + "Gothic Rock", + "Progressive Rock", + "Psychedelic Rock", + "Symphonic Rock", + "Slow Rock", + "Big Band", + "Chorus", + "Easy Listening", + "Acoustic", + "Humour", + "Speech", + "Chanson", + "Opera", + "Chamber Music", + "Sonata", + "Symphony", + "Booty Bass", + "Primus", + "Porn Groove", + "Satire", + "Slow Jam", + "Club", + "Tango", + "Samba", + "Folklore", + "Ballad", + "Power Ballad", + "Rhythmic Soul", + "Freestyle", + "Duet", + "Punk Rock", + "Drum Solo", + "A capella", + "Euro-House", + "Dance Hall", + "Goa", + "Drum & Bass", + "Club-House", + "Hardcore", + "Terror", + "Indie", + "BritPop", + "Negerpunk", + "Polsk Punk", + "Beat", + "Christian Gangsta Rap", + "Heavy Metal", + "Black Metal", + "Crossover", + "Contemporary Christian", + "Christian Rock", + "Merengue", + "Salsa", + "Thrash Metal", + "Anime", + "J-Pop", + "Synthpop", +]; diff --git a/rust_backend/crates/core/src/tags/library.rs b/rust_backend/crates/core/src/tags/library.rs new file mode 100644 index 00000000..22f8ea4a --- /dev/null +++ b/rust_backend/crates/core/src/tags/library.rs @@ -0,0 +1,274 @@ +//! Library scan DTOs composed from the existing tag and quality readers. + +use super::{AudioMetadata, CoverArt, file::ObservedReader, read_tags}; +use crate::lyrics::lrc::has_usable_content; +use crate::media::{mp3_quality, ogg_quality, probe_mp4_quality, probe_quality, riff_quality}; +use serde_json::{Value, json}; +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; + +pub fn library_extension(path: &str, hint: &str) -> String { + let suffix = |path: &str| { + let name = path.rsplit('/').next().unwrap_or_default(); + name.rfind('.') + .map(|offset| name[offset + 1..].to_lowercase()) + }; + suffix(path).or_else(|| suffix(hint)).unwrap_or_default() +} + +pub fn library_id(path: &str) -> String { + let hash = path.chars().fold(5381_u32, |hash, value| { + hash.wrapping_mul(33).wrapping_add(value as u32) + }); + format!("lib_{hash:x}") +} + +fn filename(path: &str) -> String { + let name = Path::new(path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(path); + name[..name.rfind('.').unwrap_or(name.len())].into() +} + +/// Missing/unsupported audio still produces a filename-derived track in Go. +/// WAV/AIFF use defaults even when their tag/quality readers fail. +pub fn library_metadata(path: &str, hint: &str, scan_time: &str, mod_time: i64) -> Value { + let mut format = library_extension(path, hint); + let riff = matches!(format.as_str(), "wav" | "aiff" | "aif" | "aifc"); + if matches!(format.as_str(), "aiff" | "aif" | "aifc") { + format = "aiff".into(); + } + let name = filename(if hint.is_empty() { path } else { hint }); + let mut result = json!({ + "id": library_id(path), "filePath":path, "scannedAt":scan_time, + "trackName":name, "artistName":"Unknown Artist", "albumName":"Unknown Album", + "hasLyrics":false, + }); + if !format.is_empty() { + result["format"] = format.into(); + } + if mod_time != 0 { + result["fileModTime"] = mod_time.into(); + } + if !riff { + result["metadataFromFilename"] = true.into(); + let numeric = |s: &str| !s.is_empty() && s.bytes().all(|c| c.is_ascii_digit()); + if let Some((first, title)) = name.split_once(" - ") { + result["trackName"] = title.into(); + if first.len() > 3 || !numeric(first) { + result["artistName"] = first.into(); + } + } else if name.len() > 3 && name.as_bytes()[..2].iter().all(u8::is_ascii_digit) { + result["trackName"] = name[2..].trim_start_matches([' ', '.', '-']).into(); + } + let parent = Path::new(path).parent().unwrap_or(Path::new(".")); + let album = parent + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_else(|| if parent == Path::new("/") { "/" } else { "." }); + if !matches!(album, "" | "." | "fd" | "self") { + result["albumName"] = album.into(); + } + } + result +} + +pub fn read_library_metadata( + file: &mut (impl Read + Seek), + path: &str, + hint: &str, + scan_time: &str, + mod_time: i64, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Value, String> { + read_observed(file, path, hint, scan_time, mod_time, check, None) +} + +/// FLAC scans collect artwork during the existing tag pass. Other formats keep +/// their existing readers; callers request this only on a FLAC cover-cache miss. +pub fn read_library_metadata_with_cover( + file: &mut (impl Read + Seek), + path: &str, + hint: &str, + scan_time: &str, + mod_time: i64, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(Value, Option<CoverArt>), String> { + let mut cover = None; + let metadata = read_observed( + file, + path, + hint, + scan_time, + mod_time, + check, + Some(&mut cover), + )?; + Ok((metadata, cover)) +} + +fn read_observed( + file: &mut (impl Read + Seek), + path: &str, + hint: &str, + scan_time: &str, + mod_time: i64, + check: &dyn Fn() -> Result<(), String>, + cover: Option<&mut Option<CoverArt>>, +) -> Result<Value, String> { + check()?; + let mut reader = ObservedReader { + file, + check, + failure: None, + }; + let result = read(&mut reader, path, hint, scan_time, mod_time, cover); + if let Some(error) = reader.failure { + return Err(error); + } + check()?; + result +} + +fn read( + file: &mut (impl Read + Seek), + path: &str, + hint: &str, + scan_time: &str, + mod_time: i64, + mut cover: Option<&mut Option<CoverArt>>, +) -> Result<Value, String> { + let mut result = library_metadata(path, hint, scan_time, mod_time); + let format = library_extension(path, hint); + let metadata = read_tags(file, &format, &|| Ok(()), cover.as_deref_mut()); + let tagged = metadata.is_ok(); + if !tagged && let Some(cover) = cover { + *cover = None; + } + if let Ok(metadata) = metadata { + apply_tags(&mut result, metadata, &format, path, hint)?; + } + // MP4 and RIFF probe quality independently of whether tags are present. + if !tagged + && !matches!( + format.as_str(), + "m4a" | "mp4" | "aac" | "wav" | "aiff" | "aif" | "aifc" + ) + { + return Ok(result); + } + let size = file + .seek(SeekFrom::End(0)) + .map_err(|error| error.to_string())?; + file.seek(SeekFrom::Start(0)) + .map_err(|error| error.to_string())?; + let quality = match format.as_str() { + "flac" => probe_quality(file, &|| Ok(())), + "m4a" | "mp4" | "aac" => probe_mp4_quality(file, &|| Ok(())), + "mp3" => mp3_quality(file, size as i64), + "ogg" | "opus" => ogg_quality(file, size as i64, path), + "wav" | "aiff" | "aif" | "aifc" => riff_quality(file, size as i64, format != "wav"), + _ => return Ok(result), + }; + if let Ok(quality) = quality { + for (key, value) in [ + ("bitDepth", quality.bit_depth), + ("sampleRate", quality.sample_rate), + ("duration", quality.duration), + ] { + if value != 0 { + result[key] = value.into(); + } + } + let bitrate = match format.as_str() { + "flac" if quality.total_samples > 0 && quality.sample_rate > 0 => { + (size as f64 * 8.0 + / (quality.total_samples as f64 / quality.sample_rate as f64) + / 1000.0) as i64 + } + "mp3" | "ogg" | "opus" => quality.bitrate / 1000, + "m4a" | "mp4" | "aac" => { + let mapped = match quality.codec.trim().to_lowercase().as_str() { + "flac" => "flac", + "alac" => "alac", + "eac3" | "ec-3" => "eac3", + "ac3" | "ac-3" => "ac3", + "ac4" | "ac-4" => "ac4", + "opus" => "opus", + "aac" | "mp4a" => "m4a", + _ => "", + }; + if !mapped.is_empty() { + result["format"] = mapped.into(); + } + quality.bitrate + } + _ => 0, + }; + if bitrate > 0 { + result["bitrate"] = bitrate.into(); + } + } + Ok(result) +} + +fn apply_tags( + result: &mut Value, + mut tags: AudioMetadata, + format: &str, + path: &str, + hint: &str, +) -> Result<(), String> { + if format != "flac" && tags.date.is_empty() { + tags.date = tags.year.clone(); + } + result + .as_object_mut() + .unwrap() + .remove("metadataFromFilename"); + result["trackName"] = if tags.title.is_empty() { + filename(if hint.is_empty() { path } else { hint }).into() + } else { + tags.title.clone().into() + }; + result["artistName"] = if tags.artist.is_empty() { + "Unknown Artist".into() + } else { + tags.artist.clone().into() + }; + result["albumName"] = if tags.album.is_empty() { + "Unknown Album".into() + } else { + tags.album.clone().into() + }; + result["hasLyrics"] = has_usable_content(&tags.lyrics).into(); + let tags = serde_json::to_value(tags).map_err(|error| error.to_string())?; + for (source, target) in [ + ("album_artist", "albumArtist"), + ("isrc", "isrc"), + ("track_number", "trackNumber"), + ("total_tracks", "totalTracks"), + ("disc_number", "discNumber"), + ("total_discs", "totalDiscs"), + ("date", "releaseDate"), + ("genre", "genre"), + ("composer", "composer"), + ("label", "label"), + ("copyright", "copyright"), + ("comment", "comment"), + ("album_type", "albumType"), + ("explicit", "explicit"), + ("upc", "upc"), + ] { + let value = &tags[source]; + if !matches!(value, Value::Null) + && value != "" + && value != &json!(0) + && value != &json!(false) + { + result[target] = value.clone(); + } + } + Ok(()) +} diff --git a/rust_backend/crates/core/src/tags/mp4.rs b/rust_backend/crates/core/src/tags/mp4.rs new file mode 100644 index 00000000..0a3e343a --- /dev/null +++ b/rust_backend/crates/core/src/tags/mp4.rs @@ -0,0 +1,252 @@ +use super::{AudioMetadata, CoverArt, bytes, seek, truthy, utf8}; +use crate::matching::uppercase; +use std::io::{Read, Seek, SeekFrom}; + +#[derive(Clone, Copy)] +struct Atom { + payload: u64, + end: u64, + kind: [u8; 4], +} + +struct Reader<'a, R> { + file: &'a mut R, + size: u64, +} + +pub(super) fn read(file: &mut (impl Read + Seek)) -> Result<AudioMetadata, String> { + let size = seek(file, SeekFrom::End(0))?; + let mut reader = Reader { file, size }; + let ilst = reader.ilst()?; + let mut metadata = AudioMetadata::default(); + let mut position = ilst.payload; + while position.saturating_add(8) <= ilst.end { + let atom = reader.atom(position, ilst.end)?; + match &atom.kind { + b"\xa9nam" => metadata.title = reader.text(atom), + b"\xa9ART" => metadata.artist = reader.text(atom), + b"\xa9alb" => metadata.album = reader.text(atom), + b"aART" => metadata.album_artist = reader.text(atom), + b"\xa9day" => { + metadata.date = reader.text(atom); + metadata.year = metadata.date.clone(); + } + b"\xa9gen" => metadata.genre = reader.text(atom), + b"\xa9wrt" => metadata.composer = reader.text(atom), + b"\xa9cmt" => metadata.comment = reader.text(atom), + b"cprt" => metadata.copyright = reader.text(atom), + b"\xa9lyr" => metadata.lyrics = reader.text(atom), + b"trkn" => (metadata.track_number, metadata.total_tracks) = reader.pair(atom), + b"disk" => (metadata.disc_number, metadata.total_discs) = reader.pair(atom), + b"rtng" => { + if let Ok(data) = reader.data(atom) + && let Some(value) = data.last() + { + metadata.explicit = *value == 1; + } + } + b"cpil" => { + if let Ok(data) = reader.data(atom) + && data.last().is_some_and(|value| *value != 0) + && metadata.album_type.is_empty() + { + metadata.album_type = "compilation".into(); + } + } + b"----" => { + if let Ok((name, value)) = reader.freeform(atom) { + apply(&mut metadata, &name, value); + } + } + _ => {} + } + position = atom.end; + } + if metadata.title.is_empty() + && metadata.artist.is_empty() + && metadata.album.is_empty() + && metadata.album_artist.is_empty() + && metadata.lyrics.is_empty() + && metadata.track_number == 0 + && metadata.disc_number == 0 + { + Err("no M4A tags found".into()) + } else { + Ok(metadata) + } +} + +pub(super) fn cover(file: &mut (impl Read + Seek)) -> Result<CoverArt, String> { + let size = seek(file, SeekFrom::End(0))?; + let mut reader = Reader { file, size }; + let ilst = reader.ilst()?; + let cover = reader + .find(ilst.payload, ilst.end, b"covr") + .ok() + .flatten() + .ok_or("cover atom not found")?; + let data = reader + .find(cover.payload, cover.end, b"data") + .ok() + .flatten() + .ok_or("data atom not found in cover")?; + if data.end <= data.payload + 8 { + return Err("empty cover data".into()); + } + let data = reader.payload(data, 8)?; + let mime = if data.len() >= 8 && data.starts_with(b"\x89PNG") { + "image/png" + } else { + "image/jpeg" + }; + Ok(CoverArt { + data, + mime: mime.into(), + }) +} + +impl<R: Read + Seek> Reader<'_, R> { + fn ilst(&mut self) -> Result<Atom, String> { + let moov = self + .find(0, self.size, b"moov") + .ok() + .flatten() + .ok_or("moov not found")?; + let mut ilst = None; + if let Ok(Some(udta)) = self.find(moov.payload, moov.end, b"udta") { + ilst = self.metadata(udta); + } + ilst.or_else(|| self.metadata(moov)) + .ok_or_else(|| "ilst not found (tried moov>udta>meta>ilst and moov>meta>ilst)".into()) + } + fn atom(&mut self, offset: u64, end: u64) -> Result<Atom, String> { + if offset.saturating_add(8) > self.size { + return Err("unexpected EOF".into()); + } + seek(self.file, SeekFrom::Start(offset))?; + let header = bytes(self.file, 8)?; + let kind: [u8; 4] = header[4..].try_into().unwrap(); + let mut length = u64::from(u32::from_be_bytes(header[..4].try_into().unwrap())); + let mut header_size = 8; + if length == 1 { + length = u64::from_be_bytes(bytes(self.file, 8)?.try_into().unwrap()); + header_size = 16; + } else if length == 0 { + length = end.saturating_sub(offset); + } + if length < header_size || length > i64::MAX as u64 || offset.checked_add(length).is_none() + { + return Err(format!("invalid atom size for {}", utf8(&kind))); + } + Ok(Atom { + payload: offset + header_size, + end: offset + length, + kind, + }) + } + + fn find(&mut self, mut offset: u64, end: u64, kind: &[u8; 4]) -> Result<Option<Atom>, String> { + while offset.saturating_add(8) <= end { + let atom = self.atom(offset, end)?; + if &atom.kind == kind { + return Ok(Some(atom)); + } + offset = atom.end; + } + Ok(None) + } + + fn metadata(&mut self, parent: Atom) -> Option<Atom> { + let meta = self.find(parent.payload, parent.end, b"meta").ok()??; + self.find(meta.payload + 4, meta.end, b"ilst") + .ok() + .flatten() + .or_else(|| self.find(meta.payload, meta.end, b"ilst").ok().flatten()) + } + + fn payload(&mut self, atom: Atom, prefix: u64) -> Result<Vec<u8>, String> { + let start = atom.payload + prefix; + let length = atom.end.checked_sub(start).ok_or("invalid atom payload")?; + // Bound selected metadata and artwork, including malformed 64-bit + // declarations on 32-bit Android. Unknown atoms are skipped. + if length == 0 || length > 32 * 1024 * 1024 || atom.end > self.size { + return Err("invalid atom payload".into()); + } + seek(self.file, SeekFrom::Start(start))?; + bytes(self.file, length as usize) + } + + fn data(&mut self, parent: Atom) -> Result<Vec<u8>, String> { + let atom = self + .find(parent.payload, parent.end, b"data")? + .ok_or("data atom not found")?; + self.payload(atom, 8) + } + + fn text(&mut self, atom: Atom) -> String { + self.data(atom).map(|data| text(&data)).unwrap_or_default() + } + + fn pair(&mut self, atom: Atom) -> (i64, i64) { + if let Ok(data) = self.data(atom) + && data.len() >= 6 + { + ( + i64::from(u16::from_be_bytes(data[2..4].try_into().unwrap())), + i64::from(u16::from_be_bytes(data[4..6].try_into().unwrap())), + ) + } else { + (0, 0) + } + } + + fn freeform(&mut self, parent: Atom) -> Result<(String, String), String> { + let mut offset = parent.payload; + let mut name = String::new(); + let mut value = String::new(); + while offset.saturating_add(8) <= parent.end { + let atom = self.atom(offset, parent.end)?; + if atom.kind == *b"name" + && let Ok(data) = self.payload(atom, 4) + { + name = text(&data); + } + if atom.kind == *b"data" + && let Ok(data) = self.payload(atom, 8) + { + value = text(&data); + } + offset = atom.end; + } + if name.is_empty() || value.is_empty() { + Err("freeform M4A tag incomplete".into()) + } else { + Ok((name, value)) + } + } +} + +fn text(data: &[u8]) -> String { + utf8(data).trim_end_matches('\0').trim().into() +} + +fn apply(metadata: &mut AudioMetadata, name: &str, value: String) { + match uppercase(name.trim()).as_str() { + "ISRC" => metadata.isrc = value, + "LABEL" | "ORGANIZATION" => metadata.label = value, + "COMMENT" if metadata.comment.is_empty() => metadata.comment = value, + "COMPOSER" if metadata.composer.is_empty() => metadata.composer = value, + "COPYRIGHT" if metadata.copyright.is_empty() => metadata.copyright = value, + "LYRICS" | "UNSYNCEDLYRICS" | "SYNCEDLYRICS" if metadata.lyrics.is_empty() => { + metadata.lyrics = value + } + "REPLAYGAIN_TRACK_GAIN" => metadata.replay_gain_track_gain = value, + "REPLAYGAIN_TRACK_PEAK" => metadata.replay_gain_track_peak = value, + "REPLAYGAIN_ALBUM_GAIN" => metadata.replay_gain_album_gain = value, + "REPLAYGAIN_ALBUM_PEAK" => metadata.replay_gain_album_peak = value, + "ITUNESADVISORY" => metadata.explicit = truthy(&value), + "RELEASETYPE" => metadata.album_type = value, + "BARCODE" | "UPC" => metadata.upc = value, + _ => {} + } +} diff --git a/rust_backend/crates/core/src/tags/write.rs b/rust_backend/crates/core/src/tags/write.rs new file mode 100644 index 00000000..50c7943d --- /dev/null +++ b/rust_backend/crates/core/src/tags/write.rs @@ -0,0 +1,715 @@ +//! Streaming metadata rewrites. The native owner supplies a separate staged +//! output and publishes it only after this operation succeeds and is synced. + +mod ape; +mod id3; +mod mp4; +mod ogg; +mod riff; + +pub(super) use id3::cover as embedded_cover; + +use super::{CheckedReader, bytes, exact, pair, seek}; +use crate::matching::{lowercase, uppercase}; +use regex::Regex; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{BufReader, Cursor, Read, Seek, SeekFrom, Write}; +use std::sync::LazyLock; + +type Fields = BTreeMap<String, String>; +const MAX_TAG_BYTES: usize = 64 * 1024 * 1024; + +struct Section { + start: u64, + end: u64, + data: Vec<u8>, +} + +pub fn rewrite_audio_tags( + source: &mut (impl Read + Seek), + output: &mut impl Write, + format: &str, + fields: &Fields, + cover: Option<&[u8]>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(), String> { + check()?; + validate_fields(fields)?; + let mut source = BufReader::new(CheckedReader { + reader: source, + check, + }); + seek(&mut source, SeekFrom::Start(0))?; + if matches!(format, "wav" | "aiff" | "aif" | "aifc") { + let sections = riff::edit(&mut source, format != "wav", fields, cover, check)?; + return write_sections(&mut source, output, sections, check); + } + let cover = cover.filter(|data| !data.is_empty()); + if matches!(format, "ogg" | "opus") { + return ogg::rewrite(&mut source, output, fields, cover, check); + } + if matches!(format, "ape" | "wv" | "mpc") { + let section = ape::edit(&mut source, fields, cover, check)?; + return write_sections(&mut source, output, vec![section], check); + } + if matches!(format, "m4a" | "mp4" | "m4b") { + let sections = mp4::edit(&mut source, fields, cover, check)?; + return write_sections(&mut source, output, sections, check); + } + let (head, audio_start) = match format { + "flac" => flac_header(&mut source, fields, cover, None, check)?, + "mp3" => id3::header(&mut source, fields, cover, check)?, + _ => return Err(format!("unsupported tag writer: {format}")), + }; + write_sections( + &mut source, + output, + vec![Section { + start: 0, + end: audio_start, + data: head, + }], + check, + ) +} + +/// Embed canonical Vorbis keys without clearing empty values or removing +/// legacy aliases. This is the enrichment contract, distinct from the editor. +pub fn embed_flac_metadata( + source: &mut (impl Read + Seek), + output: &mut impl Write, + fields: &Fields, + artist_tag_mode: &str, + cover: Option<&[u8]>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(), String> { + check()?; + validate_fields(fields)?; + let mut source = BufReader::new(CheckedReader { + reader: source, + check, + }); + seek(&mut source, SeekFrom::Start(0))?; + let (data, end) = flac_header( + &mut source, + fields, + cover.filter(|data| !data.is_empty()), + Some(artist_tag_mode), + check, + )?; + write_sections( + &mut source, + output, + vec![Section { + start: 0, + end, + data, + }], + check, + ) +} + +fn validate_fields(fields: &Fields) -> Result<(), String> { + if fields + .iter() + .try_fold(0usize, |total, (key, value)| { + total.checked_add(key.len())?.checked_add(value.len()) + }) + .is_none_or(|length| length > MAX_TAG_BYTES) + { + return Err("tag edit fields exceed 64 MiB".into()); + } + Ok(()) +} + +/// Rewrite only existing M4A freeform tags. False means a no-op and no output +/// was written, so the native owner can discard its staging file immediately. +pub fn rewrite_m4a_freeform( + source: &mut (impl Read + Seek), + output: &mut impl Write, + fields: &Fields, + replay_gain_only: bool, + check: &dyn Fn() -> Result<(), String>, +) -> Result<bool, String> { + check()?; + validate_fields(fields)?; + let mut source = BufReader::new(CheckedReader { + reader: source, + check, + }); + let sections = mp4::edit_freeform(&mut source, fields, replay_gain_only, check)?; + write_optional_sections(&mut source, output, sections, check) +} + +/// Normalize an AC-4 container and lazily open its original configuration source. +pub fn rewrite_ac4_config<R: Read + Seek>( + source: &mut (impl Read + Seek), + output: &mut impl Write, + reference: impl FnOnce() -> Result<R, String>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<bool, String> { + check()?; + let mut source = BufReader::new(CheckedReader { + reader: source, + check, + }); + let sections = mp4::ac4::config(&mut source, reference, check)?; + write_optional_sections(&mut source, output, sections, check) +} + +/// Replace AC-4 metadata; false means no output, and the cover is read only for AC-4. +pub fn rewrite_ac4_metadata( + source: &mut (impl Read + Seek), + output: &mut impl Write, + metadata_json: &str, + cover: impl FnOnce() -> Result<Option<Vec<u8>>, String>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<bool, String> { + check()?; + let mut source = BufReader::new(CheckedReader { + reader: source, + check, + }); + let sections = mp4::ac4::metadata(&mut source, metadata_json, cover, check)?; + write_optional_sections(&mut source, output, sections, check) +} + +fn write_optional_sections( + source: &mut (impl Read + Seek), + output: &mut impl Write, + sections: Vec<Section>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<bool, String> { + if sections.is_empty() { + check()?; + return Ok(false); + } + write_sections(source, output, sections, check)?; + Ok(true) +} + +fn write_sections( + source: &mut (impl Read + Seek), + output: &mut impl Write, + mut sections: Vec<Section>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(), String> { + sections.sort_by_key(|section| section.start); + let mut position = 0; + for section in §ions { + if section.start < position || section.end < section.start { + return Err("overlapping or invalid file sections".into()); + } + position = section.end; + } + seek(source, SeekFrom::Start(0))?; + position = 0; + for section in sections { + copy(source, output, Some(section.start - position), check)?; + for chunk in section.data.chunks(65536) { + check()?; + output.write_all(chunk).map_err(|error| error.to_string())?; + } + seek(source, SeekFrom::Start(section.end))?; + position = section.end; + } + copy(source, output, None, check) +} + +fn copy( + source: &mut impl Read, + output: &mut impl Write, + mut remaining: Option<u64>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(), String> { + let mut buffer = [0; 65536]; + loop { + check()?; + let limit = remaining.map_or(buffer.len(), |n| n.min(buffer.len() as u64) as usize); + if limit == 0 { + break; + } + let count = source + .read(&mut buffer[..limit]) + .map_err(|error| error.to_string())?; + if count == 0 { + if remaining.is_some() { + return Err("unexpected EOF while copying audio".into()); + } + break; + } + output + .write_all(&buffer[..count]) + .map_err(|error| error.to_string())?; + remaining = remaining.map(|n| n - count as u64); + } + check() +} + +fn flac_header( + source: &mut (impl Read + Seek), + fields: &Fields, + cover: Option<&[u8]>, + embed_mode: Option<&str>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(Vec<u8>, u64), String> { + if bytes(source, 4)? != b"fLaC" { + return Err("fLaC head incorrect".into()); + } + let mut blocks = Vec::new(); + let mut total = 0; + loop { + check()?; + if blocks.len() >= 65536 { + return Err("FLAC metadata block count exceeds 65536".into()); + } + let h = bytes(source, 4)?; + let length = u32::from_be_bytes([0, h[1], h[2], h[3]]) as usize; + total += length + 4; + if total > MAX_TAG_BYTES { + return Err("FLAC metadata exceeds 64 MiB".into()); + } + blocks.push((h[0] & 0x7f, bytes(source, length)?)); + if h[0] & 0x80 != 0 { + break; + } + } + let audio_start = seek(source, SeekFrom::Current(0))?; + let sync = bytes(source, 2)?; + if sync[0] != 0xff || sync[1] >> 2 != 0x3e { + return Err("frames do not begin with sync code".into()); + } + let index = blocks.iter().position(|(kind, _)| *kind == 4); + let (vendor, mut comments) = if let Some(index) = index { + parse_comments(&blocks[index].1)? + } else { + (b"SpotiFLAC".to_vec(), Vec::new()) + }; + if let Some(mode) = embed_mode { + for (key, value) in fields.iter().filter(|(_, value)| !value.is_empty()) { + if matches!(key.as_str(), "ARTIST" | "ALBUMARTIST") { + let values = if mode.trim().eq_ignore_ascii_case("split_vorbis") { + split_artists(value) + } else { + vec![value.clone()] + }; + if !values.is_empty() { + set_comment(&mut comments, key, ""); + for value in values.iter().filter(|value| !value.trim().is_empty()) { + comments.push(format!("{key}={value}").into_bytes()); + } + } + } else { + set_comment(&mut comments, key, value); + } + } + } else { + edit_comments(&mut comments, fields); + } + let mut data = Vec::new(); + put_string(&mut data, &vendor); + data.extend((comments.len() as u32).to_le_bytes()); + for comment in comments { + put_string(&mut data, &comment); + } + if let Some(index) = index { + blocks[index].1 = data; + } else { + blocks.push((4, data)); + } + if let Some(cover) = cover { + blocks.retain(|(kind, _)| *kind != 6); + if let Some(picture) = picture(cover, check)? { + blocks.push((6, picture)); + } + } + let mut head = b"fLaC".to_vec(); + let count = blocks.len(); + for (index, (kind, data)) in blocks.into_iter().enumerate() { + if head.len() + data.len() + 4 > MAX_TAG_BYTES { + return Err("FLAC metadata exceeds 64 MiB".into()); + } + if data.len() > 0xff_ffff { + return Err("FLAC metadata block exceeds 24-bit size".into()); + } + head.push(kind | if index + 1 == count { 0x80 } else { 0 }); + head.extend(&(data.len() as u32).to_be_bytes()[1..]); + head.extend(data); + } + Ok((head, audio_start)) +} + +fn parse_comments(data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), String> { + fn string(reader: &mut Cursor<&[u8]>) -> Result<Vec<u8>, String> { + let mut length = [0; 4]; + exact(reader, &mut length)?; + let length = u32::from_le_bytes(length) as usize; + if length + > reader + .get_ref() + .len() + .saturating_sub(reader.position() as usize) + { + return Err("truncated Vorbis comment".into()); + } + bytes(reader, length) + } + let mut reader = Cursor::new(data); + let vendor = string(&mut reader)?; + let mut count = [0; 4]; + exact(&mut reader, &mut count)?; + let mut comments = Vec::new(); + for _ in 0..u32::from_le_bytes(count) { + comments.push(string(&mut reader)?); + } + Ok((vendor, comments)) +} + +fn put_string(output: &mut Vec<u8>, data: &[u8]) { + output.extend((data.len() as u32).to_le_bytes()); + output.extend(data); +} + +fn comment_key(value: &[u8]) -> String { + value + .iter() + .position(|value| *value == b'=') + .filter(|index| *index > 0) + .map(|index| uppercase(&String::from_utf8_lossy(&value[..index]))) + .unwrap_or_default() +} + +fn set_comment(comments: &mut Vec<Vec<u8>>, key: &str, value: &str) { + comments.retain(|comment| comment_key(comment) != key); + if !value.is_empty() { + comments.push(format!("{key}={value}").into_bytes()); + } +} + +fn get_comment(comments: &[Vec<u8>], key: &str) -> String { + comments + .iter() + .find(|comment| comment_key(comment) == key) + .map(|comment| String::from_utf8_lossy(&comment[key.len() + 1..]).into_owned()) + .unwrap_or_default() +} + +fn edit_comments(comments: &mut Vec<Vec<u8>>, fields: &Fields) { + for (field, key) in [ + ("title", "TITLE"), + ("album", "ALBUM"), + ("date", "DATE"), + ("isrc", "ISRC"), + ("genre", "GENRE"), + ("label", "ORGANIZATION"), + ("copyright", "COPYRIGHT"), + ("composer", "COMPOSER"), + ("comment", "COMMENT"), + ("explicit", "ITUNESADVISORY"), + ("album_type", "RELEASETYPE"), + ("upc", "BARCODE"), + ("barcode", "BARCODE"), + ("compilation", "COMPILATION"), + ("replaygain_track_gain", "REPLAYGAIN_TRACK_GAIN"), + ("replaygain_track_peak", "REPLAYGAIN_TRACK_PEAK"), + ("replaygain_album_gain", "REPLAYGAIN_ALBUM_GAIN"), + ("replaygain_album_peak", "REPLAYGAIN_ALBUM_PEAK"), + ] { + if let Some(value) = fields.get(field) { + set_comment(comments, key, value); + } + } + for (field, aliases) in [ + ("label", &["LABEL", "PUBLISHER"][..]), + ("date", &["YEAR"][..]), + ] { + if fields.contains_key(field) { + for key in aliases { + set_comment(comments, key, ""); + } + } + } + for (field, key) in [("artist", "ARTIST"), ("album_artist", "ALBUMARTIST")] { + if let Some(value) = fields.get(field) { + set_comment(comments, key, ""); + let values = if fields + .get("artist_tag_mode") + .is_some_and(|mode| mode.trim().eq_ignore_ascii_case("split_vorbis")) + { + split_artists(value) + } else { + vec![value.clone()] + }; + for value in values { + if !value.trim().is_empty() { + comments.push(format!("{key}={value}").into_bytes()); + } + } + if field == "album_artist" { + for alias in ["ALBUM ARTIST", "ALBUM_ARTIST"] { + set_comment(comments, alias, ""); + } + } + } + } + for (field, total, key, alias) in [ + ("track_number", "track_total", "TRACKNUMBER", "TRACK"), + ("disc_number", "disc_total", "DISCNUMBER", "DISC"), + ] { + if fields.contains_key(field) || fields.contains_key(total) { + let mut current = pair(&get_comment(comments, key)); + if current == (0, 0) { + current = pair(&get_comment(comments, alias)); + } + set_comment(comments, key, &edit_index(current, fields, field, total)); + set_comment(comments, alias, ""); + } + } + if let Some(value) = fields.get("lyrics") { + set_comment(comments, "SYNCEDLYRICS", ""); + set_comment(comments, "LYRICS", value); + set_comment(comments, "UNSYNCEDLYRICS", value); + } +} + +fn edit_index( + (mut number, mut total): (i64, i64), + fields: &Fields, + key: &str, + total_key: &str, +) -> String { + if let Some(value) = fields.get(key) { + number = value.trim().parse::<i64>().unwrap_or(0); + } + if let Some(value) = fields.get(total_key) { + total = value.trim().parse::<i64>().unwrap_or(0); + } + if number <= 0 { + String::new() + } else if total > 0 { + format!("{number}/{total}") + } else { + number.to_string() + } +} + +fn metadata_fields(metadata: &super::AudioMetadata, fields: &Fields) -> Fields { + let mut result = Fields::new(); + for (field, current) in [ + ("title", &metadata.title), + ("artist", &metadata.artist), + ("album", &metadata.album), + ("album_artist", &metadata.album_artist), + ("date", &metadata.date), + ("genre", &metadata.genre), + ("composer", &metadata.composer), + ("label", &metadata.label), + ("copyright", &metadata.copyright), + ("isrc", &metadata.isrc), + ("lyrics", &metadata.lyrics), + ("comment", &metadata.comment), + ("album_type", &metadata.album_type), + ("upc", &metadata.upc), + ("replaygain_track_gain", &metadata.replay_gain_track_gain), + ("replaygain_track_peak", &metadata.replay_gain_track_peak), + ("replaygain_album_gain", &metadata.replay_gain_album_gain), + ("replaygain_album_peak", &metadata.replay_gain_album_peak), + ] { + result.insert(field.into(), fields.get(field).unwrap_or(current).clone()); + } + static INTEGER: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[+-]?[0-9]+").unwrap()); + for (field, current) in [ + ("track_number", metadata.track_number), + ("track_total", metadata.total_tracks), + ("disc_number", metadata.disc_number), + ("disc_total", metadata.total_discs), + ] { + let value = fields.get(field).map_or(current, |value| { + INTEGER + .find(value.trim()) + .and_then(|n| n.as_str().parse::<isize>().ok()) + .unwrap_or(0) as i64 + }); + result.insert(field.into(), value.to_string()); + } + let explicit = fields + .get("explicit") + .map_or(metadata.explicit, |value| super::truthy(value)); + result.insert("explicit".into(), if explicit { "1" } else { "" }.into()); + let compilation = result["album_type"] + .trim() + .eq_ignore_ascii_case("compilation"); + result.insert( + "compilation".into(), + if compilation { "1" } else { "" }.into(), + ); + result +} + +fn split_artists(value: &str) -> Vec<String> { + static SPLIT: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"(?-u:\s*(?:,|&|\bx\b)\s*|\s+\b(?:feat(?:uring)?|ft|with)\.?\s*)").unwrap() + }); + let value = value.trim(); + if value.is_empty() { + return Vec::new(); + } + let mut seen = BTreeSet::new(); + let mut result: Vec<String> = SPLIT + .split(value) + .map(str::trim) + .filter(|part| !part.is_empty()) + .filter(|part| seen.insert(lowercase(part))) + .map(str::to_owned) + .collect(); + if result.is_empty() { + result.push(value.into()); + } + result +} + +fn cover_mime(data: &[u8]) -> &'static str { + if data.starts_with(b"\x89PNG\r\n\x1a\n") { + "image/png" + } else if data.starts_with(b"\xff\xd8\xff") { + "image/jpeg" + } else if data.starts_with(b"GIF87a") || data.starts_with(b"GIF89a") { + "image/gif" + } else if data.len() >= 12 && &data[..4] == b"RIFF" && &data[8..12] == b"WEBP" { + "image/webp" + } else { + "image/jpeg" + } +} + +fn picture( + original: &[u8], + check: &dyn Fn() -> Result<(), String>, +) -> Result<Option<Vec<u8>>, String> { + let mut resized = None; + if original.len() > 16_000_000 { + let Ok(image) = image::ImageReader::new(Cursor::new(original)) + .with_guessed_format() + .map_err(|e| e.to_string())? + .decode() + else { + return Ok(None); + }; + for quality in [90, 80, 70, 60] { + check()?; + let mut encoded = Vec::new(); + if image::codecs::jpeg::JpegEncoder::new_with_quality(&mut encoded, quality) + .encode_image(&image.to_rgb8()) + .is_ok() + && encoded.len() <= 16_000_000 + { + resized = Some(encoded); + break; + } + } + if resized.is_none() { + for dimension in [1500, 1200, 1000, 800] { + check()?; + let scaled = image + .resize(dimension, dimension, image::imageops::FilterType::Nearest) + .to_rgb8(); + let mut encoded = Vec::new(); + if image::codecs::jpeg::JpegEncoder::new_with_quality(&mut encoded, 85) + .encode_image(&scaled) + .is_ok() + && encoded.len() <= 16_000_000 + { + resized = Some(encoded); + break; + } + } + } + if resized.is_none() { + return Ok(None); + } + } + check()?; + let data = resized.as_deref().unwrap_or(original); + let mime = cover_mime(data); + let dimensions = image::ImageReader::new(Cursor::new(data)) + .with_guessed_format() + .map_err(|e| e.to_string())? + .into_dimensions() + .ok(); + let (width, height, depth) = dimensions.map_or((0, 0, 0), |(w, h)| { + ( + w, + h, + match mime { + "image/png" => 32, + "image/jpeg" => 24, + _ => 0, + }, + ) + }); + let mut result = 3_u32.to_be_bytes().to_vec(); + for value in [mime.as_bytes(), b"Front Cover"] { + result.extend((value.len() as u32).to_be_bytes()); + result.extend(value); + } + for value in [width, height, depth, 0, data.len() as u32] { + result.extend(value.to_be_bytes()); + } + result.extend(data); + Ok(Some(result)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn oversized_flac_cover_reencodes_or_omits_invalid_image() { + let image = image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel( + 3, + 2, + image::Rgba([30, 60, 90, 255]), + )); + let mut encoded = Cursor::new(Vec::new()); + image + .write_to(&mut encoded, image::ImageFormat::Png) + .unwrap(); + let mut cover = encoded.into_inner(); + // Valid PNG with trailing data forces the same size fallback as a + // large artwork file without an expensive image-generation fixture. + cover.resize(16_000_001, 0); + let block = picture(&cover, &|| Ok(())).unwrap().unwrap(); + let mut reader = Cursor::new(block); + let mut number = || { + let mut value = [0; 4]; + reader.read_exact(&mut value).unwrap(); + u32::from_be_bytes(value) + }; + assert_eq!(number(), 3); + assert_eq!(number(), 10); + assert_eq!(bytes(&mut reader, 10).unwrap(), b"image/jpeg"); + assert_eq!(bytes(&mut reader, 4).unwrap(), 11_u32.to_be_bytes()); + assert_eq!(bytes(&mut reader, 11).unwrap(), b"Front Cover"); + assert_eq!(bytes(&mut reader, 4).unwrap(), 3_u32.to_be_bytes()); + assert_eq!(bytes(&mut reader, 4).unwrap(), 2_u32.to_be_bytes()); + assert_eq!(bytes(&mut reader, 4).unwrap(), 24_u32.to_be_bytes()); + assert_eq!(bytes(&mut reader, 4).unwrap(), [0; 4]); + let length = u32::from_be_bytes(bytes(&mut reader, 4).unwrap().try_into().unwrap()); + assert!(length < 16_000_000); + let jpeg = bytes(&mut reader, length as usize).unwrap(); + assert_eq!( + image::ImageReader::new(Cursor::new(jpeg)) + .with_guessed_format() + .unwrap() + .into_dimensions() + .unwrap(), + (3, 2) + ); + cover.fill(0); + assert!(picture(&cover, &|| Ok(())).unwrap().is_none()); + assert_eq!( + picture(b"small cover", &|| Err("cancelled".into())).unwrap_err(), + "cancelled" + ); + } +} diff --git a/rust_backend/crates/core/src/tags/write/ape.rs b/rust_backend/crates/core/src/tags/write/ape.rs new file mode 100644 index 00000000..44fd1db1 --- /dev/null +++ b/rust_backend/crates/core/src/tags/write/ape.rs @@ -0,0 +1,198 @@ +use super::{Fields, MAX_TAG_BYTES, Section, bytes, edit_index, metadata_fields, seek}; +use crate::matching::uppercase; +use crate::tags::{AudioMetadata, containers::ApeFooter}; +use std::collections::BTreeSet; +use std::io::{Read, Seek, SeekFrom}; + +struct Item { + key: Vec<u8>, + value: Vec<u8>, + flags: u32, +} + +pub(super) fn edit( + source: &mut (impl Read + Seek), + fields: &Fields, + cover: Option<&[u8]>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Section, String> { + let end = seek(source, SeekFrom::End(0))?; + let mut start = end; + let mut existing = None; + for offset in [end.checked_sub(32), end.checked_sub(161).map(|_| end - 160)] + .into_iter() + .flatten() + { + check()?; + let Some(footer) = ApeFooter::read(source, offset)? else { + continue; + }; + if start == end { + let size = u64::from(footer.size) + if footer.flags & (1 << 31) != 0 { 32 } else { 0 }; + start = (offset + 32) + .checked_sub(size) + .ok_or("invalid APE tag size")?; + } + if existing.is_none() + && let Ok(begin) = footer.items_start(offset) + { + let size = offset - begin; + if size > MAX_TAG_BYTES as u64 { + return Err("APE metadata exceeds 64 MiB".into()); + } + seek(source, SeekFrom::Start(begin))?; + let data = bytes(source, size as usize)?; + existing = Some(parse_items(&data, footer.count)); + } + } + let metadata = metadata_fields(&AudioMetadata::default(), fields); + let mut added = Vec::new(); + let mut remove = BTreeSet::new(); + for (field, key) in [ + ("title", "Title"), + ("artist", "Artist"), + ("album", "Album"), + ("album_artist", "Album Artist"), + ("genre", "Genre"), + ("date", "Year"), + ("isrc", "ISRC"), + ("lyrics", "Lyrics"), + ("label", "Label"), + ("copyright", "Copyright"), + ("composer", "Composer"), + ("comment", "Comment"), + ("explicit", "ITUNESADVISORY"), + ("album_type", "RELEASETYPE"), + ("upc", "BARCODE"), + ("compilation", "COMPILATION"), + ("replaygain_track_gain", "REPLAYGAIN_TRACK_GAIN"), + ("replaygain_track_peak", "REPLAYGAIN_TRACK_PEAK"), + ("replaygain_album_gain", "REPLAYGAIN_ALBUM_GAIN"), + ("replaygain_album_peak", "REPLAYGAIN_ALBUM_PEAK"), + ] { + if fields.contains_key(field) { + // Go's clear-date override names DATE; a newly emitted Year also + // replaces YEAR, but clearing DATE alone leaves the old YEAR item. + remove.insert(if field == "date" { + "DATE".into() + } else { + uppercase(key) + }); + } + let value = &metadata[field]; + if !value.is_empty() { + added.push(Item { + key: key.as_bytes().to_vec(), + value: value.as_bytes().to_vec(), + flags: 0, + }); + } + } + for (field, total, key) in [ + ("track_number", "track_total", "Track"), + ("disc_number", "disc_total", "Disc"), + ] { + if fields.contains_key(field) { + remove.insert(uppercase(key)); + } + if fields.contains_key(field) || fields.contains_key(total) { + remove.insert(format!("{}NUMBER", uppercase(key))); + } + let value = edit_index((0, 0), &metadata, field, total); + if !value.is_empty() { + added.push(Item { + key: key.as_bytes().to_vec(), + value: value.into_bytes(), + flags: 0, + }); + } + } + for (field, aliases) in [ + ("album_artist", &["ALBUMARTIST"][..]), + ("label", &["PUBLISHER"][..]), + ("lyrics", &["UNSYNCEDLYRICS", "SYNCEDLYRICS"][..]), + ] { + if fields.contains_key(field) { + remove.extend(aliases.iter().map(|s| (*s).to_owned())); + } + } + if fields + .get("cover_path") + .is_some_and(|path| !path.trim().is_empty()) + { + remove.insert("COVER ART (FRONT)".into()); + if let Some(cover) = cover { + let mut value = b"cover.jpg\0".to_vec(); + value.extend(cover); + added.push(Item { + key: b"Cover Art (Front)".to_vec(), + value, + flags: 2, + }); + } + } + remove.extend( + added + .iter() + .map(|item| uppercase(&String::from_utf8_lossy(&item.key))), + ); + let mut items = existing.unwrap_or_default(); + items.retain(|item| !remove.contains(&uppercase(&String::from_utf8_lossy(&item.key)))); + items.extend(added); + if items.is_empty() { + return Err("empty APE tag".into()); + } + let mut body = Vec::new(); + let count = items.len() as u32; + for item in items { + check()?; + if body.len() + 8 + item.key.len() + 1 + item.value.len() + 64 > MAX_TAG_BYTES { + return Err("APE metadata exceeds 64 MiB".into()); + } + body.extend((item.value.len() as u32).to_le_bytes()); + body.extend(item.flags.to_le_bytes()); + body.extend(item.key); + body.push(0); + body.extend(item.value); + } + let size = body.len() as u32 + 32; + let header = |flags: u32| { + let mut result = b"APETAGEX".to_vec(); + for value in [2000_u32, size, count, flags, 0, 0] { + result.extend(value.to_le_bytes()); + } + result + }; + let mut data = header(0xa000_0000); + data.extend(body); + data.extend(header(0x8000_0000)); + Ok(Section { start, end, data }) +} + +fn parse_items(data: &[u8], count: u32) -> Vec<Item> { + let mut items = Vec::new(); + let mut position = 0; + for _ in 0..count { + let Some(header) = data.get(position..position + 8) else { + break; + }; + let size = u32::from_le_bytes(header[..4].try_into().unwrap()) as usize; + let flags = u32::from_le_bytes(header[4..].try_into().unwrap()); + position += 8; + let Some(key_size) = data[position..].iter().position(|byte| *byte == 0) else { + break; + }; + let key = data[position..position + key_size].to_vec(); + position += key_size + 1; + let Some(value) = data.get(position..position.saturating_add(size)) else { + break; + }; + position += size; + items.push(Item { + key, + value: value.to_vec(), + flags, + }); + } + items +} diff --git a/rust_backend/crates/core/src/tags/write/id3.rs b/rust_backend/crates/core/src/tags/write/id3.rs new file mode 100644 index 00000000..8c565671 --- /dev/null +++ b/rust_backend/crates/core/src/tags/write/id3.rs @@ -0,0 +1,381 @@ +use super::{Fields, MAX_TAG_BYTES, bytes, cover_mime, edit_index, pair, seek}; +use crate::matching::uppercase; +use crate::tags::id3::{decode, user_text}; +use std::collections::BTreeSet; +use std::io::{Cursor, Read, Seek, SeekFrom}; + +struct Frame { + id: Vec<u8>, + data: Vec<u8>, +} + +pub(super) fn header( + source: &mut (impl Read + Seek), + fields: &Fields, + cover: Option<&[u8]>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(Vec<u8>, u64), String> { + edit_header(source, fields, cover, None, 512, check) +} + +pub(super) fn fresh( + fields: &Fields, + cover: Option<(&[u8], &str)>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Vec<u8>, String> { + edit_header( + &mut Cursor::new(Vec::<u8>::new()), + fields, + cover.map(|p| p.0), + cover.map(|p| p.1), + 0, + check, + ) + .map(|result| result.0) +} + +fn edit_header( + source: &mut (impl Read + Seek), + fields: &Fields, + cover: Option<&[u8]>, + mime_override: Option<&str>, + padding: usize, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(Vec<u8>, u64), String> { + let (frames, audio_start) = read_frames(source, check)?; + let mut drop = BTreeSet::<Vec<u8>>::new(); + let mut added = Vec::new(); + for (field, id, aliases) in [ + ("title", "TIT2", &[][..]), + ("artist", "TPE1", &[][..]), + ("album", "TALB", &[][..]), + ("album_artist", "TPE2", &[][..]), + ("date", "TDRC", &["TYER", "TDAT", "TIME"][..]), + ("genre", "TCON", &[][..]), + ("label", "TPUB", &[][..]), + ("copyright", "TCOP", &[][..]), + ("composer", "TCOM", &[][..]), + ("isrc", "TSRC", &[][..]), + ("compilation", "TCMP", &[][..]), + ] { + if let Some(value) = fields.get(field) { + drop.insert(id.as_bytes().to_vec()); + for alias in aliases { + drop.insert(alias.as_bytes().to_vec()); + } + if !value.trim().is_empty() { + added.push(frame(id, text(value))); + } + } + } + for (field, id) in [("comment", "COMM"), ("lyrics", "USLT")] { + if let Some(value) = fields.get(field) { + drop.insert(id.as_bytes().to_vec()); + if !value.trim().is_empty() { + let mut data = b"\x03eng\0".to_vec(); + data.extend(value.as_bytes()); + added.push(frame(id, data)); + } + } + } + for (field, total, id) in [ + ("track_number", "track_total", "TRCK"), + ("disc_number", "disc_total", "TPOS"), + ] { + if fields.contains_key(field) || fields.contains_key(total) { + let current = frames + .iter() + .find(|frame| frame.id == id.as_bytes()) + .map(|frame| { + let value = frame + .data + .split_first() + .map(|(encoding, data)| decode(*encoding, data)) + .unwrap_or_default(); + pair(value.split('\0').next().unwrap_or_default()) + }) + .unwrap_or_default(); + drop.insert(id.as_bytes().to_vec()); + let value = edit_index(current, fields, field, total); + if !value.is_empty() { + added.push(frame(id, text(&value))); + } + } + } + let mut drop_descriptions = BTreeSet::new(); + for (field, description) in [ + ("replaygain_track_gain", "REPLAYGAIN_TRACK_GAIN"), + ("replaygain_track_peak", "REPLAYGAIN_TRACK_PEAK"), + ("replaygain_album_gain", "REPLAYGAIN_ALBUM_GAIN"), + ("replaygain_album_peak", "REPLAYGAIN_ALBUM_PEAK"), + ("explicit", "ITUNESADVISORY"), + ("album_type", "RELEASETYPE"), + ("upc", "BARCODE"), + ] { + if let Some(value) = fields.get(field) { + drop_descriptions.insert(description.to_owned()); + if !value.trim().is_empty() { + let mut data = text(description); + data.push(0); + data.extend(value.as_bytes()); + added.push(frame("TXXX", data)); + } + } + } + if let Some(cover) = cover { + drop.insert(b"APIC".to_vec()); + let mime = if let Some(mime) = mime_override { + mime + } else if cover_mime(cover) != "image/jpeg" || cover.starts_with(b"\xff\xd8\xff") { + cover_mime(cover) + } else if cover.starts_with(b"BM") { + "image/bmp" + } else if cover.starts_with(b"%PDF-") { + "application/pdf" + } else if cover + .iter() + .take(512) + .all(|b| !matches!(b, 0..=8 | 11 | 14..=26 | 28..=31)) + { + "text/plain; charset=utf-8" + } else { + "application/octet-stream" + }; + let mut data = text(mime); + data.extend([0, 3, 0]); + data.extend(cover); + added.push(frame("APIC", data)); + } + let kept = frames.into_iter().filter(|frame| { + !drop.contains(&frame.id) + && !(frame.id == b"TXXX" + && drop_descriptions.contains(&uppercase(user_text(&frame.data).0.trim()))) + }); + let mut body = Vec::new(); + for frame in kept.chain(added) { + check()?; + if body.len() + frame.data.len() + 10 + padding > MAX_TAG_BYTES { + return Err("ID3 metadata exceeds 64 MiB".into()); + } + body.extend(frame.id); + body.extend(syncsafe(frame.data.len())); + body.extend([0, 0]); + body.extend(frame.data); + } + body.resize(body.len() + padding, 0); + let mut result = b"ID3\x04\0\0".to_vec(); + result.extend(syncsafe(body.len())); + result.extend(body); + Ok((result, audio_start)) +} + +pub(in crate::tags) fn cover(tag: &[u8]) -> Option<(Vec<u8>, String)> { + let (frames, _) = read_frames(&mut Cursor::new(tag), &|| Ok(())).ok()?; + let data = &frames.iter().find(|frame| frame.id == b"APIC")?.data; + let encoding = *data.first()?; + let mime_end = data[1..].iter().position(|byte| *byte == 0)? + 1; + let mime = String::from_utf8_lossy(&data[1..mime_end]).into_owned(); + let mut start = mime_end + 2; + if matches!(encoding, 0 | 3) { + start += data.get(start..)?.iter().position(|byte| *byte == 0)? + 1; + } else { + start += data + .get(start..)? + .windows(2) + .position(|pair| pair == [0, 0])? + + 2; + } + (start < data.len()).then(|| (data[start..].to_vec(), mime)) +} + +fn frame(id: &str, data: Vec<u8>) -> Frame { + Frame { + id: id.as_bytes().to_vec(), + data, + } +} +fn text(value: &str) -> Vec<u8> { + let mut data = vec![3]; + data.extend(value.as_bytes()); + data +} + +fn syncsafe(size: usize) -> [u8; 4] { + [ + (size >> 21 & 127) as u8, + (size >> 14 & 127) as u8, + (size >> 7 & 127) as u8, + (size & 127) as u8, + ] +} + +fn size(data: &[u8], syncsafe: bool) -> usize { + if syncsafe { + ((data[0] as usize) << 21) + | ((data[1] as usize) << 14) + | ((data[2] as usize) << 7) + | data[3] as usize + } else { + u32::from_be_bytes(data.try_into().unwrap()) as usize + } +} + +fn read_frames( + source: &mut (impl Read + Seek), + check: &dyn Fn() -> Result<(), String>, +) -> Result<(Vec<Frame>, u64), String> { + let mut header = [0; 10]; + let mut received = 0; + while received < header.len() { + let n = source + .read(&mut header[received..]) + .map_err(|e| e.to_string())?; + if n == 0 { + break; + } + received += n; + } + if received < 10 || &header[..3] != b"ID3" { + return Ok((Vec::new(), 0)); + } + let version = header[3]; + let flags = header[5]; + let length = size(&header[6..], true); + if length == 0 { + return Err("invalid ID3v2 tag size".into()); + } + if length > MAX_TAG_BYTES { + return Err("ID3 metadata exceeds 64 MiB".into()); + } + let end = seek(source, SeekFrom::End(0))?; + if 10 + length as u64 > end { + return Err("truncated ID3v2 tag".into()); + } + seek(source, SeekFrom::Start(10))?; + let mut data = bytes(source, length)?; + let audio_start = 10 + length as u64 + if flags & 0x10 != 0 { 10 } else { 0 }; + if flags & 0x10 != 0 && data.len() >= 10 && &data[data.len() - 10..data.len() - 7] == b"3DI" { + data.truncate(data.len() - 10); + } + let mut offset = 0; + if flags & 0x40 != 0 && data.len() >= 4 && matches!(version, 3 | 4) { + let length = size(&data[..4], version == 4); + if length > 0 { + let skip = if length + 4 <= data.len() { + length + 4 + } else if length <= data.len() { + length + } else { + 0 + }; + if skip < data.len() { + offset = skip; + } + } + } + let mut frames = Vec::new(); + let header_length = if version == 2 { 6 } else { 10 }; + while offset + header_length < data.len() { + check()?; + if frames.len() >= 65536 { + return Err("ID3 frame count exceeds 65536".into()); + } + let id_length = if version == 2 { 3 } else { 4 }; + let id = &data[offset..offset + id_length]; + if id[0] == 0 { + break; + } + let length = if version == 2 { + ((data[offset + 3] as usize) << 16) + | ((data[offset + 4] as usize) << 8) + | data[offset + 5] as usize + } else { + size(&data[offset + 4..offset + 8], version == 4) + }; + if length == 0 || length > data.len() - offset - header_length { + break; + } + let frame_flags = if version == 2 { 0 } else { data[offset + 9] }; + let mut payload = &data[offset + header_length..offset + header_length + length]; + offset += header_length + length; + if version == 3 { + if frame_flags & 0xc0 != 0 { + continue; + } + if frame_flags & 0x20 != 0 { + payload = &payload[1..]; + } + } else if version != 2 { + if frame_flags & 0x0c != 0 { + continue; + } + if frame_flags & 0x40 != 0 { + payload = &payload[1..]; + } + if frame_flags & 1 != 0 { + if payload.len() < 4 { + continue; + } + payload = &payload[4..]; + } + } + let mut payload = + if flags & 0x80 != 0 || (version != 2 && version != 3 && frame_flags & 2 != 0) { + let mut result = Vec::new(); + let mut offset = 0; + while offset < payload.len() { + result.push(payload[offset]); + offset += if payload[offset] == 0xff && payload.get(offset + 1) == Some(&0) { + 2 + } else { + 1 + }; + } + result + } else { + payload.to_vec() + }; + let id = if version == 2 { + match id { + b"TT2" => b"TIT2".as_slice(), + b"TP1" => b"TPE1", + b"TP2" => b"TPE2", + b"TAL" => b"TALB", + b"TYE" => b"TDRC", + b"TCO" => b"TCON", + b"TRK" => b"TRCK", + b"TPA" => b"TPOS", + b"TCM" => b"TCOM", + b"TPB" => b"TPUB", + b"TCR" => b"TCOP", + b"TXX" => b"TXXX", + b"ULT" => b"USLT", + b"COM" => b"COMM", + b"PIC" => { + if payload.len() < 5 { + continue; + } + let mime = if payload[1..4].eq_ignore_ascii_case(b"PNG") { + "image/png" + } else { + "image/jpeg" + }; + let mut converted = vec![payload[0]]; + converted.extend(mime.as_bytes()); + converted.push(0); + converted.extend(&payload[4..]); + payload = converted; + b"APIC" + } + _ => continue, + } + } else { + id + }; + frames.push(Frame { + id: id.to_vec(), + data: payload, + }); + } + Ok((frames, audio_start)) +} diff --git a/rust_backend/crates/core/src/tags/write/mp4.rs b/rust_backend/crates/core/src/tags/write/mp4.rs new file mode 100644 index 00000000..eab3781b --- /dev/null +++ b/rust_backend/crates/core/src/tags/write/mp4.rs @@ -0,0 +1,608 @@ +pub(super) mod ac4; + +use super::{Fields, MAX_TAG_BYTES, Section, bytes, seek}; +use crate::matching::uppercase; +use crate::media::mp4::{Atom, Reader}; +use regex::Regex; +use std::cell::Cell; +use std::collections::BTreeSet; +use std::io::{Cursor, Read, Seek, SeekFrom}; +use std::sync::LazyLock; + +type Check<'a> = &'a dyn Fn() -> Result<(), String>; + +struct Location { + ilst: Atom, + ancestors: Vec<Atom>, +} + +pub(super) fn edit( + source: &mut (impl Read + Seek), + fields: &Fields, + cover: Option<&[u8]>, + check: Check<'_>, +) -> Result<Vec<Section>, String> { + let mut section = load(source, b"moov", check)?.ok_or("moov not found")?; + let data = &mut section.data; + let location = ensure_ilst(data, section.start, check)?; + let mut drop = BTreeSet::new(); + let mut appended = Vec::new(); + for (field, kind) in [ + ("title", b"\xa9nam"), + ("artist", b"\xa9ART"), + ("album", b"\xa9alb"), + ("album_artist", b"aART"), + ("date", b"\xa9day"), + ("genre", b"\xa9gen"), + ("composer", b"\xa9wrt"), + ("comment", b"\xa9cmt"), + ("copyright", b"cprt"), + ("lyrics", b"\xa9lyr"), + ] { + if let Some(value) = fields.get(field) { + drop.insert(*kind); + if field == "genre" { + drop.insert(*b"gnre"); + } + if !value.trim().is_empty() { + appended.extend(value_atom(kind, 1, value.as_bytes())); + } + } + } + let mut remove_names = BTreeSet::new(); + for (field, name) in [ + ("isrc", "ISRC"), + ("label", "LABEL"), + ("album_type", "RELEASETYPE"), + ("upc", "BARCODE"), + ] { + if let Some(value) = fields.get(field) { + remove_names.insert(name.to_owned()); + if !value.trim().is_empty() { + appended.extend(freeform(name, value.trim())); + } + } + } + if fields.contains_key("label") { + remove_names.insert("ORGANIZATION".into()); + } + if fields.contains_key("lyrics") { + remove_names.extend(["LYRICS", "UNSYNCEDLYRICS", "SYNCEDLYRICS"].map(str::to_owned)); + } + for (field, kind, data_type) in [("explicit", b"rtng", 21), ("compilation", b"cpil", 22)] { + if let Some(value) = fields.get(field) { + drop.insert(*kind); + if matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "explicit" + ) { + appended.extend(value_atom(kind, data_type, &[1])); + } + } + } + let replay_gain = replay_gain(fields); + if !replay_gain.is_empty() { + // Go replaces the entire ReplayGain group when any supplied value is + // nonempty. A request containing only empty values leaves it alone. + remove_names.extend( + [ + "REPLAYGAIN_TRACK_GAIN", + "REPLAYGAIN_TRACK_PEAK", + "REPLAYGAIN_ALBUM_GAIN", + "REPLAYGAIN_ALBUM_PEAK", + "ITUNNORM", + ] + .map(str::to_owned), + ); + for (name, value) in replay_gain { + appended.extend(freeform(&name, &value)); + } + } + let edit_track = fields.contains_key("track_number") || fields.contains_key("track_total"); + let edit_disc = fields.contains_key("disc_number") || fields.contains_key("disc_total"); + if cover.is_some() { + drop.insert(*b"covr"); + } + let mut body = Vec::new(); + let mut track = (0, 0); + let mut disc = (0, 0); + for child in children(data, location.ilst.payload, location.ilst.end, check)? { + let keep = if drop.contains(&child.kind) { + false + } else if child.kind == *b"trkn" { + track = index_pair(data, child, check)?; + !edit_track + } else if child.kind == *b"disk" { + disc = index_pair(data, child, check)?; + !edit_disc + } else if child.kind == *b"----" { + !remove_names.contains(&uppercase(&freeform_name(data, child, check)?)) + } else { + true + }; + if keep { + body.extend_from_slice(&data[child.start as usize..child.end as usize]); + } + } + for (edited, current, field, total, kind) in [ + (edit_track, track, "track_number", "track_total", b"trkn"), + (edit_disc, disc, "disc_number", "disc_total", b"disk"), + ] { + if edited { + let number = fields.get(field).map_or(current.0, |v| positive(v)); + let total = fields.get(total).map_or(current.1, |v| positive(v)); + if number > 0 { + let mut pair = vec![0; if kind == b"disk" { 6 } else { 8 }]; + pair[2..4].copy_from_slice(&(number as u16).to_be_bytes()); + pair[4..6].copy_from_slice(&(total as u16).to_be_bytes()); + appended.extend(value_atom(kind, 0, &pair)); + } + } + } + if let Some(cover) = cover { + appended.extend(value_atom( + b"covr", + if cover.starts_with(b"\x89PNG") { + 14 + } else { + 13 + }, + cover, + )); + } + if body.len() + appended.len() > MAX_TAG_BYTES { + return Err("MP4 metadata exceeds 64 MiB".into()); + } + body.extend(appended); + replace( + data, + location.ilst.start, + location.ilst.end, + &build(b"ilst", &body), + &location.ancestors, + section.start, + check, + )?; + Ok(vec![section]) +} + +fn load( + source: &mut (impl Read + Seek), + kind: &[u8; 4], + check: Check<'_>, +) -> Result<Option<Section>, String> { + let size = seek(source, SeekFrom::End(0))?; + let mut reader = Reader { + file: source, + size, + check, + }; + let mut start = 0; + for _ in 0..65536 { + if start + 8 > size { + return Ok(None); + } + let atom = reader.atom(start, size)?; + if atom.end > size { + return Err("MP4 atom extends past end of file".into()); + } + if &atom.kind == kind { + let length = atom.end - atom.start; + if length > MAX_TAG_BYTES as u64 { + return Err("MP4 metadata exceeds 64 MiB".into()); + } + seek(reader.file, SeekFrom::Start(start))?; + return Ok(Some(Section { + start, + end: atom.end, + data: bytes(reader.file, length as usize)?, + })); + } + start = atom.end; + } + Err("MP4 atom count exceeds 65536".into()) +} + +fn atom(data: &[u8], start: u64, end: u64, check: Check<'_>) -> Result<Atom, String> { + let mut source = Cursor::new(data); + let atom = Reader { + file: &mut source, + size: data.len() as u64, + check, + } + .atom(start, end)?; + if atom.end > end || atom.end > data.len() as u64 { + return Err("MP4 atom extends past parent".into()); + } + Ok(atom) +} + +fn children(data: &[u8], mut start: u64, end: u64, check: Check<'_>) -> Result<Vec<Atom>, String> { + let mut result = Vec::new(); + while start + 8 <= end { + if result.len() >= 65536 { + return Err("MP4 atom count exceeds 65536".into()); + } + let child = atom(data, start, end, check)?; + start = child.end; + result.push(child); + } + Ok(result) +} + +fn find( + data: &[u8], + mut start: u64, + end: u64, + kind: &[u8; 4], + check: Check<'_>, +) -> Result<Option<Atom>, String> { + for _ in 0..65536 { + if start + 8 > end { + return Ok(None); + } + let child = atom(data, start, end, check)?; + if &child.kind == kind { + return Ok(Some(child)); + } + start = child.end; + } + Err("MP4 atom count exceeds 65536".into()) +} + +fn locate_meta( + data: &[u8], + meta: Atom, + quicktime: bool, + check: Check<'_>, +) -> Result<Option<Atom>, String> { + let cancelled = Cell::new(false); + let guard = || { + let result = check(); + if result.is_err() { + cancelled.set(true); + } + result + }; + let iso = find(data, meta.payload + 4, meta.end, b"ilst", &guard); + if !quicktime || cancelled.get() || iso.as_ref().is_ok_and(Option::is_some) { + return iso; + } + find(data, meta.payload, meta.end, b"ilst", check) +} + +fn locate(data: &[u8], quicktime: bool, check: Check<'_>) -> Result<Option<Location>, String> { + let moov = atom(data, 0, data.len() as u64, check)?; + if moov.kind != *b"moov" { + return Err("moov not found".into()); + } + if let Some(udta) = find(data, moov.payload, moov.end, b"udta", check)? + && let Some(meta) = find(data, udta.payload, udta.end, b"meta", check)? + && let Some(ilst) = locate_meta(data, meta, quicktime, check)? + { + return Ok(Some(Location { + ilst, + ancestors: vec![moov, udta, meta], + })); + } + if let Some(meta) = find(data, moov.payload, moov.end, b"meta", check)? + && let Some(ilst) = locate_meta(data, meta, quicktime, check)? + { + return Ok(Some(Location { + ilst, + ancestors: vec![moov, meta], + })); + } + Ok(None) +} + +fn meta(body: &[u8]) -> Vec<u8> { + let mut handler = vec![0; 25]; + handler[8..12].copy_from_slice(b"mdir"); + handler[12..16].copy_from_slice(b"appl"); + let mut payload = vec![0; 4]; + payload.extend(build(b"hdlr", &handler)); + payload.extend(build(b"ilst", body)); + build(b"meta", &payload) +} + +fn ensure_ilst(data: &mut Vec<u8>, base: u64, check: Check<'_>) -> Result<Location, String> { + if let Some(location) = locate(data, false, check)? { + return Ok(location); + } + let moov = atom(data, 0, data.len() as u64, check)?; + let mut ancestors = vec![moov]; + let (position, inserted) = + if let Some(udta) = find(data, moov.payload, moov.end, b"udta", check)? { + ancestors.push(udta); + if let Some(meta) = find(data, udta.payload, udta.end, b"meta", check)? { + ancestors.push(meta); + (meta.end, build(b"ilst", &[])) + } else { + (udta.end, meta(&[])) + } + } else { + (moov.end, build(b"udta", &meta(&[]))) + }; + replace(data, position, position, &inserted, &ancestors, base, check)?; + locate(data, false, check)?.ok_or_else(|| "failed to create ilst".into()) +} + +fn replace( + data: &mut Vec<u8>, + start: u64, + end: u64, + replacement: &[u8], + ancestors: &[Atom], + base: u64, + check: Check<'_>, +) -> Result<(), String> { + check()?; + let delta = replacement.len() as i64 - (end - start) as i64; + if (data.len() as u64) + .checked_add_signed(delta) + .is_none_or(|n| n > MAX_TAG_BYTES as u64) + { + return Err("MP4 metadata exceeds 64 MiB".into()); + } + data.splice(start as usize..end as usize, replacement.iter().copied()); + for ancestor in ancestors { + grow(data, *ancestor, delta)?; + } + shift_offsets(data, base + start, delta, check) +} + +fn grow(data: &mut [u8], atom: Atom, delta: i64) -> Result<(), String> { + let size = (atom.end - atom.start) + .checked_add_signed(delta) + .ok_or("invalid resized MP4 atom")?; + let start = atom.start as usize; + if atom.payload - atom.start == 16 { + data[start..start + 4].copy_from_slice(&1_u32.to_be_bytes()); + data[start + 8..start + 16].copy_from_slice(&size.to_be_bytes()); + } else { + let size = u32::try_from(size).map_err(|_| "MP4 atom too large for 32-bit header")?; + data[start..start + 4].copy_from_slice(&size.to_be_bytes()); + } + Ok(()) +} + +fn shift_offsets( + data: &mut [u8], + position: u64, + delta: i64, + check: Check<'_>, +) -> Result<(), String> { + if delta == 0 { + return Ok(()); + } + let moov = atom(data, 0, data.len() as u64, check)?; + for track in children(data, moov.payload, moov.end, check)? + .into_iter() + .filter(|atom| atom.kind == *b"trak") + { + let mut parent = Some(track); + for kind in [b"mdia", b"minf", b"stbl"] { + parent = match parent { + Some(p) => find(data, p.payload, p.end, kind, check)?, + None => None, + }; + } + let Some(parent) = parent else { + continue; + }; + for (kind, width) in [(b"stco", 4), (b"co64", 8)] { + let Some(table) = find(data, parent.payload, parent.end, kind, check)? else { + continue; + }; + if table.payload + 8 > table.end { + continue; + } + let begin = table.payload as usize; + let count = u32::from_be_bytes(data[begin + 4..begin + 8].try_into().unwrap()) as usize; + for offset in (begin + 8..table.end as usize).step_by(width).take(count) { + check()?; + if offset + width > table.end as usize { + break; + } + let value = if width == 4 { + u32::from_be_bytes(data[offset..offset + 4].try_into().unwrap()) as u64 + } else { + u64::from_be_bytes(data[offset..offset + 8].try_into().unwrap()) + }; + if value >= position { + let shifted = value + .checked_add_signed(delta) + .ok_or("MP4 chunk offset overflow")?; + if width == 4 { + let shifted = + u32::try_from(shifted).map_err(|_| "MP4 stco offset overflow")?; + data[offset..offset + 4].copy_from_slice(&shifted.to_be_bytes()); + } else { + data[offset..offset + 8].copy_from_slice(&shifted.to_be_bytes()); + } + } + } + } + } + Ok(()) +} + +fn build(kind: &[u8; 4], payload: &[u8]) -> Vec<u8> { + let mut result = ((payload.len() + 8) as u32).to_be_bytes().to_vec(); + result.extend(kind); + result.extend(payload); + result +} + +fn value_atom(kind: &[u8; 4], data_type: u32, value: &[u8]) -> Vec<u8> { + let mut payload = data_type.to_be_bytes().to_vec(); + payload.extend([0; 4]); + payload.extend(value); + build(kind, &build(b"data", &payload)) +} + +fn freeform(name: &str, value: &str) -> Vec<u8> { + let mut body = build(b"mean", b"\0\0\0\0com.apple.iTunes"); + let mut name_payload = vec![0; 4]; + name_payload.extend(name.as_bytes()); + body.extend(build(b"name", &name_payload)); + let mut value_payload = vec![0, 0, 0, 1, 0, 0, 0, 0]; + value_payload.extend(value.as_bytes()); + body.extend(build(b"data", &value_payload)); + build(b"----", &body) +} + +fn freeform_name(data: &[u8], parent: Atom, check: Check<'_>) -> Result<String, String> { + let Some(name) = find(data, parent.payload, parent.end, b"name", check)? else { + return Ok(String::new()); + }; + if name.payload + 4 >= name.end { + return Ok(String::new()); + } + Ok( + String::from_utf8_lossy(&data[name.payload as usize + 4..name.end as usize]) + .trim_end_matches('\0') + .trim() + .into(), + ) +} + +fn index_pair(data: &[u8], parent: Atom, check: Check<'_>) -> Result<(i64, i64), String> { + let Some(value) = find(data, parent.payload, parent.end, b"data", check)? else { + return Ok((0, 0)); + }; + if value.payload + 14 > value.end { + return Ok((0, 0)); + } + let start = value.payload as usize + 8; + Ok(( + u16::from_be_bytes(data[start + 2..start + 4].try_into().unwrap()) as i64, + u16::from_be_bytes(data[start + 4..start + 6].try_into().unwrap()) as i64, + )) +} + +fn positive(value: &str) -> i64 { + value.trim().parse::<isize>().unwrap_or(0).max(0) as i64 +} + +fn replay_gain(fields: &Fields) -> Fields { + let mut result: Fields = [ + "replaygain_track_gain", + "replaygain_track_peak", + "replaygain_album_gain", + "replaygain_album_peak", + ] + .into_iter() + .filter_map(|name| { + fields + .get(name) + .map(|v| (name.to_owned(), v.trim().to_owned())) + }) + .filter(|(_, value)| !value.is_empty()) + .collect(); + static NUMBER: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"[+-]?[0-9]+(?:\.[0-9]+)?").unwrap()); + let gain = result + .get("replaygain_track_gain") + .and_then(|v| NUMBER.find(v)) + .and_then(|m| m.as_str().parse::<f64>().ok()); + let peak = result + .get("replaygain_track_peak") + .and_then(|v| v.parse::<f64>().ok()) + .filter(|v| v.is_nan() || *v > 0.0); + if let (Some(gain), Some(peak)) = (gain, peak) { + let clamp = |value: f64| -> i64 { + let value = value.round(); + if !value.is_finite() + || !(-9223372036854775808.0..9223372036854775808.0).contains(&value) + { + 0 + } else { + (value as i64).clamp(0, 65534) + } + }; + let g1 = clamp(10_f64.powf(gain / -10.0) * 1000.0); + let g2 = clamp(10_f64.powf(gain / -10.0) * 2500.0); + let peak = clamp(peak * 32768.0); + result.insert( + "iTunNORM".into(), + [g1, g1, g2, g2, 0, 0, peak, peak, 0, 0] + .map(|n| format!("{n:08X}")) + .join(" "), + ); + } + result +} + +pub(super) fn edit_freeform( + source: &mut (impl Read + Seek), + fields: &Fields, + replay_gain_only: bool, + check: Check<'_>, +) -> Result<Vec<Section>, String> { + let mut remove = BTreeSet::new(); + let values = if replay_gain_only { + let values = replay_gain(fields); + if values.is_empty() { + return Ok(Vec::new()); + } + remove.extend( + [ + "REPLAYGAIN_TRACK_GAIN", + "REPLAYGAIN_TRACK_PEAK", + "REPLAYGAIN_ALBUM_GAIN", + "REPLAYGAIN_ALBUM_PEAK", + "ITUNNORM", + ] + .map(str::to_owned), + ); + values + } else { + let mut values = Fields::new(); + for (field, name) in [("isrc", "ISRC"), ("label", "LABEL")] { + if let Some(value) = fields.get(field) { + remove.insert(name.to_owned()); + values.insert(name.to_owned(), value.trim().to_owned()); + if field == "label" { + remove.insert("ORGANIZATION".into()); + } + } + } + if values.is_empty() { + return Ok(Vec::new()); + } + values + }; + let mut section = load(source, b"moov", check)?.ok_or("moov not found")?; + let data = &mut section.data; + let Some(location) = locate(data, true, check)? else { + return Ok(Vec::new()); + }; + let mut body = Vec::new(); + for child in children(data, location.ilst.payload, location.ilst.end, check)? { + if child.kind != *b"----" + || !remove.contains(&uppercase(&freeform_name(data, child, check)?)) + { + body.extend_from_slice(&data[child.start as usize..child.end as usize]); + } + } + for (name, value) in values { + if !value.is_empty() { + body.extend(freeform(&name, &value)); + } + if body.len() > MAX_TAG_BYTES { + return Err("MP4 metadata exceeds 64 MiB".into()); + } + } + replace( + data, + location.ilst.start, + location.ilst.end, + &build(b"ilst", &body), + &location.ancestors, + section.start, + check, + )?; + Ok(vec![section]) +} diff --git a/rust_backend/crates/core/src/tags/write/mp4/ac4.rs b/rust_backend/crates/core/src/tags/write/mp4/ac4.rs new file mode 100644 index 00000000..a7e5c199 --- /dev/null +++ b/rust_backend/crates/core/src/tags/write/mp4/ac4.rs @@ -0,0 +1,267 @@ +use super::{Atom, Check, Fields, MAX_TAG_BYTES, Section, atom, build, children, find, freeform}; +use super::{load, meta, replace, seek, value_atom}; +use crate::tags::CheckedReader; +use serde::de::{MapAccess, Visitor}; +use serde::{Deserialize, Deserializer}; +use std::fmt; +use std::io::{BufReader, ErrorKind, Read, Seek, SeekFrom}; + +struct Location { + ancestors: Vec<Atom>, + entry: Atom, +} + +fn locate(data: &[u8], check: Check<'_>) -> Result<Option<Location>, String> { + let moov = atom(data, 0, data.len() as u64, check)?; + for track in children(data, moov.payload, moov.end, check)? + .into_iter() + .filter(|a| a.kind == *b"trak") + { + let mut ancestors = vec![moov, track]; + for kind in [b"mdia", b"minf", b"stbl", b"stsd"] { + let parent = *ancestors.last().unwrap(); + let Some(child) = find(data, parent.payload, parent.end, kind, check)? else { + break; + }; + ancestors.push(child); + } + if ancestors.len() == 6 { + let stsd = ancestors[5]; + if let Some(entry) = find(data, stsd.payload + 8, stsd.end, b"ac-4", check)? { + return Ok(Some(Location { ancestors, entry })); + } + } + } + Ok(None) +} + +fn audio_header(data: &[u8], entry: Atom) -> Result<(u16, u64), String> { + let base = entry.payload as usize; + if entry.payload + 10 > entry.end { + return Err("malformed ac-4 sample entry".into()); + } + let version = u16::from_be_bytes(data[base + 8..base + 10].try_into().unwrap()); + let length = match version { + 1 => 44, + 2 => 64, + _ => 28, + }; + if entry.payload + length > entry.end { + return Err("malformed ac-4 sample entry".into()); + } + Ok((version, length)) +} + +pub(in super::super) fn config<R: Read + Seek>( + source: &mut (impl Read + Seek), + reference: impl FnOnce() -> Result<R, String>, + check: Check<'_>, +) -> Result<Vec<Section>, String> { + seek(source, SeekFrom::Start(0))?; + let mut head = [0; 8]; + match source.read_exact(&mut head) { + Err(error) if error.kind() == ErrorKind::UnexpectedEof => return Ok(Vec::new()), + Err(error) => return Err(error.to_string()), + Ok(()) => {} + } + if !head[4..].iter().all(|byte| (0x20..=0x7e).contains(byte)) { + return Ok(Vec::new()); + } + let Some(mut section) = load(source, b"moov", check)? else { + return Ok(Vec::new()); + }; + let ftyp = load(source, b"ftyp", check)?; + let Some(mut location) = locate(§ion.data, check)? else { + return Ok(Vec::new()); + }; + let mut sections = Vec::new(); + if let Some(mut ftyp) = ftyp { + let header = atom(&ftyp.data, 0, ftyp.data.len() as u64, check)?; + let base = header.payload as usize; + let mut changed = false; + if base + 4 <= ftyp.data.len() && &ftyp.data[base..base + 4] != b"mp42" { + ftyp.data[base..base + 4].copy_from_slice(b"mp42"); + changed = true; + } + let brands_start = ftyp.data.len().min(base + 8); + for brand in ftyp.data[brands_start..].as_chunks_mut::<4>().0 { + check()?; + if brand == b"qt " { + brand.copy_from_slice(b"isom"); + changed = true; + } + } + if changed { + sections.push(ftyp); + } + } + let data = &mut section.data; + let (version, _) = audio_header(data, location.entry)?; + if version == 1 { + let base = location.entry.payload; + data[base as usize + 8..base as usize + 10].fill(0); + location.ancestors.push(location.entry); + replace( + data, + base + 28, + base + 44, + &[], + &location.ancestors, + section.start, + check, + )?; + location = locate(data, check)?.ok_or("ac-4 entry lost during normalization")?; + } + let (_, header) = audio_header(data, location.entry)?; + let position = location.entry.payload + header; + if find(data, position, location.entry.end, b"dac4", check)?.is_none() { + check()?; + let mut reference = reference()?; + let mut reference = BufReader::new(CheckedReader { + reader: &mut reference, + check, + }); + let reference = load(&mut reference, b"moov", check)?.ok_or("source has no moov")?; + let moov = atom(&reference.data, 0, reference.data.len() as u64, check)?; + let mut config = None; + // Encrypted sample entries can contain dac4 below opaque enca headers. + for start in moov.payload..moov.end.saturating_sub(7) { + if start % 4096 == 0 { + check()?; + } + let index = start as usize; + if &reference.data[index + 4..index + 8] == b"dac4" { + // A signature match is only a candidate. Malformed candidates + // are skipped, but cancellation always propagates. + if let Ok(found) = atom(&reference.data, start, moov.end, &|| Ok(())) { + config = Some(found); + break; + } + } + } + check()?; + let config = config.ok_or("dac4 not found in source")?; + location.ancestors.push(location.entry); + replace( + data, + position, + position, + &reference.data[config.start as usize..config.end as usize], + &location.ancestors, + section.start, + check, + )?; + } + sections.push(section); + Ok(sections) +} + +#[derive(Default)] +struct Metadata(Fields); + +impl<'de> Deserialize<'de> for Metadata { + fn deserialize<D: Deserializer<'de>>(decoder: D) -> Result<Self, D::Error> { + struct Strings; + impl<'de> Visitor<'de> for Strings { + type Value = Metadata; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("AC-4 metadata object") + } + + fn visit_map<M: MapAccess<'de>>(self, mut map: M) -> Result<Self::Value, M::Error> { + let mut result = Metadata::default(); + while let Some((key, value)) = map.next_entry::<String, serde_json::Value>()? { + // Go retains prior string values on null/type errors and + // continues decoding later fields; duplicate keys are ordered. + if let serde_json::Value::String(value) = value { + result.0.insert(key.to_ascii_lowercase(), value); + } + } + Ok(result) + } + } + decoder.deserialize_map(Strings) + } +} + +pub(in super::super) fn metadata( + source: &mut (impl Read + Seek), + metadata_json: &str, + cover: impl FnOnce() -> Result<Option<Vec<u8>>, String>, + check: Check<'_>, +) -> Result<Vec<Section>, String> { + let Some(mut section) = load(source, b"moov", check)? else { + return Ok(Vec::new()); + }; + if locate(§ion.data, check)?.is_none() { + return Ok(Vec::new()); + } + if metadata_json.len() > MAX_TAG_BYTES { + return Err("tag edit fields exceed 64 MiB".into()); + } + let metadata: Metadata = + serde_json::from_str(&crate::text::json_surrogates(metadata_json)).unwrap_or_default(); + let value = |name: &str| metadata.0.get(name).map(String::as_str).unwrap_or(""); + let mut body = Vec::new(); + for (field, kind) in [ + ("title", b"\xa9nam"), + ("artist", b"\xa9ART"), + ("album", b"\xa9alb"), + ("albumartist", b"aART"), + ("date", b"\xa9day"), + ("genre", b"\xa9gen"), + ("composer", b"\xa9wrt"), + ("copyright", b"cprt"), + ("lyrics", b"\xa9lyr"), + ] { + if !value(field).trim().is_empty() { + body.extend(value_atom(kind, 1, value(field).as_bytes())); + } + } + for (number, total, kind) in [ + ("tracknumber", "totaltracks", b"trkn"), + ("discnumber", "totaldiscs", b"disk"), + ] { + let number = value(number).trim().parse::<isize>().unwrap_or(0); + if number > 0 { + let total = value(total).trim().parse::<isize>().unwrap_or(0); + let mut pair = [0; 8]; + pair[2..4].copy_from_slice(&(number as u16).to_be_bytes()); + pair[4..6].copy_from_slice(&(total as u16).to_be_bytes()); + body.extend(value_atom(kind, 0, &pair)); + } + } + for (field, name) in [("isrc", "ISRC"), ("label", "LABEL")] { + if !value(field).trim().is_empty() { + body.extend(freeform(name, value(field).trim())); + } + } + check()?; + if let Some(cover) = cover()?.filter(|cover| !cover.is_empty()) { + if body.len().saturating_add(cover.len()) > MAX_TAG_BYTES { + return Err("MP4 metadata exceeds 64 MiB".into()); + } + let kind = if cover.len() >= 8 && cover.starts_with(b"\x89PNG") { + 14 + } else { + 13 + }; + body.extend(value_atom(b"covr", kind, &cover)); + } + let data = &mut section.data; + let moov = atom(data, 0, data.len() as u64, check)?; + let (start, end) = find(data, moov.payload, moov.end, b"udta", check)? + .map_or((moov.end, moov.end), |udta| (udta.start, udta.end)); + // The AC-4 helper replaces the whole udta, matching Go's finalization path. + replace( + data, + start, + end, + &build(b"udta", &meta(&body)), + &[moov], + section.start, + check, + )?; + Ok(vec![section]) +} diff --git a/rust_backend/crates/core/src/tags/write/ogg.rs b/rust_backend/crates/core/src/tags/write/ogg.rs new file mode 100644 index 00000000..849c9e2b --- /dev/null +++ b/rust_backend/crates/core/src/tags/write/ogg.rs @@ -0,0 +1,228 @@ +use super::{ + Fields, MAX_TAG_BYTES, copy, edit_comments, parse_comments, picture, put_string, seek, + set_comment, +}; +use crate::tags::containers::OggPage; +use base64::{Engine, engine::general_purpose::STANDARD}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::sync::LazyLock; + +type Check<'a> = &'a dyn Fn() -> Result<(), String>; + +fn read_page(source: &mut impl Read) -> Result<Option<OggPage>, String> { + let page = OggPage::read(source)?; + if page.as_ref().is_some_and(|page| page.header[4] != 0) { + return Err("invalid ogg page".into()); + } + Ok(page) +} + +fn write_page(page: &OggPage, output: &mut impl Write, check: Check<'_>) -> Result<(), String> { + check()?; + let mut bytes = Vec::with_capacity(page.len()); + bytes.extend(page.header); + bytes[22..26].fill(0); + bytes[26] = page.segments.len() as u8; + bytes.extend(&page.segments); + bytes.extend(&page.data); + static CRC: LazyLock<[u32; 256]> = LazyLock::new(|| { + std::array::from_fn(|index| { + let mut value = (index as u32) << 24; + for _ in 0..8 { + value = (value << 1) + ^ if value & 0x8000_0000 != 0 { + 0x04c1_1db7 + } else { + 0 + }; + } + value + }) + }); + let crc = bytes.iter().fold(0_u32, |crc, byte| { + (crc << 8) ^ CRC[((crc >> 24) as u8 ^ byte) as usize] + }); + bytes[22..26].copy_from_slice(&crc.to_le_bytes()); + output.write_all(&bytes).map_err(|error| error.to_string()) +} + +pub(super) fn rewrite( + source: &mut (impl Read + Seek), + output: &mut impl Write, + fields: &Fields, + cover: Option<&[u8]>, + check: Check<'_>, +) -> Result<(), String> { + let first = read_page(source)?.ok_or("ogg stream too short")?; + let (opus, packet_count) = if first.data.starts_with(b"OpusHead") { + (true, 2) + } else if first.data.starts_with(b"\x01vorbis") { + (false, 3) + } else { + return Err("unsupported ogg codec".into()); + }; + let serial = &first.header[14..18]; + let mut packets = Vec::new(); + let mut current = Vec::new(); + let mut page = first.clone(); + let mut header_bytes = 0; + let mut header_pages = 0; + loop { + check()?; + header_pages += 1; + header_bytes += page.len(); + if header_bytes > MAX_TAG_BYTES { + return Err("Ogg headers exceed 64 MiB".into()); + } + if &page.header[14..18] != serial { + return Err("multiplexed ogg streams are not supported".into()); + } + let mut offset = 0; + for (index, length) in page.segments.iter().copied().enumerate() { + let end = offset + usize::from(length); + current.extend_from_slice(&page.data[offset..end]); + offset = end; + if length < 255 { + packets.push(std::mem::take(&mut current)); + if packets.len() == packet_count { + if index + 1 != page.segments.len() { + return Err("header packet shares a page with audio".into()); + } + break; + } + } + } + if packets.len() == packet_count { + break; + } + if header_pages >= 1024 { + return Err("ogg header spans too many pages".into()); + } + page = read_page(source)?.ok_or("incomplete ogg header packets")?; + } + let prefix: &[u8] = if opus { b"OpusTags" } else { b"\x03vorbis" }; + let body = packets[1] + .strip_prefix(prefix) + .ok_or("comment header not found")?; + let (vendor, mut comments) = parse_comments(body)?; + edit_comments(&mut comments, fields); + if opus { + opus_gain(&mut comments, fields)?; + } + if let Some(cover) = cover + && let Some(picture) = picture(cover, check)? + { + set_comment( + &mut comments, + "METADATA_BLOCK_PICTURE", + &STANDARD.encode(picture), + ); + } + let length = prefix.len() + + 8 + + vendor.len() + + comments + .iter() + .map(|comment| 4 + comment.len()) + .sum::<usize>() + + usize::from(!opus); + if length > MAX_TAG_BYTES { + return Err("Ogg comments exceed 64 MiB".into()); + } + let mut comment = prefix.to_vec(); + put_string(&mut comment, &vendor); + comment.extend((comments.len() as u32).to_le_bytes()); + for value in comments { + put_string(&mut comment, &value); + } + if !opus { + comment.push(1); + } + packets[1] = comment; + // Retain the BOS page (including OpusHead output gain) and the Vorbis + // setup packet; only the comment/setup pages are repaginated. + write_page(&first, output, check)?; + let next_sequence = write_headers(&packets[1..], serial, output, check)?; + if next_sequence as usize == header_pages { + seek(source, SeekFrom::Start(header_bytes as u64))?; + return copy(source, output, None, check); + } + let mut sequence = next_sequence; + while let Some(mut page) = read_page(source)? { + check()?; + if &page.header[14..18] != serial { + return Err("multiplexed ogg streams are not supported".into()); + } + page.header[18..22].copy_from_slice(&sequence.to_le_bytes()); + sequence = sequence.wrapping_add(1); + write_page(&page, output, check)?; + } + check() +} + +fn write_headers( + packets: &[Vec<u8>], + serial: &[u8], + output: &mut impl Write, + check: Check<'_>, +) -> Result<u32, String> { + let mut page = OggPage { + header: [0; 27], + segments: Vec::new(), + data: Vec::new(), + }; + page.header[..4].copy_from_slice(b"OggS"); + page.header[14..18].copy_from_slice(serial); + let mut sequence = 1_u32; + for packet in packets { + let mut rest = packet.as_slice(); + loop { + let count = rest.len().min(255); + page.segments.push(count as u8); + page.data.extend_from_slice(&rest[..count]); + rest = &rest[count..]; + if page.segments.len() == 255 { + page.header[18..22].copy_from_slice(&sequence.to_le_bytes()); + write_page(&page, output, check)?; + sequence = sequence.wrapping_add(1); + page.segments.clear(); + page.data.clear(); + page.header[5] = u8::from(count == 255); + } + if count < 255 { + break; + } + } + } + if !page.segments.is_empty() { + page.header[18..22].copy_from_slice(&sequence.to_le_bytes()); + write_page(&page, output, check)?; + sequence = sequence.wrapping_add(1); + } + Ok(sequence) +} + +fn opus_gain(comments: &mut Vec<Vec<u8>>, fields: &Fields) -> Result<(), String> { + for (scope, upper) in [("track", "TRACK"), ("album", "ALBUM")] { + let Some(raw) = fields.get(&format!("replaygain_{scope}_gain")) else { + continue; + }; + let raw = raw.trim(); + let value = if raw.is_empty() { + String::new() + } else { + let db = raw.strip_suffix("dB").unwrap_or(raw).trim().parse::<f64>(); + let q = db.map(|db| ((db - 5.0) * 256.0).round()); + match q { + Ok(q) if q.is_finite() && (-32768.0..=32767.0).contains(&q) => { + (q as i32).to_string() + } + _ => return Err(format!("invalid Opus {scope} ReplayGain: {raw:?}")), + } + }; + set_comment(comments, &format!("R128_{upper}_GAIN"), &value); + set_comment(comments, &format!("REPLAYGAIN_{upper}_GAIN"), ""); + set_comment(comments, &format!("REPLAYGAIN_{upper}_PEAK"), ""); + } + Ok(()) +} diff --git a/rust_backend/crates/core/src/tags/write/riff.rs b/rust_backend/crates/core/src/tags/write/riff.rs new file mode 100644 index 00000000..35cbb5b3 --- /dev/null +++ b/rust_backend/crates/core/src/tags/write/riff.rs @@ -0,0 +1,140 @@ +use super::{Fields, Section, bytes, id3, metadata_fields, seek}; +use crate::tags::{file::ObservedReader, read_audio_tags}; +use std::io::{Read, Seek, SeekFrom}; + +pub(super) fn edit( + source: &mut (impl Read + Seek), + aiff: bool, + fields: &Fields, + cover: Option<&[u8]>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Vec<Section>, String> { + let mut observed = ObservedReader { + file: source, + check, + failure: None, + }; + let metadata = read_audio_tags(&mut observed, if aiff { "aiff" } else { "wav" }, &|| Ok(())) + .unwrap_or_default(); + if let Some(error) = observed.failure { + return Err(error); + } + let source = observed.file; + let length = seek(source, SeekFrom::End(0))?; + seek(source, SeekFrom::Start(0))?; + let header = bytes(source, 12)?; + if &header[..4] != if aiff { b"FORM" } else { b"RIFF" } { + return Err("unexpected container magic".into()); + } + let mut sections = Vec::new(); + let mut start = 12_u64; + let mut body_size = 4_u64; + let mut embedded_cover = None; + for _ in 0..65536 { + check()?; + if start + 8 > length { + break; + } + seek(source, SeekFrom::Start(start))?; + let header = bytes(source, 8)?; + let size = if aiff { + u32::from_be_bytes(header[4..].try_into().unwrap()) + } else { + u32::from_le_bytes(header[4..].try_into().unwrap()) + }; + let end = start + 8 + u64::from(size) + u64::from(size & 1); + if header[..4].eq_ignore_ascii_case(b"id3 ") { + if cover.is_none() + && size > 0 + && size <= 16 * 1024 * 1024 + && end <= length + && matches!(&header[..4], b"ID3 " | b"id3 ") + { + embedded_cover = id3::cover(&bytes(source, size as usize)?); + } + sections.push(Section { + start, + end: end.min(length), + data: Vec::new(), + }); + } else { + if end > length { + return Err("unexpected EOF while copying audio".into()); + } + body_size += end - start; + } + start = end; + } + if start + 8 <= length { + return Err("RIFF chunk count exceeds 65536".into()); + } + let cover_path = fields.get("cover_path").map(|p| p.trim()).unwrap_or(""); + if !cover_path.is_empty() && cover.is_none() { + return Err("read cover art: file not found".into()); + } + if cover.is_some_and(|cover| cover.is_empty()) { + return Err("cover art is empty".into()); + } + let cover = if let Some(cover) = cover { + let mime = if cover.len() >= 8 && cover.starts_with(b"\x89PNG") { + "image/png" + } else if cover.len() >= 12 && &cover[..4] == b"RIFF" && &cover[8..12] == b"WEBP" { + "image/webp" + } else if cover.starts_with(b"GIF87a") || cover.starts_with(b"GIF89a") { + "image/gif" + } else { + match std::path::Path::new(cover_path) + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("") + .to_ascii_lowercase() + .as_str() + { + "png" => "image/png", + "webp" => "image/webp", + "gif" => "image/gif", + _ => "image/jpeg", + } + }; + Some((cover, mime)) + } else { + embedded_cover + .as_ref() + .map(|(data, mime)| (data.as_slice(), mime.as_str())) + }; + let tag = id3::fresh(&metadata_fields(&metadata, fields), cover, check)?; + let tag_size = tag.len() as u32; + let mut chunk = if aiff { + b"ID3 ".to_vec() + } else { + b"id3 ".to_vec() + }; + chunk.extend(if aiff { + tag_size.to_be_bytes() + } else { + tag_size.to_le_bytes() + }); + chunk.extend(tag); + if tag_size & 1 == 1 { + chunk.push(0) + } + body_size += chunk.len() as u64; + let size = u32::try_from(body_size).map_err(|_| "RIFF container exceeds 32-bit size")?; + sections.push(Section { + start: 4, + end: 8, + data: if aiff { + size.to_be_bytes() + } else { + size.to_le_bytes() + } + .to_vec(), + }); + // Go drops an incomplete trailing chunk header, then appends the new ID3. + sections.push(Section { + start: start.min(length), + end: length, + data: chunk, + }); + Ok(sections) +} diff --git a/rust_backend/crates/core/src/text.rs b/rust_backend/crates/core/src/text.rs new file mode 100644 index 00000000..fc390b84 --- /dev/null +++ b/rust_backend/crates/core/src/text.rs @@ -0,0 +1,69 @@ +/// Go replaces every malformed UTF-8 byte separately when serializing JSON. +/// Rust's lossy decoder can combine a truncated sequence into one replacement. +pub(crate) fn utf8(mut bytes: &[u8]) -> String { + let mut text = String::new(); + while !bytes.is_empty() { + match std::str::from_utf8(bytes) { + Ok(valid) => { + text.push_str(valid); + break; + } + Err(error) => { + let valid = error.valid_up_to(); + text.push_str(std::str::from_utf8(&bytes[..valid]).expect("valid UTF-8 prefix")); + text.push(char::REPLACEMENT_CHARACTER); + bytes = &bytes[valid + 1..]; + } + } + } + text +} + +/// encoding/json replaces unpaired UTF-16 escapes, including in ignored fields. +pub fn json_surrogates(json: &str) -> std::borrow::Cow<'_, str> { + fn unit(bytes: &[u8]) -> Option<u16> { + let digits = bytes.get(..4)?; + if !digits.iter().all(u8::is_ascii_hexdigit) { + return None; + } + u16::from_str_radix(std::str::from_utf8(digits).ok()?, 16).ok() + } + let bytes = json.as_bytes(); + let (mut position, mut copied, mut quoted) = (0, 0, false); + let mut output = String::new(); + while position < bytes.len() { + if quoted && bytes[position] == b'\\' { + if bytes.get(position + 1) == Some(&b'u') + && let Some(code) = unit(&bytes[position + 2..]) + { + if (0xd800..=0xdbff).contains(&code) + && bytes.get(position + 6..position + 8) == Some(b"\\u") + && unit(&bytes[position + 8..]) + .is_some_and(|code| (0xdc00..=0xdfff).contains(&code)) + { + position += 12; + continue; + } + if (0xd800..=0xdfff).contains(&code) { + output.push_str(&json[copied..position]); + output.push_str("\\ufffd"); + copied = position + 6; + } + position += 6; + continue; + } + position += 2; + continue; + } + if bytes[position] == b'"' { + quoted = !quoted; + } + position += 1; + } + if copied == 0 { + std::borrow::Cow::Borrowed(json) + } else { + output.push_str(&json[copied..]); + std::borrow::Cow::Owned(output) + } +} diff --git a/rust_backend/crates/core/tests/optimization_tags.rs b/rust_backend/crates/core/tests/optimization_tags.rs new file mode 100644 index 00000000..efabfa07 --- /dev/null +++ b/rust_backend/crates/core/tests/optimization_tags.rs @@ -0,0 +1,343 @@ +use spotiflac_core::tags::{ + AudioMetadata, extract_cover, read_audio_tags, read_library_metadata, + read_library_metadata_with_cover, +}; +use std::cell::Cell; +use std::io::{self, Cursor, Read, Seek, SeekFrom}; + +struct CountingReader { + source: Cursor<Vec<u8>>, + read_calls: usize, + bytes_read: usize, + seek_calls: usize, +} + +impl CountingReader { + fn new(data: Vec<u8>) -> Self { + Self { + source: Cursor::new(data), + read_calls: 0, + bytes_read: 0, + seek_calls: 0, + } + } +} + +impl Read for CountingReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> { + self.read_calls += 1; + let count = self.source.read(buffer)?; + self.bytes_read += count; + Ok(count) + } +} + +impl Seek for CountingReader { + fn seek(&mut self, position: SeekFrom) -> io::Result<u64> { + self.seek_calls += 1; + self.source.seek(position) + } +} + +fn atom(kind: &[u8; 4], body: &[u8]) -> Vec<u8> { + let mut result = ((body.len() + 8) as u32).to_be_bytes().to_vec(); + result.extend(kind); + result.extend(body); + result +} + +fn data_atom(value: &[u8]) -> Vec<u8> { + let mut body = vec![0; 8]; + body.extend(value); + atom(b"data", &body) +} + +fn text_item(kind: &[u8; 4], value: &str) -> Vec<u8> { + atom(kind, &data_atom(value.as_bytes())) +} + +fn pair_item(kind: &[u8; 4], first: u16, second: u16) -> Vec<u8> { + let mut value = vec![0, 0]; + value.extend(first.to_be_bytes()); + value.extend(second.to_be_bytes()); + atom(kind, &data_atom(&value)) +} + +fn flag_item(kind: &[u8; 4]) -> Vec<u8> { + atom(kind, &data_atom(&[0, 0, 0, 1])) +} + +fn freeform_item() -> Vec<u8> { + let mut name = vec![0; 4]; + name.extend(b"ISRC"); + let mut body = atom(b"name", &name); + body.extend(data_atom(b"EXAMPLE123456")); + atom(b"----", &body) +} + +fn dense_m4a() -> Vec<u8> { + let mut ilst = Vec::new(); + for (kind, value) in [ + (*b"\xa9nam", "Dense title"), + (*b"\xa9ART", "Dense artist"), + (*b"\xa9alb", "Dense album"), + (*b"aART", "Dense album artist"), + (*b"\xa9day", "2026"), + (*b"\xa9gen", "Example"), + (*b"\xa9wrt", "Example composer"), + (*b"\xa9cmt", "Example comment"), + (*b"cprt", "Example copyright"), + (*b"\xa9lyr", "Example lyrics"), + ] { + ilst.extend(text_item(&kind, value)); + } + ilst.extend(pair_item(b"trkn", 3, 12)); + ilst.extend(pair_item(b"disk", 1, 2)); + ilst.extend(flag_item(b"rtng")); + ilst.extend(flag_item(b"cpil")); + ilst.extend(freeform_item()); + for index in 0..96_u8 { + ilst.extend(atom(b"free", &[index; 8])); + } + + let mut meta_body = vec![0; 4]; + meta_body.extend(atom(b"ilst", &ilst)); + let mut moov_body = atom(b"free", &[0; 16]); + moov_body.extend(atom(b"meta", &meta_body)); + let mut result = atom(b"ftyp", &[0; 24]); + result.extend(atom(b"moov", &moov_body)); + result.extend(atom(b"mdat", &vec![0; 4 << 20])); + result +} + +fn assert_dense_metadata(metadata: AudioMetadata) { + assert_eq!(metadata.title, "Dense title"); + assert_eq!(metadata.artist, "Dense artist"); + assert_eq!(metadata.album, "Dense album"); + assert_eq!(metadata.album_artist, "Dense album artist"); + assert_eq!(metadata.date, "2026"); + assert_eq!(metadata.year, "2026"); + assert_eq!(metadata.genre, "Example"); + assert_eq!(metadata.composer, "Example composer"); + assert_eq!(metadata.comment, "Example comment"); + assert_eq!(metadata.copyright, "Example copyright"); + assert_eq!(metadata.lyrics, "Example lyrics"); + assert_eq!((metadata.track_number, metadata.total_tracks), (3, 12)); + assert_eq!((metadata.disc_number, metadata.total_discs), (1, 2)); + assert_eq!(metadata.isrc, "EXAMPLE123456"); + assert_eq!(metadata.album_type, "compilation"); + assert!(metadata.explicit); +} + +#[test] +fn dense_mp4_reuses_buffered_reads_across_metadata_seeks() { + let mut reader = CountingReader::new(dense_m4a()); + assert_dense_metadata(read_audio_tags(&mut reader, "m4a", &|| Ok(())).unwrap()); + assert!( + reader.bytes_read < 64 << 10, + "{} bytes read", + reader.bytes_read + ); + assert!(reader.read_calls <= 8, "{} reads", reader.read_calls); + assert!(reader.seek_calls <= 8, "{} seeks", reader.seek_calls); +} + +#[test] +fn dense_mp4_supports_a_descriptor_position_and_rejects_malformed_eof_and_cancel() { + let data = dense_m4a(); + let mut descriptor = Cursor::new(data); + descriptor.set_position(descriptor.get_ref().len() as u64); + assert_dense_metadata(read_audio_tags(&mut descriptor, "m4a", &|| Ok(())).unwrap()); + + let mut malformed = CountingReader::new(b"not an mp4".to_vec()); + assert!(read_audio_tags(&mut malformed, "m4a", &|| Ok(())).is_err()); + + let mut truncated = dense_m4a(); + truncated.truncate(64); + let mut eof = CountingReader::new(truncated); + let error = read_audio_tags(&mut eof, "m4a", &|| Ok(())).unwrap_err(); + assert!( + error.contains("EOF") || error.contains("not found"), + "{error}" + ); + assert!(eof.bytes_read < 64 << 10, "{} bytes read", eof.bytes_read); + + let checks = Cell::new(0); + let mut canceled = CountingReader::new(dense_m4a()); + let error = read_audio_tags(&mut canceled, "m4a", &|| { + let count = checks.get(); + checks.set(count + 1); + if count >= 3 { + Err("cancelled".into()) + } else { + Ok(()) + } + }) + .unwrap_err(); + assert_eq!(error, "cancelled"); + assert!(checks.get() > 3); +} + +fn flac_block(kind: u8, last: bool, data: &[u8]) -> Vec<u8> { + let mut result = vec![kind | (if last { 0x80 } else { 0 })]; + let length = (data.len() as u32).to_be_bytes(); + result.extend_from_slice(&length[1..]); + result.extend_from_slice(data); + result +} + +fn flac_streaminfo() -> Vec<u8> { + let mut info = vec![0; 34]; + info[..2].copy_from_slice(&4096_u16.to_be_bytes()); + info[2..4].copy_from_slice(&4096_u16.to_be_bytes()); + let packed = (44_100_u64 << 44) | (1_u64 << 41) | (15_u64 << 36) | 44_100; + info[10..18].copy_from_slice(&packed.to_be_bytes()); + info +} + +fn flac_comments(items: &[&str]) -> Vec<u8> { + let mut result = 0_u32.to_le_bytes().to_vec(); + result.extend_from_slice(&(items.len() as u32).to_le_bytes()); + for item in items { + result.extend_from_slice(&(item.len() as u32).to_le_bytes()); + result.extend_from_slice(item.as_bytes()); + } + result +} + +fn flac_picture(kind: u32, mime: &[u8], data: &[u8]) -> Vec<u8> { + let mut result = kind.to_be_bytes().to_vec(); + result.extend_from_slice(&(mime.len() as u32).to_be_bytes()); + result.extend_from_slice(mime); + result.extend_from_slice(&0_u32.to_be_bytes()); + result.extend_from_slice(&[0; 16]); + result.extend_from_slice(&(data.len() as u32).to_be_bytes()); + result.extend_from_slice(data); + result +} + +fn flac_file(pictures: &[Vec<u8>]) -> Vec<u8> { + let comments = flac_comments(&[ + "TITLE=Dense FLAC", + "ARTIST=Example Artist", + "ALBUM=Example Album", + "ALBUMARTIST=Example Album Artist", + "DATE=2026", + "TRACKNUMBER=2/9", + "DISCNUMBER=1/2", + "GENRE=Example", + "ISRC=EXAMPLE123456", + "RELEASETYPE=album", + ]); + let mut result = b"fLaC".to_vec(); + result.extend(flac_block(0, false, &flac_streaminfo())); + result.extend(flac_block(4, false, &comments)); + for picture in pictures { + result.extend(flac_block(6, false, picture)); + } + result.extend(flac_block(1, true, &[0; 8])); + result.extend_from_slice(&[0xff, 0xf8]); + result.extend_from_slice(&[0; 32]); + result +} + +fn valid_flac() -> Vec<u8> { + flac_file(&[ + flac_picture(2, b"image/jpeg", b"\xff\xd8secondary"), + flac_picture(3, b"image/png", b"\x89PNG\r\nfront-cover"), + ]) +} + +#[test] +fn flac_combined_metadata_and_cover_match_separate_reads_with_less_io() { + let data = valid_flac(); + let path = "/music/Example Album/Example Artist - Dense FLAC.flac"; + + let mut metadata_reader = CountingReader::new(data.clone()); + let expected_metadata = read_library_metadata( + &mut metadata_reader, + path, + "", + "2026-09-14T00:00:00Z", + 42, + &|| Ok(()), + ) + .unwrap(); + let mut cover_reader = CountingReader::new(data.clone()); + let expected_cover = extract_cover(&mut cover_reader, "flac", &|| Ok(())).unwrap(); + + let mut combined_reader = CountingReader::new(data); + let (actual_metadata, actual_cover) = read_library_metadata_with_cover( + &mut combined_reader, + path, + "", + "2026-09-14T00:00:00Z", + 42, + &|| Ok(()), + ) + .unwrap(); + + assert_eq!(actual_metadata, expected_metadata); + assert_eq!(actual_metadata["trackName"], "Dense FLAC"); + assert_eq!(actual_metadata["artistName"], "Example Artist"); + assert_eq!(actual_metadata["bitDepth"], 16); + assert_eq!(actual_metadata["sampleRate"], 44_100); + assert_eq!(actual_metadata["duration"], 1); + let actual_cover = actual_cover.expect("front cover"); + assert_eq!(actual_cover.data.as_slice(), expected_cover.data.as_slice()); + assert_eq!(actual_cover.mime, expected_cover.mime); + assert_eq!(actual_cover.data, b"\x89PNG\r\nfront-cover"); + assert_eq!(actual_cover.mime, "image/png"); + + let separate_bytes = metadata_reader.bytes_read + cover_reader.bytes_read; + assert!( + combined_reader.bytes_read < separate_bytes, + "combined {} vs separate {} bytes", + combined_reader.bytes_read, + separate_bytes + ); + let separate_reads = metadata_reader.read_calls + cover_reader.read_calls; + assert!( + combined_reader.read_calls < separate_reads, + "combined {} vs separate {} reads", + combined_reader.read_calls, + separate_reads + ); +} + +#[test] +fn flac_combined_handles_bad_cover_metadata_eof_and_cancel() { + let path = "Dense FLAC.flac"; + let mut bad_cover = Cursor::new(flac_file(&[vec![0; 4]])); + let (metadata, cover) = + read_library_metadata_with_cover(&mut bad_cover, path, "", "scan", 0, &|| Ok(())).unwrap(); + assert_eq!(metadata["trackName"], "Dense FLAC"); + assert_eq!(metadata["bitDepth"], 16); + assert!(cover.is_none()); + + let mut truncated = valid_flac(); + truncated.truncate(12); + let (metadata, cover) = + read_library_metadata_with_cover(&mut Cursor::new(truncated), path, "", "scan", 0, &|| { + Ok(()) + }) + .unwrap(); + assert_eq!(metadata["metadataFromFilename"], true); + assert!(metadata.get("bitDepth").is_none()); + assert!(cover.is_none()); + + let checks = Cell::new(0); + let mut canceled = Cursor::new(valid_flac()); + let error = read_library_metadata_with_cover(&mut canceled, path, "", "scan", 0, &|| { + let count = checks.get(); + checks.set(count + 1); + if count >= 4 { + Err("cancelled".into()) + } else { + Ok(()) + } + }) + .unwrap_err(); + assert_eq!(error, "cancelled"); + assert!(checks.get() > 4); +} diff --git a/rust_backend/crates/extensions/Cargo.toml b/rust_backend/crates/extensions/Cargo.toml new file mode 100644 index 00000000..cfdad077 --- /dev/null +++ b/rust_backend/crates/extensions/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "spotiflac-extensions" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +spotiflac-core.workspace = true +spotiflac-network.workspace = true +spotiflac-providers.workspace = true +tokio.workspace = true +rquickjs.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +base64.workspace = true +sha2.workspace = true +sha1.workspace = true +md-5.workspace = true +hmac.workspace = true +aes-gcm.workspace = true +aes.workspace = true +blowfish.workspace = true +zeroize.workspace = true +tempfile.workspace = true +zip.workspace = true +regex.workspace = true +time.workspace = true +chrono.workspace = true +httpdate.workspace = true +cap-std.workspace = true +rustix.workspace = true + +[dev-dependencies] +rustls.workspace = true +rcgen = { version = "=0.14.10", default-features = false, features = ["crypto", "ring", "pem"] } + +[lints] +workspace = true diff --git a/rust_backend/crates/extensions/src/auth.rs b/rust_backend/crates/extensions/src/auth.rs new file mode 100644 index 00000000..0368e700 --- /dev/null +++ b/rust_backend/crates/extensions/src/auth.rs @@ -0,0 +1,354 @@ +//! Shared OAuth state and one-time callback ownership for managed runtimes. + +use aes_gcm::aead::{OsRng, rand_core::RngCore}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use serde::Serialize; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::{SystemTime, UNIX_EPOCH}; +use zeroize::Zeroizing; + +mod client; +pub use client::ExtensionAuth; + +const SECOND: i128 = 1_000_000_000; +const PENDING_TTL: i128 = 180 * SECOND; + +pub trait AuthClock: Send + Sync { + fn now_nanos(&self) -> i128; +} + +struct SystemClock; +impl AuthClock for SystemClock { + fn now_nanos(&self) -> i128 { + SystemTime::now().duration_since(UNIX_EPOCH).map_or_else( + |error| -(error.duration().as_nanos() as i128), + |time| time.as_nanos() as i128, + ) + } +} + +#[derive(Clone, Serialize)] +pub struct PendingAuthRequest { + pub extension_id: String, + pub auth_url: String, + pub callback_url: String, + #[serde(skip)] + pub state: String, + #[serde(skip)] + pub created_at: i128, +} + +#[derive(Default)] +struct Record { + code: Zeroizing<String>, + access_token: Zeroizing<String>, + refresh_token: Zeroizing<String>, + verifier: Zeroizing<String>, + challenge: String, + authenticated: bool, + expires_at: Option<i128>, +} + +#[derive(Default)] +struct State { + records: BTreeMap<String, Record>, + generations: BTreeMap<String, u64>, + pending: BTreeMap<String, PendingAuthRequest>, + owners: BTreeMap<String, String>, + closed: bool, +} + +pub struct AuthRegistry { + state: Mutex<State>, + clock: Arc<dyn AuthClock>, +} + +impl Default for AuthRegistry { + fn default() -> Self { + Self::with_clock(Arc::new(SystemClock)) + } +} + +impl AuthRegistry { + pub fn with_clock(clock: Arc<dyn AuthClock>) -> Self { + Self { + state: Mutex::default(), + clock, + } + } + + pub fn now(&self) -> i128 { + self.clock.now_nanos() + } + + fn edit<T>(&self, id: &str, edit: impl FnOnce(&mut Record) -> T) -> Result<T, String> { + let mut state = self.state.lock().expect("auth registry lock"); + if state.closed { + return Err("extension environment closed".into()); + } + let generation = state.generations.entry(id.to_owned()).or_default(); + *generation = generation.wrapping_add(1); + Ok(edit(state.records.entry(id.to_owned()).or_default())) + } + + pub fn set_code(&self, id: &str, code: &str) -> Result<(), String> { + self.edit(id, |record| record.code = Zeroizing::new(code.to_owned())) + } + + pub fn set_tokens( + &self, + id: &str, + access: &str, + refresh: &str, + expires_in: i64, + ) -> Result<(), String> { + let expires = (expires_in > 0) + .then(|| self.now() + i128::from(expires_in.wrapping_mul(1_000_000_000))); + self.edit(id, |record| { + record.access_token = Zeroizing::new(access.to_owned()); + record.refresh_token = Zeroizing::new(refresh.to_owned()); + record.authenticated = !access.is_empty(); + record.expires_at = expires; + }) + } + + pub fn code(&self, id: &str) -> Option<String> { + self.state + .lock() + .expect("auth registry lock") + .records + .get(id) + .filter(|record| !record.code.is_empty()) + .map(|record| record.code.to_string()) + } + + pub fn authenticated(&self, id: &str) -> bool { + let now = self.now(); + self.state + .lock() + .expect("auth registry lock") + .records + .get(id) + .is_some_and(|record| { + record.authenticated && record.expires_at.is_none_or(|expires| now <= expires) + }) + } + + pub fn tokens(&self, id: &str) -> Value { + let state = self.state.lock().expect("auth registry lock"); + let Some(record) = state.records.get(id) else { + return json!({}); + }; + let mut result = json!({"access_token":record.access_token.as_str(),"refresh_token":record.refresh_token.as_str(),"is_authenticated":record.authenticated}); + if let Some(expires) = record.expires_at { + result["expires_at"] = json!(expires.div_euclid(SECOND) as i64); + result["is_expired"] = json!(self.now() > expires); + } + result + } + + pub fn clear(&self, id: &str) { + let mut state = self.state.lock().expect("auth registry lock"); + if state.closed { + return; + } + state.records.remove(id); + let generation = state.generations.entry(id.to_owned()).or_default(); + *generation = generation.wrapping_add(1); + Self::remove_pending(&mut state, id, self.now()); + } + + pub fn register_pending(&self, mut request: PendingAuthRequest) -> Result<(), String> { + if request.extension_id.trim().is_empty() { + return Err("extension id is required".into()); + } + if request.state.is_empty() { + request.state = callback_state()?; + } + if request.created_at == 0 { + request.created_at = self.now(); + } + let mut state = self.state.lock().expect("auth registry lock"); + if state.closed { + return Err("extension environment closed".into()); + } + if let Some(owner) = state + .owners + .get(&request.state) + .filter(|owner| **owner != request.extension_id) + { + let same = state.pending.get(owner).is_some_and(|previous| { + previous.state == request.state + && previous.auth_url == request.auth_url + && previous.callback_url == request.callback_url + && previous.created_at == request.created_at + }); + if !same { + return Err("callback state is already registered".into()); + } + } + if state + .pending + .get(&request.extension_id) + .is_some_and(|previous| previous.state != request.state) + { + Self::remove_pending(&mut state, &request.extension_id, self.now()); + } + state + .owners + .entry(request.state.clone()) + .or_insert_with(|| request.extension_id.clone()); + state.pending.insert(request.extension_id.clone(), request); + Ok(()) + } + + fn remove_pending(state: &mut State, id: &str, now: i128) { + let Some(request) = state.pending.remove(id) else { + return; + }; + if state + .owners + .get(&request.state) + .is_none_or(|owner| owner != id) + { + return; + } + state.owners.remove(&request.state); + if let Some((id, _)) = state.pending.iter().find(|(_, candidate)| { + candidate.state == request.state && now - candidate.created_at < PENDING_TTL + }) { + state.owners.insert(request.state, id.clone()); + } + } + + pub fn clear_pending(&self, id: &str) { + Self::remove_pending( + &mut self.state.lock().expect("auth registry lock"), + id, + self.now(), + ); + } + + pub fn pending(&self, id: &str) -> Option<PendingAuthRequest> { + let mut state = self.state.lock().expect("auth registry lock"); + if state + .pending + .get(id) + .is_some_and(|request| self.now() - request.created_at >= PENDING_TTL) + { + Self::remove_pending(&mut state, id, self.now()); + } + state.pending.get(id).cloned() + } + + pub(crate) fn has_fresh_challenge(&self, id: &str) -> bool { + self.pending(id).is_some_and(|request| { + request.extension_id == id + && !request.auth_url.trim().is_empty() + && (0..PENDING_TTL).contains(&(self.now() - request.created_at)) + }) + } + + pub fn all_pending(&self) -> Vec<PendingAuthRequest> { + self.state + .lock() + .expect("auth registry lock") + .pending + .values() + .cloned() + .collect() + } + + pub fn resolve_callback(&self, nonce: &str, consume: bool) -> Result<String, String> { + let nonce = nonce.trim(); + if nonce.is_empty() { + return Err("callback state is required".into()); + } + let mut state = self.state.lock().expect("auth registry lock"); + let owner = state.owners.get(nonce).cloned(); + let valid = owner + .as_ref() + .and_then(|owner| state.pending.get(owner)) + .is_some_and(|request| { + request.state == nonce && self.now() - request.created_at < PENDING_TTL + }); + if !valid || consume { + state.owners.remove(nonce); + state.pending.retain(|_, request| request.state != nonce); + } + if valid { + Ok(owner.expect("validated callback owner")) + } else { + Err("callback state is invalid, expired, or already used".into()) + } + } + + pub fn shutdown(&self) { + let mut state = self.state.lock().expect("auth registry lock"); + *state = State { + closed: true, + ..State::default() + }; + } +} + +pub fn callback_state() -> Result<String, String> { + let mut random = Zeroizing::new([0_u8; 32]); + OsRng + .try_fill_bytes(random.as_mut()) + .map_err(|error| format!("generate callback state: {error}"))?; + Ok(URL_SAFE_NO_PAD.encode(random.as_ref())) +} + +pub fn pkce_verifier(length: usize) -> Result<String, String> { + let length = length.clamp(43, 128); + let mut random = Zeroizing::new(vec![0; length]); + OsRng + .try_fill_bytes(&mut random) + .map_err(|error| error.to_string())?; + let mut verifier = URL_SAFE_NO_PAD.encode(random.as_slice()); + verifier.truncate(length); + Ok(verifier) +} + +pub fn pkce_challenge(verifier: &str) -> String { + URL_SAFE_NO_PAD.encode(Sha256::digest(verifier)) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Clock; + impl AuthClock for Clock { + fn now_nanos(&self) -> i128 { + 1000 * SECOND + } + } + + #[test] + fn verification_requires_own_nonempty_challenge_between_zero_and_180_seconds_old() { + for (age, url, expected) in [ + (-1, "https://example.test/verify", false), + (0, "https://example.test/verify", true), + (179, "https://example.test/verify", true), + (180, "https://example.test/verify", false), + (0, " \n", false), + ] { + let auth = AuthRegistry::with_clock(Arc::new(Clock)); + auth.register_pending(PendingAuthRequest { + extension_id: "example.auth".into(), + auth_url: url.into(), + callback_url: "spotiflac://callback".into(), + state: "state".into(), + created_at: (1000 - age) * SECOND, + }) + .unwrap(); + assert_eq!(auth.has_fresh_challenge("example.auth"), expected); + assert!(!auth.has_fresh_challenge("example.other")); + } + } +} diff --git a/rust_backend/crates/extensions/src/auth/client.rs b/rust_backend/crates/extensions/src/auth/client.rs new file mode 100644 index 00000000..7f82aece --- /dev/null +++ b/rust_backend/crates/extensions/src/auth/client.rs @@ -0,0 +1,324 @@ +use super::{AuthRegistry, PendingAuthRequest, callback_state, pkce_challenge, pkce_verifier}; +use serde_json::{Map, Value, json}; +use spotiflac_core::app_version::AppVersion; +use spotiflac_network::{HttpRequest, NetworkSession, query}; +use std::collections::BTreeMap; +use std::sync::Arc; +use zeroize::Zeroizing; + +pub struct ExtensionAuth { + id: String, + registry: Arc<AuthRegistry>, + network: Arc<NetworkSession>, + app_version: AppVersion, +} + +impl ExtensionAuth { + pub fn new( + id: &str, + registry: Arc<AuthRegistry>, + network: Arc<NetworkSession>, + app_version: impl Into<AppVersion>, + ) -> Self { + Self { + id: id.to_owned(), + registry, + network, + app_version: app_version.into(), + } + } + + pub fn call( + &self, + method: &str, + arguments: &[Value], + expires_is_float: bool, + check: impl Fn() -> Result<(), String>, + ) -> Result<Value, String> { + check()?; + let first = arguments.first().unwrap_or(&Value::Null); + match method { + "getAuthCode" => Ok(self + .registry + .code(&self.id) + .map_or(Value::Null, Value::String)), + "getTokens" => Ok(self.registry.tokens(&self.id)), + "isAuthenticated" => Ok(json!(self.registry.authenticated(&self.id))), + "clearAuth" => { + self.registry.clear(&self.id); + Ok(json!(true)) + } + "setAuthCode" => { + if arguments.is_empty() { + return Ok(json!(false)); + } + self.registry.edit(&self.id, |record| { + if let Some(code) = first.as_str() { + record.code = Zeroizing::new(code.to_owned()); + } else if let Some(value) = first.as_object() { + if let Some(code) = value.get("code").and_then(Value::as_str) { + record.code = Zeroizing::new(code.to_owned()); + } + if let Some(access) = value.get("access_token").and_then(Value::as_str) { + record.access_token = Zeroizing::new(access.to_owned()); + record.authenticated = true; + } + if let Some(refresh) = value.get("refresh_token").and_then(Value::as_str) { + record.refresh_token = Zeroizing::new(refresh.to_owned()); + } + if expires_is_float + && let Some(expires) = value.get("expires_in").and_then(Value::as_f64) + { + record.expires_at = Some( + self.registry.now() + + i128::from((expires as i64).wrapping_mul(1_000_000_000)), + ); + } + } + })?; + Ok(json!(true)) + } + "generatePKCE" => { + let length = first.as_u64().unwrap_or(64) as usize; + let verifier = pkce_verifier(length)?; + self.store_pkce(&verifier, false)?; + Ok( + json!({"verifier":verifier,"challenge":pkce_challenge(&verifier),"method":"S256"}), + ) + } + "getPKCE" => { + let state = self.registry.state.lock().expect("auth registry lock"); + Ok(state.records.get(&self.id).filter(|record| !record.verifier.is_empty()) + .map_or_else(|| json!({}), |record| json!({"verifier":record.verifier.as_str(),"challenge":record.challenge,"method":"S256"}))) + } + "openAuthUrl" => { + if arguments.is_empty() { + return Err("auth URL is required".into()); + } + let mut url = self + .network + .validate_auth_url(first.as_str().unwrap_or(""), &check)?; + let nonce = callback_state()?; + let mut params = query::parse(&url.raw_query); + query::set(&mut params, "state", &nonce); + url.raw_query = query::encode(¶ms); + self.registry.register_pending(PendingAuthRequest { + extension_id: self.id.clone(), + auth_url: url.display_url(), + callback_url: arguments + .get(1) + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(), + state: nonce, + created_at: self.registry.now(), + })?; + self.registry.set_code(&self.id, "")?; + Ok(json!({"success":true,"message":"Auth URL will be opened by the app"})) + } + "startOAuthWithPKCE" | "exchangeCodeWithPKCE" => { + if arguments.is_empty() { + return Err("config object is required".into()); + } + let config = first + .as_object() + .ok_or_else(|| "config must be an object".to_owned())?; + if method == "startOAuthWithPKCE" { + self.start(config, &check) + } else { + self.exchange(config, &check) + } + } + _ => Err("unknown auth method".into()), + } + } + + fn store_pkce(&self, verifier: &str, clear_code: bool) -> Result<(), String> { + self.registry.edit(&self.id, |record| { + record.verifier = Zeroizing::new(verifier.to_owned()); + record.challenge = pkce_challenge(verifier); + if clear_code { + record.code = Zeroizing::default(); + } + }) + } + + fn start( + &self, + config: &Map<String, Value>, + check: &impl Fn() -> Result<(), String>, + ) -> Result<Value, String> { + let auth_url = string(config, "authUrl"); + let client = string(config, "clientId"); + let redirect = string(config, "redirectUri"); + if auth_url.is_empty() || client.is_empty() || redirect.is_empty() { + return Err("authUrl, clientId, and redirectUri are required".into()); + } + let mut url = self.network.validate_auth_url(auth_url, check)?; + let verifier = Zeroizing::new( + pkce_verifier(64).map_err(|error| format!("failed to generate PKCE: {error}"))?, + ); + let challenge = pkce_challenge(&verifier); + self.store_pkce(&verifier, true)?; + let mut params = query::parse(&url.raw_query); + for (key, value) in [ + ("client_id", client), + ("redirect_uri", redirect), + ("response_type", "code"), + ("code_challenge", &challenge), + ("code_challenge_method", "S256"), + ] { + query::set(&mut params, key, value); + } + let scope = string(config, "scope"); + if !scope.is_empty() { + query::set(&mut params, "scope", scope); + } + extra_params(config, &mut params); + let nonce = + callback_state().map_err(|error| format!("failed to generate OAuth state: {error}"))?; + query::set(&mut params, "state", &nonce); + url.raw_query = query::encode(¶ms); + let auth_url = url.display_url(); + self.registry + .register_pending(PendingAuthRequest { + extension_id: self.id.clone(), + auth_url: auth_url.clone(), + callback_url: redirect.to_owned(), + state: nonce, + created_at: self.registry.now(), + }) + .map_err(|error| format!("failed to register OAuth callback: {error}"))?; + Ok( + json!({"success":true,"authUrl":auth_url,"pkce":{"verifier":verifier.as_str(),"challenge":challenge,"method":"S256"}}), + ) + } + + fn exchange( + &self, + config: &Map<String, Value>, + check: &impl Fn() -> Result<(), String>, + ) -> Result<Value, String> { + let token_url = string(config, "tokenUrl"); + let client = string(config, "clientId"); + let code = string(config, "code"); + if token_url.is_empty() || client.is_empty() || code.is_empty() { + return Err("tokenUrl, clientId, and code are required".into()); + } + let (verifier, generation) = { + let state = self.registry.state.lock().expect("auth registry lock"); + let verifier = state + .records + .get(&self.id) + .map(|record| record.verifier.clone()) + .unwrap_or_default(); + ( + verifier, + state.generations.get(&self.id).copied().unwrap_or(0), + ) + }; + if verifier.is_empty() { + return Err( + "no PKCE verifier found - call generatePKCE or startOAuthWithPKCE first".into(), + ); + } + self.network.validate_url(token_url)?; + let mut params = query::Query::new(); + for (key, value) in [ + ("grant_type", "authorization_code"), + ("client_id", client), + ("code", code), + ("code_verifier", &verifier), + ] { + query::set(&mut params, key, value); + } + let redirect = string(config, "redirectUri"); + if !redirect.is_empty() { + query::set(&mut params, "redirect_uri", redirect); + } + extra_params(config, &mut params); + let response = self.network.request( + HttpRequest { + url: token_url.to_owned(), + method: "POST".into(), + body: query::encode(¶ms), + headers: BTreeMap::from([( + "Content-Type".into(), + "application/x-www-form-urlencoded".into(), + )]), + default_json: false, + user_agent: self.app_version.user_agent(), + }, + check, + )?; + let body = Zeroizing::new(response.body); + let token = match serde_json::from_slice::<Value>(&body) { + Ok(Value::Object(token)) => token, + Ok(Value::Null) => Map::new(), + result => { + let message = match result { + Err(error) => error.to_string(), + _ => "token response must be an object".into(), + }; + return Ok( + json!({"success":false,"error":format!("failed to parse token response: {message}"),"body":crate::redact::preview(&body, 1000)}), + ); + } + }; + if let Some(error) = token.get("error").and_then(Value::as_str) { + return Ok( + json!({"success":false,"error":error,"error_description":string(&token,"error_description")}), + ); + } + let access = string(&token, "access_token"); + let refresh = string(&token, "refresh_token"); + let expires = token + .get("expires_in") + .and_then(Value::as_f64) + .unwrap_or(0.0); + if access.is_empty() { + return Ok( + json!({"success":false,"error":"no access_token in response","body":crate::redact::preview(&body, 1000)}), + ); + } + check()?; + let mut state = self.registry.state.lock().expect("auth registry lock"); + if state.closed || state.generations.get(&self.id).copied().unwrap_or(0) != generation { + return Err("authentication state changed during token exchange".into()); + } + let record = state.records.entry(self.id.clone()).or_default(); + record.access_token = Zeroizing::new(access.to_owned()); + record.refresh_token = Zeroizing::new(refresh.to_owned()); + record.authenticated = true; + if expires > 0.0 { + record.expires_at = Some( + self.registry.now() + i128::from((expires as i64).wrapping_mul(1_000_000_000)), + ); + } + record.verifier = Zeroizing::default(); + record.challenge.clear(); + let generation = state.generations.entry(self.id.clone()).or_default(); + *generation = generation.wrapping_add(1); + let mut result = json!({"success":true,"access_token":access,"refresh_token":refresh,"token_type":token.get("token_type")}); + if expires > 0.0 { + result["expires_in"] = json!(expires); + } + if let Some(scope) = token.get("scope").and_then(Value::as_str) { + result["scope"] = json!(scope); + } + Ok(result) + } +} + +fn string<'a>(config: &'a Map<String, Value>, key: &str) -> &'a str { + config.get(key).and_then(Value::as_str).unwrap_or("") +} + +// The JS bridge exports extraParams with Go's fmt formatting, rather than JSON. +fn extra_params(config: &Map<String, Value>, params: &mut query::Query) { + if let Some(extras) = config.get("extraParams").and_then(Value::as_object) { + for (key, value) in extras { + query::set(params, key, value.as_str().unwrap_or("")); + } + } +} diff --git a/rust_backend/crates/extensions/src/backend.rs b/rust_backend/crates/extensions/src/backend.rs new file mode 100644 index 00000000..565fd685 --- /dev/null +++ b/rust_backend/crates/extensions/src/backend.rs @@ -0,0 +1,330 @@ +//! Root ownership for installed extensions and shared provider services. + +mod cover; +mod downloads; +mod health; +mod library; +mod lyrics; +mod metadata; +mod musicbrainz; +mod provider_metadata; +mod reenrich; +mod share; +mod tags; + +pub use lyrics::LyricsRequest; +pub use metadata::MetadataOptions; + +use crate::RuntimeLimits; +use crate::environment::ExtensionEnvironment; +use crate::lyrics::InstalledLyricsFetcher; +use crate::manager::{ExtensionManager, ManagerError}; +use spotiflac_core::lyrics::config::{self, FetchOptions}; +use spotiflac_providers::deezer::DeezerClient; +use spotiflac_providers::lyrics::{LyricsService, builtin::BuiltinLyricsClient}; +use spotiflac_providers::musicbrainz::MusicBrainzClient; +use spotiflac_providers::resolver::{PlatformResolverChain, availability::AvailabilityService}; +use std::ops::Deref; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; + +/// Retained by the native owner, never by a VM or provider worker. Shutdown +/// finishes provider work and persistence before releasing directory locks. +pub struct Backend { + manager: Arc<ExtensionManager>, + lyrics: Arc<LyricsService>, + availability: Arc<AvailabilityService>, + deezer: Arc<DeezerClient>, + musicbrainz: MusicBrainzClient, + share_cache: Mutex<share::Cache>, + cover: cover::Cover, + library_cover_directory: Mutex<String>, + library_scan: library::ScanState, + prepared_downloads: Mutex<downloads::PreparedDownloads>, + health: Arc<health::Health>, + closed: AtomicBool, + operations: Mutex<usize>, + idle: Condvar, + lyrics_settings: Mutex<()>, + shutdown: Mutex<Option<Result<(), String>>>, +} + +impl Backend { + pub fn new( + sources: &Path, + data: &Path, + master_key: &str, + app_version: &str, + limits: RuntimeLimits, + ) -> Result<Self, ManagerError> { + let environment = ExtensionEnvironment::new(data, master_key, app_version) + .map_err(|error| ManagerError(error.to_string()))?; + Self::with_environment(sources, environment, limits, MetadataOptions::default()) + } + + /// Apply persisted settings before restoring caches that depend on them. + pub fn with_lyrics_settings( + sources: &Path, + data: &Path, + master_key: &str, + app_version: &str, + limits: RuntimeLimits, + providers_json: &str, + options_json: &str, + ) -> Result<Self, ManagerError> { + let providers = config::decode_providers(providers_json) + .map_err(|error| ManagerError(error.to_string()))?; + let mut options = FetchOptions::default(); + options + .update_json(options_json) + .map_err(|error| ManagerError(error.to_string()))?; + let environment = ExtensionEnvironment::new(data, master_key, app_version) + .map_err(|error| ManagerError(error.to_string()))?; + Self::with_environment_and_lyrics( + sources, + environment, + limits, + MetadataOptions::default(), + &providers, + options, + ) + } + + /// Transfer a native environment into this root. Provider configuration is + /// native-owned and all services reuse the environment's network/policy. + pub fn with_environment( + sources: &Path, + environment: ExtensionEnvironment, + limits: RuntimeLimits, + options: MetadataOptions, + ) -> Result<Self, ManagerError> { + Self::with_environment_and_lyrics( + sources, + environment, + limits, + options, + &[], + FetchOptions::default(), + ) + } + + fn with_environment_and_lyrics( + sources: &Path, + environment: ExtensionEnvironment, + limits: RuntimeLimits, + options: MetadataOptions, + lyrics_providers: &[String], + lyrics_options: FetchOptions, + ) -> Result<Self, ManagerError> { + let manager = Arc::new(ExtensionManager::with_environment( + sources, + environment, + limits, + )?); + let environment = manager.environment(); + let network = environment + .network_service() + .map_err(|error| ManagerError(error.to_string()))?; + let deezer = Arc::new( + DeezerClient::with_endpoint(&network, &options.deezer_endpoint) + .map_err(|error| ManagerError(error.to_string()))?, + ); + let musicbrainz = MusicBrainzClient::with_options(&network, options.musicbrainz) + .map_err(|error| ManagerError(error.to_string()))?; + let resolver = PlatformResolverChain::with_options(&network, options.resolver) + .map_err(|error| ManagerError(error.to_string()))?; + let availability = Arc::new(AvailabilityService::new(Arc::new(resolver), deezer.clone())); + let lyrics = Arc::new(LyricsService::new(Arc::new(InstalledLyricsFetcher::new( + &manager, + BuiltinLyricsClient::new( + &network, + environment.shared_app_version(), + availability.clone(), + ), + )))); + lyrics + .set_providers(lyrics_providers) + .and_then(|()| lyrics.set_options(lyrics_options)) + .map_err(|error| ManagerError(error.to_string()))?; + lyrics + .set_persistence_path(&environment.data_directory().join(".lyrics_cache.json")) + .map_err(|error| ManagerError(error.to_string()))?; + environment.attach_lyrics(&lyrics); + let cover = cover::Cover::new(&network, environment.shared_app_version()); + let health = Arc::new(health::Health::new( + &network, + environment.shared_app_version(), + )); + Ok(Self { + manager, + lyrics, + availability, + deezer, + musicbrainz, + share_cache: Mutex::default(), + cover, + library_cover_directory: Mutex::default(), + library_scan: library::ScanState::default(), + prepared_downloads: Mutex::default(), + health, + closed: AtomicBool::new(false), + operations: Mutex::new(0), + idle: Condvar::new(), + lyrics_settings: Mutex::new(()), + shutdown: Mutex::new(None), + }) + } + + pub fn lyrics(&self) -> Arc<LyricsService> { + Arc::clone(&self.lyrics) + } + + pub fn availability(&self) -> Arc<AvailabilityService> { + Arc::clone(&self.availability) + } + + pub fn get_app_version(&self) -> Result<String, String> { + let _operation = self.enter()?; + self.manager + .environment() + .get_app_version() + .map_err(|error| error.to_string()) + } + + pub fn set_app_version(&self, version: &str) -> Result<(), String> { + let _operation = self.enter()?; + self.manager + .environment() + .set_app_version(version) + .map_err(|error| error.to_string()) + } + + pub fn get_extension_pending_auth_json(&self, id: &str) -> Result<String, String> { + let _operation = self.enter()?; + let id = id.trim(); + if id.is_empty() { + return Ok(String::new()); + } + let environment = self.manager.environment(); + if let Some(pending) = environment + .pending_auth(id) + .map_err(|error| error.to_string())? + { + return serde_json::to_string(&pending).map_err(|error| error.to_string()); + } + // A missing or expired challenge may require network preflight. Reuse + // cancellable provider work so owner shutdown interrupts that request. + self.metadata_provider_work(&|| self.check(), |lease| { + self.manager + .preflight_auth(id, lease) + .map(|_| "null".into()) + }) + .map_err(|error| error.to_string())?; + environment + .pending_auth(id) + .map_err(|error| error.to_string())? + .map(|pending| serde_json::to_string(&pending).map_err(|error| error.to_string())) + .transpose() + .map(Option::unwrap_or_default) + } + + /// Drop disposable resources while active requests retain their owner and IO. + /// Memory pressure does not clear persisted lyrics, settings or credentials. + pub fn release_memory(&self, under_pressure: bool) -> Result<(), String> { + let _operation = self.enter()?; + self.manager + .release_idle_download_runtimes() + .map_err(|error| error.to_string())?; + self.manager + .environment() + .cleanup_connections() + .map_err(|error| error.to_string())?; + if under_pressure { + self.cover.clear(); + self.lyrics + .drop_memory() + .map_err(|error| error.to_string())?; + self.health.clear_memory_cache(); + } + Ok(()) + } + + pub fn shutdown_checked(&self) -> Result<(), String> { + let mut state = self.shutdown.lock().expect("backend shutdown lock"); + if let Some(result) = state.as_ref() { + return result.clone(); + } + self.closed.store(true, Ordering::Release); + self.health.shutdown(); + self.musicbrainz.shutdown(); + self.manager.environment().shared_app_version().close(); + let result = self.lyrics.shutdown(); + let mut operations = self.operations.lock().expect("backend operations lock"); + while *operations != 0 { + operations = self.idle.wait(operations).expect("backend operations wait"); + } + drop(operations); + self.share_cache.lock().expect("share cache lock").clear(); + self.cover.clear(); + self.prepared_downloads + .lock() + .expect("prepared downloads lock") + .clear(); + if let Err(error) = &result { + let _ = self.manager.environment().log_buffer().add( + "ERROR", + "Lyrics", + &format!("Failed to persist lyrics cache: {error}"), + ); + } + self.availability.shutdown(); + self.manager.shutdown(); + *state = Some(result.clone()); + result + } + + /// Preserve the native manager's void shutdown API; flush failures are + /// logged, and native callers can also inspect `shutdown_checked`. + pub fn shutdown(&self) { + let _ = self.shutdown_checked(); + } + + fn check(&self) -> Result<(), String> { + if self.closed.load(Ordering::Acquire) || self.manager.environment().is_closed() { + Err("backend is closed".into()) + } else { + Ok(()) + } + } + + fn enter(&self) -> Result<Operation<'_>, String> { + let mut operations = self.operations.lock().expect("backend operations lock"); + self.check()?; + *operations += 1; + Ok(Operation(self)) + } +} + +struct Operation<'a>(&'a Backend); + +impl Drop for Operation<'_> { + fn drop(&mut self) { + *self.0.operations.lock().expect("backend operations lock") -= 1; + self.0.idle.notify_all(); + } +} + +impl Deref for Backend { + type Target = ExtensionManager; + + fn deref(&self) -> &Self::Target { + &self.manager + } +} + +impl Drop for Backend { + fn drop(&mut self) { + self.shutdown(); + } +} diff --git a/rust_backend/crates/extensions/src/backend/cover.rs b/rust_backend/crates/extensions/src/backend/cover.rs new file mode 100644 index 00000000..800f2b00 --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/cover.rs @@ -0,0 +1,643 @@ +use super::Backend; +use cap_std::fs::OpenOptions; +use spotiflac_core::app_version::AppVersion; +use spotiflac_core::cover::{MAX_DOWNLOAD_BYTES, resize}; +use spotiflac_core::tags::{CoverArt, extract_cover}; +use spotiflac_network::{ + HttpRequest, NetworkService, NetworkSession, random_user_agent, url::UrlParts, +}; +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::io::Write; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::Path; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant, UNIX_EPOCH}; + +type Check<'a> = dyn Fn() -> Result<(), String> + 'a; +type Outcome = Result<Arc<[u8]>, String>; +const CACHE_TTL: Duration = Duration::from_secs(15 * 60); +const SIZE_ERROR: &str = "cover download exceeds 24 MiB limit"; + +struct Entry { + data: Arc<[u8]>, + expires: Instant, +} + +#[derive(Default)] +struct Cache { + entries: BTreeMap<String, Entry>, + bytes: usize, + flights: BTreeMap<String, Arc<Flight>>, +} + +#[derive(Default)] +struct Flight { + // Outer None is pending; inner None asks followers to retry after the + // leader's cancellation. One caller cannot cancel another caller's work. + outcome: Mutex<Option<Option<Outcome>>>, + ready: Condvar, +} + +pub(super) struct Cover { + network: Arc<NetworkSession>, + version: AppVersion, + cache: Mutex<Cache>, +} + +impl Cover { + pub fn new(network: &Arc<NetworkService>, version: AppVersion) -> Self { + Self { + network: network.native_media_session(Duration::from_secs(60)), + version, + cache: Mutex::default(), + } + } + + pub fn clear(&self) { + let mut cache = self.cache.lock().expect("cover cache lock"); + cache.entries.clear(); + cache.bytes = 0; + } + + pub(super) fn download(&self, url: &str, limit: i64, check: &Check<'_>) -> Outcome { + let key = if limit <= 0 { + url.to_owned() + } else { + format!("{url}\0max-dimension={limit}") + }; + self.cached(&key, check, &|| { + if limit <= 0 { + return self.fetch(url, check); + } + let original = self.download(url, 0, check)?; + match resize(&original, limit, check) + .map_err(|error| format!("resize artwork: {error}"))? + { + std::borrow::Cow::Borrowed(_) => Ok(original), + std::borrow::Cow::Owned(data) => Ok(data.into()), + } + }) + } + + fn cached(&self, key: &str, check: &Check<'_>, fetch: &dyn Fn() -> Outcome) -> Outcome { + loop { + check()?; + let (flight, leader) = { + let mut cache = self.cache.lock().expect("cover cache lock"); + if let Some(entry) = cache.entries.get(key) { + if Instant::now() < entry.expires { + return Ok(entry.data.clone()); + } + let entry = cache.entries.remove(key).expect("expired cover entry"); + cache.bytes -= entry.data.len(); + } + if let Some(flight) = cache.flights.get(key) { + (flight.clone(), false) + } else { + let flight = Arc::new(Flight::default()); + cache.flights.insert(key.into(), flight.clone()); + (flight, true) + } + }; + if !leader { + let mut outcome = flight.outcome.lock().expect("cover flight lock"); + loop { + check()?; + match outcome.as_ref() { + Some(Some(result)) => return result.clone(), + Some(None) => break, + None => { + outcome = flight + .ready + .wait_timeout(outcome, Duration::from_millis(25)) + .expect("cover flight wait") + .0; + } + } + } + continue; + } + // A decoder/transport panic must not strand coalesced callers. + let result = catch_unwind(AssertUnwindSafe(fetch)) + .unwrap_or_else(|_| Err("cover fetch aborted".into())); + let cancelled = check().err(); + let mut cache = self.cache.lock().expect("cover cache lock"); + if cancelled.is_none() + && let Ok(data) = &result + && !data.is_empty() + && data.len() <= MAX_DOWNLOAD_BYTES + { + cache.bytes += data.len(); + cache.entries.insert( + key.into(), + Entry { + data: data.clone(), + expires: Instant::now() + CACHE_TTL, + }, + ); + while cache.bytes > MAX_DOWNLOAD_BYTES { + let oldest = cache + .entries + .iter() + .min_by_key(|(_, entry)| entry.expires) + .map(|(key, _)| key.clone()) + .expect("nonempty cover cache"); + cache.bytes -= cache.entries.remove(&oldest).unwrap().data.len(); + } + } + cache.flights.remove(key); + *flight.outcome.lock().expect("cover flight lock") = + Some(cancelled.is_none().then(|| result.clone())); + flight.ready.notify_all(); + return cancelled.map_or(result, Err); + } + } + + fn fetch(&self, url: &str, check: &Check<'_>) -> Outcome { + let parsed = UrlParts::parse(url) + .ok_or_else(|| "failed to create request: invalid URL".to_owned())?; + let user_agent = if parsed.hostname.eq_ignore_ascii_case("api.zarz.moe") { + self.version.user_agent() + } else { + random_user_agent() + }; + let mut response = self + .network + .open_response_stream( + HttpRequest { + url: url.into(), + method: "GET".into(), + body: String::new(), + headers: BTreeMap::new(), + default_json: false, + user_agent, + }, + check, + ) + .map_err(|error| format!("failed to download cover: {error}"))?; + if response.response.status != 200 { + return Err(format!( + "cover download failed: HTTP {}", + response.response.status + )); + } + if response + .response + .headers + .get("Content-Length") + .and_then(|values| values.first()) + .and_then(|value| value.parse::<u64>().ok()) + .is_some_and(|length| length > MAX_DOWNLOAD_BYTES as u64) + { + return Err(SIZE_ERROR.into()); + } + let mut data = Vec::new(); + let mut buffer = [0; 64 * 1024]; + loop { + let capacity = buffer.len().min(MAX_DOWNLOAD_BYTES + 1 - data.len()); + let count = response + .read(&mut buffer[..capacity], check) + .map_err(|error| format!("failed to read cover data: {error}"))?; + if count == 0 { + break; + } + data.extend_from_slice(&buffer[..count]); + if data.len() > MAX_DOWNLOAD_BYTES { + return Err(SIZE_ERROR.into()); + } + } + Ok(data.into()) + } +} + +impl Backend { + pub fn extract_cover_to_file( + &self, + audio_path: &str, + output_path: &str, + check: &Check<'_>, + ) -> Result<(), String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + check()?; + let format = extension(audio_path); + if !matches!( + format.as_str(), + "flac" | "m4a" | "aac" | "mp3" | "ogg" | "opus" | "wav" | "aiff" | "aif" | "aifc" + ) { + return Err("unsupported audio format for cover extraction".into()); + } + let files = self.manager.environment().native_files()?; + let input = files.resolve_legacy(audio_path)?; + let cover = read_cover(&input, &format, &check) + .map_err(|error| format!("failed to extract cover: {error}"))?; + let write = || { + let output = files.resolve_legacy(output_path)?; + if input.absolute == output.absolute { + return Err("cover output must differ from the audio source".into()); + } + let _lock = files.lock(&output, &check)?; + let permissions = output_permissions(&output)?; + publish_cover(&output, permissions, &cover.data, &check) + }; + write().map_err(|error| format!("failed to write cover file: {error}")) + } + + pub fn save_cover_to_cache_with_hint_and_key( + &self, + audio_path: &str, + hint: &str, + cache_directory: &str, + explicit_key: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.cache_cover( + audio_path, + hint, + cache_directory, + explicit_key, + None, + read_cover, + check, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(super) fn save_cover_to_cache_for_scan( + &self, + audio_path: &str, + hint: &str, + cache_directory: &str, + explicit_key: &str, + read: impl FnOnce(&crate::files::FilePath, &str, &Check<'_>) -> Result<CoverArt, String>, + check: &Check<'_>, + ) -> Result<String, String> { + let format = spotiflac_core::tags::library_extension(audio_path, hint); + // The scan's MP4-family reader also accepts .mp4/.aac. The standalone + // cover export deliberately keeps its narrower legacy suffix policy. + let format = matches!(format.as_str(), "mp4" | "aac").then_some("m4a"); + self.cache_cover( + audio_path, + hint, + cache_directory, + explicit_key, + format, + read, + check, + ) + } + + #[allow(clippy::too_many_arguments)] + fn cache_cover( + &self, + audio_path: &str, + hint: &str, + cache_directory: &str, + explicit_key: &str, + format_override: Option<&str>, + read: impl FnOnce(&crate::files::FilePath, &str, &Check<'_>) -> Result<CoverArt, String>, + check: &Check<'_>, + ) -> Result<String, String> { + let _operation = self.enter()?; + let failure = RefCell::new(None::<String>); + let check = || { + if let Some(error) = failure.borrow().as_ref() { + return Err(error.clone()); + } + self.check().and_then(|()| check()).inspect_err(|error| { + *failure.borrow_mut() = Some(error.clone()); + }) + }; + check()?; + let files = self.manager.environment().native_files()?; + let input = files.resolve_legacy(audio_path)?; + input.native_display()?; + let key = if !explicit_key.trim().is_empty() { + explicit_key.trim().to_owned() + } else if let Ok(metadata) = input.metadata() { + let modified = metadata + .modified() + .map_err(|error| error.to_string())? + .into_std(); + let nanos = match modified.duration_since(UNIX_EPOCH) { + Ok(duration) => duration.as_nanos() as i128, + Err(error) => -(error.duration().as_nanos() as i128), + }; + format!("{audio_path}|{}|{nanos}", metadata.len()) + } else { + audio_path.into() + }; + let hash = key.chars().fold(5381_u32, |hash, value| { + hash.wrapping_mul(33).wrapping_add(value as u32) + }); + let path = + |extension| Path::new(cache_directory).join(format!("cover_{hash:x}.{extension}")); + let jpg = files.resolve_legacy(&path("jpg").to_string_lossy())?; + let png = files.resolve_legacy(&path("png").to_string_lossy())?; + for cached in [&jpg, &png] { + cached.native_display()?; + if cached.metadata().is_ok_and(|metadata| metadata.is_file()) { + return Ok(cached.display()); + } + } + let mut format = extension(audio_path); + if format.is_empty() { + format = extension(hint); + } + if let Some(value) = format_override { + format = value.into(); + } + if !matches!( + format.as_str(), + "flac" | "m4a" | "mp3" | "ogg" | "opus" | "wav" | "aiff" | "aif" | "aifc" + ) { + return Err(format!( + "unsupported format: {}", + if format.is_empty() { + String::new() + } else { + format!(".{format}") + } + )); + } + let mut cover = read(&input, &format, &check)?; + if let Ok(std::borrow::Cow::Owned(resized)) = resize( + &cover.data, + spotiflac_core::cover::LIBRARY_MAX_DIMENSION, + &check, + ) { + cover.mime = if resized.starts_with(b"\x89PNG") { + "image/png" + } else { + "image/jpeg" + } + .into(); + cover.data = resized; + } + // Library caching may retain unsupported image bytes, but must never + // swallow cancellation while attempting the optional resize. + check()?; + let output = if cover.mime.contains("png") { png } else { jpg }; + let _lock = files.lock(&output, &check)?; + if output.metadata().is_ok_and(|metadata| metadata.is_file()) { + return Ok(output.display()); + } + output + .mkdir_parent() + .map_err(|error| format!("failed to create cache dir: {error}"))?; + let permissions = output_permissions(&output) + .map_err(|error| format!("failed to write cover: {error}"))?; + publish_cover(&output, permissions, &cover.data, &check) + .map_err(|error| format!("failed to write cover: {error}"))?; + Ok(output.display()) + } + + pub fn clear_cover_memory_cache(&self) -> Result<(), String> { + let _operation = self.enter()?; + self.cover.clear(); + Ok(()) + } + + pub fn download_cover_to_file_sized( + &self, + url: &str, + output_path: &str, + max_dimension: i64, + check: &Check<'_>, + ) -> Result<(), String> { + let _operation = self.enter()?; + let failure = RefCell::new(None::<String>); + let check = || { + if let Some(error) = failure.borrow().as_ref() { + return Err(error.clone()); + } + self.check().and_then(|()| check()).inspect_err(|error| { + *failure.borrow_mut() = Some(error.clone()); + }) + }; + check()?; + if url.is_empty() { + return Err("no cover URL provided".into()); + } + let files = self.manager.environment().native_files()?; + // Validate and lock before doing network work for an unauthorized or + // unavailable destination. Publication revalidates the same grant. + let prepare = || { + let output = files.resolve_legacy(output_path)?; + let lock = files.lock(&output, &check)?; + let permissions = output_permissions(&output)?; + Ok((output, lock, permissions)) + }; + let (output, _lock, permissions) = + prepare().map_err(|error: String| format!("failed to write cover file: {error}"))?; + let data = self + .cover + .download(url, max_dimension, &check) + .map_err(|error| format!("failed to download cover: {error}"))?; + publish_cover(&output, permissions, &data, &check) + .map_err(|error| format!("failed to write cover file: {error}")) + } +} + +fn extension(path: &str) -> String { + Path::new(path) + .extension() + .and_then(|value| value.to_str()) + .unwrap_or("") + .to_lowercase() +} + +fn read_cover( + input: &crate::files::FilePath, + format: &str, + check: &Check<'_>, +) -> Result<CoverArt, String> { + input.native_display()?; + let mut file = input + .open(OpenOptions::new().read(true)) + .map_err(|error| error.to_string())?; + extract_cover(&mut file, format, check) +} + +fn output_permissions( + output: &crate::files::FilePath, +) -> Result<Option<cap_std::fs::Permissions>, String> { + output.native_display()?; + output.require_parent()?; + match output.open(OpenOptions::new().write(true)) { + Ok(file) => Ok(Some( + file.metadata() + .map_err(|error| error.to_string())? + .permissions(), + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.to_string()), + } +} + +fn publish_cover( + output: &crate::files::FilePath, + permissions: Option<cap_std::fs::Permissions>, + data: &[u8], + check: &Check<'_>, +) -> Result<(), String> { + check()?; + let mut stage = output + .stage_existing_parent() + .map_err(|error| error.to_string())?; + if let Some(permissions) = permissions { + stage + .file + .set_permissions(permissions) + .map_err(|error| error.to_string())?; + } + for chunk in data.chunks(16 * 1024) { + check()?; + stage + .file + .write_all(chunk) + .map_err(|error| error.to_string())?; + } + stage.publish(check) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::mpsc; + use std::thread; + + fn cover() -> Cover { + Cover::new(&NetworkService::new().unwrap(), "1".into()) + } + + #[test] + fn cache_coalesces_and_a_cancelled_caller_does_not_cancel_its_peer() { + for cancel_leader in [false, true] { + let cover = cover(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let (waiting_tx, waiting_rx) = mpsc::channel(); + let cancelled = AtomicBool::new(false); + let calls = AtomicUsize::new(0); + thread::scope(|scope| { + let cover = &cover; + let cancelled = &cancelled; + let calls = &calls; + let leader = scope.spawn(move || { + cover.cached( + "shared", + &|| { + if cancelled.load(Ordering::Acquire) { + Err("leader cancelled".into()) + } else { + Ok(()) + } + }, + &|| { + calls.fetch_add(1, Ordering::AcqRel); + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + Ok(Arc::from(&b"leader"[..])) + }, + ) + }); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let follower = scope.spawn(|| { + let checks = Cell::new(0); + cover.cached( + "shared", + &|| { + checks.set(checks.get() + 1); + if checks.get() == 2 { + waiting_tx.send(()).unwrap(); + if !cancel_leader { + return Err("follower cancelled".into()); + } + } + Ok(()) + }, + &|| { + calls.fetch_add(1, Ordering::AcqRel); + Ok(Arc::from(&b"follower"[..])) + }, + ) + }); + waiting_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + cancelled.store(cancel_leader, Ordering::Release); + release_tx.send(()).unwrap(); + let (leader, follower) = (leader.join().unwrap(), follower.join().unwrap()); + let expected = if cancel_leader { + assert_eq!(leader.unwrap_err(), "leader cancelled"); + assert_eq!(follower.unwrap().as_ref(), b"follower"); + assert_eq!(calls.load(Ordering::Acquire), 2); + &b"follower"[..] + } else { + assert_eq!(leader.unwrap().as_ref(), b"leader"); + assert_eq!(follower.unwrap_err(), "follower cancelled"); + assert_eq!(calls.load(Ordering::Acquire), 1); + &b"leader"[..] + }; + assert_eq!( + cover + .cached("shared", &|| Ok(()), &|| panic!("cache miss")) + .unwrap() + .as_ref(), + expected + ); + }); + } + } + + #[test] + fn cache_expires_evicts_by_byte_budget_and_does_not_retain_failures() { + let cover = cover(); + let fetches = Cell::new(0); + let fetch = || { + fetches.set(fetches.get() + 1); + Ok(Arc::from(vec![42; 16 << 20])) + }; + for key in ["first", "first", "second", "first"] { + assert_eq!( + cover.cached(key, &|| Ok(()), &fetch).unwrap().len(), + 16 << 20 + ); + } + assert_eq!(fetches.get(), 3); + cover + .cache + .lock() + .unwrap() + .entries + .get_mut("first") + .unwrap() + .expires = Instant::now() - Duration::from_secs(1); + cover.cached("first", &|| Ok(()), &fetch).unwrap(); + assert_eq!(fetches.get(), 4); + cover.clear(); + cover.cached("first", &|| Ok(()), &fetch).unwrap(); + assert_eq!(fetches.get(), 5); + for _ in 0..2 { + assert_eq!( + cover + .cached("error", &|| Ok(()), &|| Err("HTTP 500".into())) + .unwrap_err(), + "HTTP 500" + ); + } + assert_eq!( + cover + .cached("panic", &|| Ok(()), &|| panic!("decoder panic")) + .unwrap_err(), + "cover fetch aborted" + ); + assert_eq!( + cover.cached("panic", &|| Ok(()), &fetch).unwrap().len(), + 16 << 20 + ); + assert_eq!(fetches.get(), 6); + } +} diff --git a/rust_backend/crates/extensions/src/backend/download_latency_tests.rs b/rust_backend/crates/extensions/src/backend/download_latency_tests.rs new file mode 100644 index 00000000..e07667e6 --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/download_latency_tests.rs @@ -0,0 +1,574 @@ +use super::*; +use crate::environment::ExtensionEnvironment; +use crate::{RuntimeLimits, backend::MetadataOptions}; +use spotiflac_network::{NetworkOptions, NetworkService}; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; + +const RECORDING: &str = r#"{"recordings":[{"tags":[{"name":"rock","count":1}],"releases":[{"title":"Album","artist-credit":[{"name":"Album Artist"}]}]}]}"#; +const CATALOG: &str = r#"{"id":7,"album":{"id":9},"genres":{"data":[{"name":"Jazz"}]},"label":"Label","copyright":"Copyright"}"#; + +fn network() -> (Arc<NetworkService>, Arc<rustls::ServerConfig>) { + let certificate = rcgen::generate_simple_self_signed(vec!["127.0.0.1".into()]).unwrap(); + let config = rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert( + vec![certificate.cert.der().clone()], + rustls::pki_types::PrivatePkcs8KeyDer::from(certificate.signing_key.serialize_der()).into(), + ) + .unwrap(); + let network = NetworkService::with_options(NetworkOptions { + extra_root_pem: certificate.cert.pem().into_bytes(), + doh_upstreams: vec![], + ..Default::default() + }) + .unwrap(); + network.set_allow_private_network(true); + (network, Arc::new(config)) +} + +fn server( + requests: usize, + body: &'static str, + config: Arc<rustls::ServerConfig>, +) -> ( + String, + mpsc::Receiver<()>, + mpsc::Sender<()>, + thread::JoinHandle<Vec<String>>, +) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("https://{}", listener.local_addr().unwrap()); + let (started, ready) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let worker = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(5); + let mut paths = Vec::new(); + for index in 0..requests { + let stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!(Instant::now() < deadline, "missing metadata request"); + thread::sleep(Duration::from_millis(1)); + } + Err(error) => panic!("{error}"), + } + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut stream = rustls::StreamOwned::new( + rustls::ServerConnection::new(Arc::clone(&config)).unwrap(), + stream, + ); + let mut request = [0; 4096]; + let count = stream.read(&mut request).unwrap(); + assert!(count > 0); + paths.push( + std::str::from_utf8(&request[..count]) + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .to_owned(), + ); + if index == 0 { + let _ = started.send(()); + released.recv_timeout(Duration::from_secs(3)).unwrap(); + } + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + } + paths + }); + (url, ready, release, worker) +} + +fn backend( + directory: &Path, + catalog: String, + recording: String, + network: Arc<NetworkService>, +) -> Arc<Backend> { + let environment = ExtensionEnvironment::with_network( + &directory.join("data"), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "1", + network, + ) + .unwrap(); + let mut options = MetadataOptions { + deezer_endpoint: catalog, + ..Default::default() + }; + options.musicbrainz.endpoint = recording; + Arc::new( + Backend::with_environment( + &directory.join("sources"), + environment, + RuntimeLimits::default(), + options, + ) + .unwrap(), + ) +} + +#[test] +fn extended_metadata_overlaps_requests_preserves_fields_and_reuses_genre_fallback() { + for catalog_body in [ + CATALOG, + r#"{"id":7,"album":{"id":9},"label":"Label","copyright":"Copyright"}"#, + ] { + let (network, config) = network(); + let (catalog, catalog_ready, release_catalog, catalog_worker) = + server(2, catalog_body, Arc::clone(&config)); + let (recording, recording_ready, release_recording, recording_worker) = + server(1, RECORDING, config); + let directory = tempfile::tempdir().unwrap(); + let backend = backend(directory.path(), catalog, recording, network); + let active = Arc::clone(&backend); + let worker = thread::spawn(move || { + let mut request = DownloadRequest { + isrc: "EXAMPLE12345".into(), + album_name: "Album".into(), + label: "Existing label".into(), + ..Default::default() + }; + active + .enrich_download_extended(&mut request, &|| Ok(())) + .unwrap(); + request + }); + // Neither server replies until both independent requests have started. + catalog_ready.recv_timeout(Duration::from_secs(2)).unwrap(); + recording_ready + .recv_timeout(Duration::from_secs(2)) + .unwrap(); + release_catalog.send(()).unwrap(); + release_recording.send(()).unwrap(); + let mut request = worker.join().unwrap(); + assert_eq!(request.album_artist, "Album Artist"); + assert_eq!(request.label, "Existing label"); + assert_eq!(request.copyright, "Copyright"); + assert_eq!( + request.genre, + if catalog_body == CATALOG { + "Jazz" + } else { + "Rock" + } + ); + assert_eq!( + catalog_worker.join().unwrap(), + ["/2.0/track/isrc:EXAMPLE12345", "/2.0/album/9"] + ); + recording_worker.join().unwrap(); + // Complete input must not query either server, which is now closed. + backend + .enrich_download_extended(&mut request, &|| Ok(())) + .unwrap(); + assert_eq!(request.label, "Existing label"); + backend.shutdown(); + } +} + +#[test] +fn extended_metadata_cancellation_joins_both_blocked_requests() { + for shutdown in [false, true] { + let (network, config) = network(); + let (catalog, catalog_ready, release_catalog, catalog_worker) = + server(1, CATALOG, Arc::clone(&config)); + let (recording, recording_ready, release_recording, recording_worker) = + server(1, RECORDING, config); + let directory = tempfile::tempdir().unwrap(); + let backend = backend(directory.path(), catalog, recording, network); + let cancelled = Arc::new(AtomicBool::new(false)); + let stop = Arc::clone(&cancelled); + let active = Arc::clone(&backend); + let worker = thread::spawn(move || { + let mut request = DownloadRequest { + isrc: "EXAMPLE12345".into(), + album_name: "Album".into(), + ..Default::default() + }; + active.enrich_download_extended(&mut request, &|| { + active.check()?; + if stop.load(Ordering::Acquire) { + Err("download cancelled".into()) + } else { + Ok(()) + } + }) + }); + catalog_ready.recv_timeout(Duration::from_secs(2)).unwrap(); + recording_ready + .recv_timeout(Duration::from_secs(2)) + .unwrap(); + if shutdown { + backend.shutdown(); + } else { + cancelled.store(true, Ordering::Release); + } + let started = Instant::now(); + while !worker.is_finished() { + assert!( + started.elapsed() < Duration::from_secs(2), + "cancel waited for HTTP replies" + ); + thread::sleep(Duration::from_millis(1)); + } + let result = worker.join().unwrap(); + if shutdown { + assert!(result.is_err(), "owner shutdown must cancel metadata"); + } else { + assert_eq!(result, Err("download cancelled".into())); + } + release_catalog.send(()).unwrap(); + release_recording.send(()).unwrap(); + catalog_worker.join().unwrap(); + recording_worker.join().unwrap(); + backend.shutdown(); + } +} + +struct GatedServer { + url: String, + started: mpsc::Receiver<(String, mpsc::Sender<()>)>, + stop: Arc<AtomicBool>, + worker: Option<thread::JoinHandle<Vec<String>>>, +} + +impl GatedServer { + fn new(config: Arc<rustls::ServerConfig>, replies: &[(&str, u16, &str)]) -> Self { + let replies: std::collections::BTreeMap<_, _> = replies + .iter() + .map(|(path, status, body)| (path.to_string(), (*status, body.to_string()))) + .collect(); + let replies = Arc::new(replies); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("https://{}", listener.local_addr().unwrap()); + let stop = Arc::new(AtomicBool::new(false)); + let stopped = stop.clone(); + let (ready, started) = mpsc::channel(); + let worker = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(8); + let mut workers = Vec::new(); + while !stopped.load(Ordering::Acquire) && Instant::now() < deadline { + let stream = match listener.accept() { + Ok((stream, _)) => stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(1)); + continue; + } + Err(error) => panic!("gated fixture accept: {error}"), + }; + assert!(workers.len() < 8, "unexpected repeated request"); + let config = config.clone(); + let ready = ready.clone(); + let replies = replies.clone(); + workers.push(thread::spawn(move || { + stream.set_nonblocking(false).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + let mut stream = rustls::StreamOwned::new( + rustls::ServerConnection::new(config).unwrap(), stream, + ); + let mut request = Vec::new(); + while !request.windows(4).any(|part| part == b"\r\n\r\n") { + let mut bytes = [0; 4096]; + let count = stream.read(&mut bytes).unwrap(); + assert!(count > 0 && request.len() + count <= 8192); + request.extend_from_slice(&bytes[..count]); + } + let path = std::str::from_utf8(&request).unwrap().split_whitespace().nth(1).unwrap() + .split('?').next().unwrap().to_owned(); + let (release, released) = mpsc::channel(); + ready.send((path.clone(), release)).unwrap(); + let _ = released.recv_timeout(Duration::from_secs(3)); + let (status, body) = replies.get(&path).expect("unexpected fixture path"); + let _ = write!(stream, "HTTP/1.1 {status} Fixture\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); + path + })); + } + workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .collect() + }); + Self { + url, + started, + stop, + worker: Some(worker), + } + } + + fn next(&self) -> (String, mpsc::Sender<()>) { + self.started.recv_timeout(Duration::from_secs(2)).unwrap() + } + + fn finish(&mut self) -> Vec<String> { + self.stop.store(true, Ordering::Release); + let mut paths = self.worker.take().unwrap().join().unwrap(); + paths.sort(); + paths + } +} + +impl Drop for GatedServer { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +#[test] +fn catalog_search_gates_required_tracks_then_overlaps_optional_categories() { + for mode in ["all", "artist", "track_failure"] { + let (network, config) = network(); + let track = if mode == "track_failure" { + r#"{"error":{"code":100,"message":"required track failure"}}"# + } else { + r#"{"data":[{"id":1,"title":"Track"}]}"# + }; + let artist = if mode == "artist" { + r#"{"data":[{"id":2,"name":"Artist"}]}"# + } else { + r#"{"error":{"code":100,"message":"optional artist failure"}}"# + }; + let mut server = GatedServer::new( + config, + &[ + ("/2.0/search/track", 200, track), + ("/2.0/search/artist", 200, artist), + ( + "/2.0/search/album", + 200, + r#"{"data":[{"id":3,"title":"Album"}]}"#, + ), + ( + "/2.0/search/playlist", + 200, + r#"{"data":[{"id":4,"title":"Playlist"}]}"#, + ), + ], + ); + let directory = tempfile::tempdir().unwrap(); + let backend = backend( + directory.path(), + server.url.clone(), + server.url.clone(), + network, + ); + let active = backend.clone(); + let worker = thread::spawn(move || { + active.search_deezer( + "needle", + 2, + 2, + if mode == "artist" { "artist" } else { "" }, + &|| Ok(()), + ) + }); + let (first, release) = server.next(); + assert_eq!( + first, + if mode == "artist" { + "/2.0/search/artist" + } else { + "/2.0/search/track" + } + ); + assert!(matches!( + server.started.recv_timeout(Duration::from_millis(100)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + release.send(()).unwrap(); + if mode == "all" { + let mut pending = Vec::new(); + for _ in 0..3 { + pending.push(server.next()); + } + let mut categories: Vec<_> = pending.iter().map(|(path, _)| path.as_str()).collect(); + categories.sort(); + assert_eq!( + categories, + [ + "/2.0/search/album", + "/2.0/search/artist", + "/2.0/search/playlist" + ] + ); + for (_, release) in pending { + release.send(()).unwrap(); + } + } + let result = worker.join().unwrap(); + if mode == "track_failure" { + assert!(result.unwrap_err().contains("required track failure")); + } else { + let value: Value = serde_json::from_str(&result.unwrap()).unwrap(); + if mode == "all" { + assert_eq!(value["tracks"][0]["spotify_id"], "deezer:1"); + assert_eq!(value["artists"], json!([])); + assert_eq!(value["albums"][0]["id"], "deezer:3"); + assert_eq!(value["playlists"][0]["id"], "deezer:4"); + } else { + assert_eq!(value["artists"][0]["id"], "deezer:2"); + assert_eq!(value["tracks"], json!([])); + assert_eq!(value["albums"], json!([])); + assert_eq!(value["playlists"], json!([])); + } + } + let paths = server.finish(); + assert_eq!(paths.len(), if mode == "all" { 4 } else { 1 }); + backend.shutdown(); + } +} + +#[test] +fn reenrich_cover_and_lyrics_overlap_preserve_best_effort_results_and_cancel() { + for mode in ["success", "cover_failure", "lyrics_failure", "cancel"] { + let (network, config) = network(); + let mut server = GatedServer::new( + config, + &[ + ( + "/cover", + if mode == "cover_failure" { 404 } else { 200 }, + "artwork", + ), + ( + "/lyrics", + if mode == "lyrics_failure" { 404 } else { 200 }, + "New lyrics", + ), + ], + ); + let directory = tempfile::tempdir().unwrap(); + let backend = backend( + directory.path(), + server.url.clone(), + server.url.clone(), + network, + ); + let archive = directory.path().join("example.lyrics.sflx"); + let mut writer = zip::ZipWriter::new(std::fs::File::create(&archive).unwrap()); + let manifest = json!({"name":"example.lyrics","displayName":"Example Lyrics","version":"1", + "description":"Generic lyrics fixture","type":["lyrics_provider"],"permissions":{"network":["127.0.0.1"]}}); + let source = format!( + r#"registerExtension({{fetchLyrics(){{const response=http.get({}); + return response.statusCode===200 ? {{plainLyrics:response.body}} : {{}};}}}});"#, + json!(format!("{}/lyrics", server.url)) + ); + for (name, body) in [ + ("manifest.json", manifest.to_string()), + ("index.js", source), + ] { + writer + .start_file(name, zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(body.as_bytes()).unwrap(); + } + writer.finish().unwrap(); + backend.install(&archive).unwrap(); + backend.set_enabled("example.lyrics", true).unwrap(); + backend + .set_lyrics_providers_json(r#"["extension:example.lyrics"]"#) + .unwrap(); + let audio = directory.path().join("data/track.mp3"); + let sidecar = directory.path().join("data/track.lrc"); + std::fs::write(&audio, "original audio").unwrap(); + std::fs::write(&sidecar, "[00:01.00]Old lyrics").unwrap(); + let request = json!({"file_path":"track.mp3","track_name":"Track","artist_name":"Artist", + "cover_url":format!("{}/cover", server.url),"embed_lyrics":true,"lyrics_mode":"both", + "update_fields":["cover","lyrics"]}) + .to_string(); + let cancelled = Arc::new(AtomicBool::new(false)); + let stop = cancelled.clone(); + let active = backend.clone(); + let worker = thread::spawn(move || { + active.reenrich_file(&request, &|| { + if stop.load(Ordering::Acquire) { + Err("reenrich cancelled".into()) + } else { + Ok(()) + } + }) + }); + let first = server.next(); + let second = server.next(); + let mut paths = [first.0.as_str(), second.0.as_str()]; + paths.sort(); + assert_eq!(paths, ["/cover", "/lyrics"]); + if mode == "cancel" { + cancelled.store(true, Ordering::Release); + let deadline = Instant::now() + Duration::from_secs(2); + while !worker.is_finished() { + assert!( + Instant::now() < deadline, + "reenrich cancellation waited for server replies" + ); + thread::sleep(Duration::from_millis(1)); + } + } else { + first.1.send(()).unwrap(); + second.1.send(()).unwrap(); + } + let result = worker.join().unwrap(); + if mode == "cancel" { + assert_eq!(result, Err("reenrich cancelled".into())); + first.1.send(()).unwrap(); + second.1.send(()).unwrap(); + } else { + let result: Value = serde_json::from_str(&result.unwrap()).unwrap(); + assert_eq!(result["method"], "ffmpeg"); + assert_eq!(result["write_external_lrc"], true); + let lyrics = result["lyrics"].as_str().unwrap(); + assert!(lyrics.contains(if mode == "lyrics_failure" { + "Old lyrics" + } else { + "New lyrics" + })); + assert_eq!(result["metadata"]["LYRICS"], lyrics); + let cover = result["cover_path"].as_str().unwrap(); + if mode == "cover_failure" { + assert!(cover.is_empty()); + } else { + assert_eq!(std::fs::read(cover).unwrap(), b"artwork"); + std::fs::remove_file(cover).unwrap(); + } + } + assert_eq!(std::fs::read(&audio).unwrap(), b"original audio"); + assert_eq!(std::fs::read(&sidecar).unwrap(), b"[00:01.00]Old lyrics"); + assert!( + std::fs::read_dir(directory.path().join("data")) + .unwrap() + .all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("reenrich_cover_")) + ); + assert_eq!(server.finish(), ["/cover", "/lyrics"]); + backend.shutdown(); + } +} diff --git a/rust_backend/crates/extensions/src/backend/downloads.rs b/rust_backend/crates/extensions/src/backend/downloads.rs new file mode 100644 index 00000000..13339893 --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/downloads.rs @@ -0,0 +1,1748 @@ +//! Application download planning. The installed manager owns VMs/transfers; +//! this root owns request lifetime, fallback and publication of regular files. + +use super::Backend; +use crate::download::native_error_response; +use crate::manager::{ProviderAvailabilityRequest, ProviderDownloadRequest}; +use crate::manifest::ExtensionManifest; +use cap_std::fs::OpenOptions; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use spotiflac_core::cancellation::RequestLease; +use spotiflac_core::downloads::DownloadRequest; +use spotiflac_core::filename::{build_filename_checked, sanitize_filename_preserving_token}; +use spotiflac_core::lyrics::matching::{normalize_loose_artist, normalize_title}; +use spotiflac_core::matching::lowercase; +use spotiflac_core::metadata::reenrich::{self, text}; +use spotiflac_core::resolver::{artists_match, titles_match, track_identity_title}; +use spotiflac_core::tags::{embed_flac_metadata, extract_cover}; +use spotiflac_providers::resolver::Check; +use std::collections::{BTreeMap, VecDeque}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +pub(super) type PreparedDownloads = BTreeMap<String, Prepared>; + +pub(super) struct Prepared { + key: String, + request: DownloadRequest, + metadata_ready: bool, + created: Instant, +} + +enum DownloadOutcome { + Retry(Value), + // Successful downloads and host finalization failures are both terminal. + Final(Value), +} + +impl Backend { + /// Go's strategy facade returns a JSON failure, including transport errors + /// from the extension facade. The latter retains its Result error boundary. + pub fn download_by_strategy(&self, raw: &str, check: &Check<'_>) -> Result<String, String> { + let _operation = self.enter()?; + let request = match DownloadRequest::parse(raw) { + Ok(request) => request, + Err(error) => { + return Ok(native_error_response(&format!("Invalid request: {error}")).to_string()); + } + }; + if !request.use_extensions { + return Ok(native_error_response( + "Extension providers are disabled; built-in download providers have been retired", + ) + .to_string()); + } + Ok(self + .download_request(request, check) + .unwrap_or_else(|error| native_error_response(&error).to_string())) + } + + pub fn download_with_extensions_json( + &self, + raw: &str, + check: &Check<'_>, + ) -> Result<String, String> { + let _operation = self.enter()?; + let request = + DownloadRequest::parse(raw).map_err(|error| format!("invalid request: {error}"))?; + self.download_request(request, check) + } + + fn download_request( + &self, + mut request: DownloadRequest, + check: &Check<'_>, + ) -> Result<String, String> { + request.normalize(); + self.check()?; + check()?; + // Descriptor adoption is an application-owner operation. Do not claim + // a successful regular-file download has written the caller's FD. + if request.output_fd > 0 { + return Err( + "output descriptor ownership is not yet connected to the Rust download planner" + .into(), + ); + } + self.set_song_link_region(&request.songlink_region)?; + let state = self.environment().download_state(); + let lease = Arc::new( + state + .acquire(&request.item_id) + .map_err(|error| error.to_string())?, + ); + lease.check_active().map_err(|error| error.to_string())?; + let item_id = request.item_id.clone(); + if !item_id.is_empty() { + state + .progress + .start(&item_id) + .map_err(|error| error.to_string())?; + state + .progress + .preparing(&item_id, "checking_session") + .map_err(|error| error.to_string())?; + } + // One lease reaches availability, auth, VM and file finalization, + // including requests without an item ID. Join the VM before returning. + let result = std::thread::scope(|scope| { + let (finished, completion) = mpsc::sync_channel(1); + let request_lease = lease.clone(); + let worker = std::thread::Builder::new() + .name("download-planner".into()) + .spawn_scoped(scope, move || { + let result = self.plan_download(request, request_lease); + let _ = finished.send(()); + result + }) + .map_err(|error| error.to_string())?; + let mut cancelled = None; + while !worker.is_finished() { + if let Err(error) = self.check().and_then(|()| check()) { + cancelled = Some(error); + lease.release(); + break; + } + // Completion wakes the caller immediately; timeouts retain the + // owner/caller cancellation heartbeat while work is active. + if !matches!( + completion.recv_timeout(Duration::from_millis(5)), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + break; + } + } + let result = worker + .join() + .map_err(|_| "download planner panicked".to_owned())?; + if let Some(error) = cancelled { + return Err(error); + } + self.check()?; + check()?; + lease.check_active().map_err(|error| error.to_string())?; + result + }); + if !item_id.is_empty() { + if result + .as_ref() + .is_ok_and(|result| result["success"] == true) + { + let _ = state.progress.complete(&item_id); + } else { + let _ = state.progress.remove(&item_id); + } + } + result.map(|value| compact(value).to_string()) + } + + fn plan_download( + &self, + mut request: DownloadRequest, + lease: Arc<RequestLease>, + ) -> Result<Value, String> { + let check = || { + self.check() + .and_then(|()| lease.check_active().map_err(|error| error.to_string())) + }; + let key = preparation_key(&request); + let session_provider = if request.service.trim().is_empty() { + request.source.trim() + } else { + request.service.trim() + } + .to_owned(); + let preflight = self.preflight_download(&session_provider, lease.clone()); + check()?; + match preflight { + Ok(true) => { + self.cache_prepared(&key, &request, false); + return Ok(failure( + &session_provider, + "Verification required before download", + "verification_required", + 0, + )); + } + Err(error) => { + return Ok(failure( + &session_provider, + &format!("Could not start verification for {session_provider}: {error}"), + "", + 0, + )); + } + Ok(false) => {} + } + let prepared = self.take_prepared(&key, &mut request); + if !request.item_id.is_empty() { + let _ = self + .environment() + .download_state() + .progress + .preparing(&request.item_id, "resolving_metadata"); + } + let mut selected = request.service.trim().to_owned(); + let priorities: Value = serde_json::from_str( + &self + .provider_priorities() + .map_err(|error| error.to_string())?, + ) + .map_err(|error| error.to_string())?; + let mut priority: Vec<String> = serde_json::from_value(priorities["download"].clone()) + .map_err(|error| error.to_string())?; + if !request.use_fallback { + if selected.is_empty() { + selected = request.source.trim().to_owned(); + } + if !selected.is_empty() { + priority = vec![selected.clone()]; + } + } else if !request.service.is_empty() { + priority.retain(|id| !id.eq_ignore_ascii_case(&request.service)); + priority.insert(0, request.service.clone()); + } + + if prepared == Some(false) + && let Some(response) = + self.download_verified_retry(&request, &selected, lease.clone())? + { + return Ok(response); + } + + // A source catalog can explicitly prohibit replacing its recording. + let mut source_availability = None; + if !request.source.is_empty() + && selected != request.source + && self + .download_manifest(&request.source) + .is_ok_and(|m| m.has_type("download_provider")) + { + let availability = self.download_availability(&request.source, &request, lease.clone()); + check()?; + if let Ok(availability) = availability + && availability["skip_fallback"] == true + { + selected = request.source.clone(); + source_availability = Some(availability); + } + } + if prepared != Some(true) { + self.enrich_download_source(&mut request, lease.clone())?; + } + + let mut last = None; + let mut attempts = VecDeque::new(); + if !request.source.is_empty() && selected == request.source { + attempts.push_back((request.source.clone(), true)); + } + let mut fallback = Some(priority); + loop { + if attempts.is_empty() { + let Some(priority) = fallback.take() else { + break; + }; + let protected = if selected.is_empty() { + &request.source + } else { + &selected + }; + let priority = self.prioritize_healthy_downloads(priority, protected, &selected)?; + attempts.extend(priority.into_iter().map(|id| (id, false))); + } + let Some((id, direct)) = attempts.pop_front() else { + break; + }; + check()?; + if id.is_empty() + || (!direct && id == request.source && request.source != selected) + || (!direct + && id != selected + && !self + .fallback_allowed(&id) + .map_err(|error| error.to_string())?) + { + continue; + } + let Ok(manifest) = self.download_manifest(&id) else { + continue; + }; + if !manifest.has_type("download_provider") { + continue; + } + let availability = if direct { + source_availability + .clone() + .unwrap_or_else(|| json!({"available":true})) + } else { + let available = self.preflight_download(&id, lease.clone()).map_err(|error| error.to_string()) + .and_then(|verification| { + if verification { Err(format!("verification_required: extension '{id}' needs signed-session verification")) } + else { self.download_availability(&id, &request, lease.clone()) } + }); + check()?; + match available { + Ok(value) => value, + Err(error) => { + let response = failure(&id, &error, "", 0); + if response["error_type"] == "verification_required" { + self.cache_prepared(&key, &request, true); + return Ok(failure( + &id, + &format!("Download failed: {error}"), + "verification_required", + 0, + )); + } + last = Some(response); + continue; + } + } + }; + let stop = availability["skip_fallback"] == true; + if availability["available"] != true { + if stop { + return Ok(stopped(&id, &availability, "")); + } + continue; + } + let track_id = if direct { + preferred_track_id(&request, &manifest, text(&availability, "track_id")) + } else { + text(&availability, "track_id").to_owned() + }; + let mut attempt = request.clone(); + attempt.download_provider = id.clone(); + attempt.provider_track_id = track_id; + if !direct { + attempt.output_ext.clear(); + } + let outcome = + self.download_resolved(&attempt, &manifest, &availability, lease.clone())?; + check()?; + let response = match outcome { + DownloadOutcome::Final(response) => return Ok(response), + DownloadOutcome::Retry(response) => response, + }; + let message = text(&response, "error"); + let kind = text(&response, "error_type"); + let retry = response["retry_after_seconds"].as_i64().unwrap_or_default(); + if kind == "verification_required" { + self.cache_prepared(&key, &request, true); + return Ok(failure( + &id, + &format!("Download failed: {message}"), + kind, + retry, + )); + } + if storage_failure(kind, message) { + return Ok(failure( + &id, + &format!("Download failed: {message}"), + "permission", + retry, + )); + } + if stop { + return Ok(stopped(&id, &availability, message)); + } + if direct && manifest.stops_provider_fallback() { + return Ok(failure( + &id, + &format!("Download failed: {message}"), + kind, + retry, + )); + } + last = Some(response); + } + check()?; + Ok(match last { + Some(last) => failure( + text(&last, "service"), + &format!("All providers failed. Last error: {}", text(&last, "error")), + if text(&last, "error_type") == "unknown" { + "not_found" + } else { + text(&last, "error_type") + }, + last["retry_after_seconds"].as_i64().unwrap_or_default(), + ), + None => failure( + "", + "No extension download providers available", + "not_found", + 0, + ), + }) + } + + /// A preflight challenge can be satisfied without consulting optional + /// metadata providers again. A different download source still gets its + /// normal opportunity to prohibit fallback before any selected download. + fn download_verified_retry( + &self, + request: &DownloadRequest, + selected: &str, + lease: Arc<RequestLease>, + ) -> Result<Option<Value>, String> { + if selected.is_empty() + || (!request.source.is_empty() + && !request.source.eq_ignore_ascii_case(selected) + && self + .download_manifest(&request.source) + .is_ok_and(|manifest| manifest.has_type("download_provider"))) + { + return Ok(None); + } + let Ok(manifest) = self.download_manifest(selected) else { + return Ok(None); + }; + if !manifest.has_type("download_provider") { + return Ok(None); + } + let direct = request.source.eq_ignore_ascii_case(selected); + let availability = if direct { + json!({"available":true}) + } else { + let result = self.download_availability(selected, request, lease.clone()); + lease.check_active().map_err(|error| error.to_string())?; + match result { + Ok(value) => value, + Err(error) => { + let response = failure(selected, &format!("Download failed: {error}"), "", 0); + return Ok( + (response["error_type"] == "verification_required").then_some(response) + ); + } + } + }; + if availability["available"] != true { + return Ok((availability["skip_fallback"] == true) + .then(|| stopped(selected, &availability, ""))); + } + let mut attempt = request.clone(); + attempt.download_provider = selected.into(); + attempt.provider_track_id = + preferred_track_id(request, &manifest, text(&availability, "track_id")); + if !direct { + attempt.output_ext.clear(); + } + let outcome = self.download_resolved(&attempt, &manifest, &availability, lease.clone())?; + lease.check_active().map_err(|error| error.to_string())?; + let response = match outcome { + DownloadOutcome::Final(response) => return Ok(Some(response)), + DownloadOutcome::Retry(response) => response, + }; + let message = text(&response, "error"); + let kind = text(&response, "error_type"); + let retry = response["retry_after_seconds"].as_i64().unwrap_or_default(); + if kind == "verification_required" || storage_failure(kind, message) { + return Ok(Some(failure( + selected, + &format!("Download failed: {message}"), + if kind == "verification_required" { + kind + } else { + "permission" + }, + retry, + ))); + } + // Ordinary failures may recover after source enrichment supplies a + // better native track ID. Preserve the normal strict/fallback path. + Ok(None) + } + + fn download_resolved( + &self, + request: &DownloadRequest, + manifest: &ExtensionManifest, + availability: &Value, + lease: Arc<RequestLease>, + ) -> Result<DownloadOutcome, String> { + let source = [&request.service, &request.source] + .into_iter() + .filter_map(|id| self.download_manifest(id.trim()).ok()) + .find(|manifest| manifest.find_quality(request.quality.trim()).is_some()); + let quality = match manifest.resolve_download_quality(&request.quality, source.as_ref()) { + Ok(quality) => quality, + Err(error) => { + return Ok(DownloadOutcome::Retry(failure( + &request.download_provider, + &error, + "quality_unavailable", + 0, + ))); + } + }; + let mut attempt = request.clone(); + attempt.quality = quality; + self.download_attempt(&attempt, manifest, availability, lease) + } + + fn download_availability( + &self, + id: &str, + request: &DownloadRequest, + lease: Arc<RequestLease>, + ) -> Result<Value, String> { + let input = ProviderAvailabilityRequest { + isrc: request.isrc.clone(), + track_name: request.track_name.clone(), + artist_name: request.artist_name.clone(), + spotify_id: request.spotify_id.clone(), + deezer_id: request.deezer_id.clone(), + tidal_id: request.tidal_id.clone(), + qobuz_id: request.qobuz_id.clone(), + duration_ms: request.duration_ms as isize, + item_id: request.item_id.clone(), + track: host_track(request).as_object().cloned(), + }; + let raw = self + .check_availability_with_lease(id, input, 30_000, Some(lease)) + .map_err(|error| error.to_string())?; + serde_json::from_str(&raw).map_err(|error| error.to_string()) + } + + fn enrich_download_source( + &self, + request: &mut DownloadRequest, + lease: Arc<RequestLease>, + ) -> Result<(), String> { + if request.source.is_empty() { + return Ok(()); + } + if self + .download_manifest(&request.source) + .is_ok_and(|manifest| manifest.has_type("metadata_provider")) + { + let mut input = host_track(request); + input["id"] = request.spotify_id.clone().into(); + for (key, value) in [ + ("provider_id", &request.source), + ("spotify_id", &request.spotify_id), + ("tidal_id", &request.tidal_id), + ("qobuz_id", &request.qobuz_id), + ("deezer_id", &request.deezer_id), + ] { + input[key] = value.clone().into(); + } + let result = self.enrich_track_with_lease( + &request.source, + &input.to_string(), + &request.item_id, + 30_000, + Some(lease.clone()), + ); + lease.check_active().map_err(|error| error.to_string())?; + if let Ok(raw) = result + && let Ok(track) = serde_json::from_str::<Value>(&raw) + { + apply_metadata(request, &track, true); + } + } + if !request.track_name.is_empty() + && !request.artist_name.is_empty() + && (request.album_name.is_empty() + || request.release_date.is_empty() + || request.isrc.is_empty()) + { + let result = self.search_metadata_providers_with_lease( + &format!("{} {}", request.track_name, request.artist_name), + 5, + true, + &request.item_id, + 30_000, + Some(lease.clone()), + ); + lease.check_active().map_err(|error| error.to_string())?; + if let Ok(raw) = result + && let Ok(tracks) = serde_json::from_str::<Vec<Value>>(&raw) + && let Some(track) = select_download_metadata(request, &tracks) + { + apply_metadata(request, track, false); + } + } + Ok(()) + } + + fn download_attempt( + &self, + request: &DownloadRequest, + manifest: &ExtensionManifest, + availability: &Value, + lease: Arc<RequestLease>, + ) -> Result<DownloadOutcome, String> { + let check = || { + self.check() + .and_then(|()| lease.check_active().map_err(|error| error.to_string())) + }; + let environment = self.environment(); + let files = environment.native_files()?; + let directory = environment + .data_directory() + .join(&manifest.name) + .join("downloads"); + let path = output_path(request, &directory, &request.album_name, &check)?; + let output = files.resolve_legacy(&path)?; + output.native_display()?; + let guard = files.lock(&output, &check)?; + // The output lock also owns this destination's attempt directories. + // Reclaim only that namespace after a process restart, including when + // a complete file already exists but its finalizer still needs to run. + let staging_prefix = format!( + ".spotiflac-download-{:x}-", + Sha256::digest(output.display().to_lowercase().as_bytes()) + ); + if directory.exists() { + for entry in directory.read_dir().map_err(|error| error.to_string())? { + check()?; + let entry = entry.map_err(|error| error.to_string())?; + if entry + .file_name() + .to_string_lossy() + .starts_with(&staging_prefix) + && entry + .file_type() + .map_err(|error| error.to_string())? + .is_dir() + { + std::fs::remove_dir_all(entry.path()).map_err(|error| error.to_string())?; + } + } + } + if !request.allow_quality_variant + && (request.album_folder_template.is_empty() + || !album_folder(request, &request.album_name).is_empty()) + && output + .metadata() + .is_ok_and(|metadata| metadata.is_file() && metadata.len() > 0) + { + let mut result = json!({"file_path":path}); + probe_result(&files, &mut result, false, &check)?; + let mut response = success(request, &result, &path, true, manifest, &check)?; + response + .as_object_mut() + .unwrap() + .remove("skip_metadata_enrichment"); + return Ok(DownloadOutcome::Final(response)); + } + // Give the extension its own temporary directory. A mismatched track, + // failed provider or cancellation can never overwrite an old library file. + let temporary_parent = + files.resolve_legacy(&directory.join(".parent").to_string_lossy())?; + temporary_parent.native_display()?; + temporary_parent + .mkdir_parent() + .map_err(|error| format!("failed to create directory: {error}"))?; + let temporary = tempfile::Builder::new() + .prefix(&staging_prefix) + .tempdir_in(&directory) + .map_err(|error| format!("failed to create directory: {error}"))?; + let _grant = environment.grant_temporary_download_directory(temporary.path())?; + let name = filename(request, &request.output_ext, &check)?; + let staging_path = temporary.path().join(name).to_string_lossy().into_owned(); + let mut prepared = availability["prepared_context"] + .as_object() + .cloned() + .unwrap_or_default(); + prepared.insert("host_track".into(), host_track(request)); + if !request.item_id.is_empty() { + let _ = environment + .download_state() + .progress + .preparing(&request.item_id, "resolving_stream"); + } + let response = self.download_with_lease( + &manifest.name, + ProviderDownloadRequest { + track_id: request.provider_track_id.clone(), + quality: request.quality.clone(), + output_path: staging_path, + item_id: request.item_id.clone(), + prepared_context: Some(prepared), + }, + 60_000, + Some(lease.clone()), + ); + check()?; + let mut result: Value = match response { + Ok(raw) => serde_json::from_str(&raw).map_err(|error| error.to_string())?, + Err(error) => { + return Ok(DownloadOutcome::Retry(failure( + &manifest.name, + &error.to_string(), + "", + 0, + ))); + } + }; + if result["success"] != true { + let message = if text(&result, "error_message").is_empty() { + "extension download failed without an error message" + } else { + text(&result, "error_message") + }; + let kind = text(&result, "error_type").trim(); + let classified = native_error_response(message); + let kind = if matches!( + kind.to_ascii_lowercase().as_str(), + "" | "unknown" + | "runtime_error" + | "api_error" + | "download_error" + | "extension_error" + ) && classified["error_type"] != "unknown" + { + text(&classified, "error_type") + } else if kind.is_empty() { + "extension_error" + } else { + kind + }; + return Ok(DownloadOutcome::Retry(failure( + &manifest.name, + message, + kind, + result["retry_after_seconds"].as_i64().unwrap_or_default(), + ))); + } + let skip_names = request.source.trim().eq_ignore_ascii_case(&manifest.name) + || manifest + .track_matching + .as_ref() + .is_some_and(|matching| matching.custom_matching); + if !matches_track(request, &result, skip_names) { + return Ok(DownloadOutcome::Retry(failure( + &manifest.name, + &format!("provider {} returned a different track", manifest.name), + "not_found", + 0, + ))); + } + let returned = text(&result, "file_path").trim(); + let already_exists = result["already_exists"] == true || returned.starts_with("EXISTS:"); + let returned = returned + .strip_prefix("EXISTS:") + .unwrap_or(returned) + .to_owned(); + let input = files.resolve_legacy(&returned)?; + input.native_display()?; + if !already_exists + && !crate::files::clean(Path::new(&returned)).starts_with(temporary.path()) + { + return Ok(DownloadOutcome::Retry(failure( + &manifest.name, + "provider output is outside its download staging directory", + "file_error", + 0, + ))); + } + if !input + .metadata() + .is_ok_and(|metadata| metadata.is_file() && metadata.len() > 0) + { + return Ok(DownloadOutcome::Retry(failure( + &manifest.name, + "provider returned no audio file", + "file_error", + 0, + ))); + } + result["file_path"] = returned.clone().into(); + let mut enriched = request.clone(); + if !manifest.skip_metadata_enrichment { + self.enrich_download_extended(&mut enriched, &check)?; + } + let request = &enriched; + probe_result( + &files, + &mut result, + request.album_name.trim().is_empty() + && request.album_folder_template.contains("{album}"), + &check, + )?; + if already_exists { + return success(request, &result, &returned, true, manifest, &check) + .map(DownloadOutcome::Final); + } + if !request.item_id.is_empty() { + let _ = environment + .download_state() + .progress + .finalizing(&request.item_id); + } + + let mut response = success(request, &result, &path, false, manifest, &check)?; + let mut resolved = request.clone(); + resolved.output_ext = text(&response, "actual_extension").to_owned(); + let mut final_path = output_path(&resolved, &directory, text(&response, "album"), &check)?; + if !request.output_path.is_empty() { + final_path = request.output_path.clone(); + } + // Keep the original lock for an unchanged destination. Release it + // before acquiring a different path lock to avoid lock-order cycles. + let destination = files.resolve_legacy(&final_path)?; + destination.native_display()?; + let _guard = if final_path == path { + guard + } else { + drop(guard); + files.lock(&destination, &check)? + }; + if final_path != path && destination.metadata().is_ok() { + return Ok(DownloadOutcome::Final(failure( + &manifest.name, + &format!("resolve album folder: open {final_path}: file exists"), + "file_error", + 0, + ))); + } + let mut source = input + .open(OpenOptions::new().read(true)) + .map_err(|error| error.to_string())?; + let info = source.metadata().map_err(|error| error.to_string())?; + if !info.is_file() || info.len() == 0 { + return Err("provider returned no audio file".into()); + } + // ReplayGain, decryption/conversion, external LRC and extension hooks + // are executed by the existing Android/Dart finalizers. This is the + // Go backend's local FLAC embed step; provider lyrics are reused. + let embed = request.embed_metadata && returned.to_ascii_lowercase().ends_with(".flac"); + let cover = if embed { + let url = [text(&response, "cover_url"), &request.cover_url] + .into_iter() + .map(str::trim) + .find(|url| !url.is_empty()) + .unwrap_or_default(); + if url.is_empty() { + None + } else { + self.cover + .download(url, request.cover_max_dimension, &check) + .ok() + } + } else { + None + }; + check()?; + let (mut staged, promoted) = if embed { + destination.stage().map(|stage| (stage, false)) + } else { + destination.stage_from(&input, &source) + } + .map_err(|error| format!("failed to create file: {error}"))?; + if embed { + let cover = cover.as_deref().filter(|bytes| !bytes.is_empty()); + embed_flac_metadata( + &mut source, + &mut staged.file, + &download_metadata_fields(request, &response), + &request.artist_tag_mode, + cover, + &check, + )?; + if cover.is_some() + && extract_cover(&mut staged.file, "flac", &check)? + .data + .is_empty() + { + return Err( + "metadata embedded but cover verification failed: empty embedded cover".into(), + ); + } + } else if !promoted { + let mut buffer = [0_u8; 65536]; + loop { + check()?; + let size = source + .read(&mut buffer) + .map_err(|error| error.to_string())?; + if size == 0 { + break; + } + staged + .file + .write_all(&buffer[..size]) + .map_err(|error| error.to_string())?; + } + } + if final_path != path { + if let Err(error) = staged.publish_new(&check) { + check()?; + let message = if error.kind() == std::io::ErrorKind::AlreadyExists { + format!("resolve album folder: open {final_path}: file exists") + } else { + format!("resolve album folder: {error}") + }; + return Ok(DownloadOutcome::Final(failure( + &manifest.name, + &message, + "file_error", + 0, + ))); + } + } else { + staged.publish(&check)?; + } + response["file_path"] = final_path.clone().into(); + if !request.output_dir.is_empty() && !text(&response, "isrc").trim().is_empty() { + // Shared index only; never build a planner-specific cache. + environment + .invalidate_isrc_cache(&request.output_dir) + .map_err(|error| error.to_string())?; + environment + .add_to_isrc_index( + destination + .absolute + .parent() + .unwrap_or(Path::new(".")) + .to_string_lossy() + .as_ref(), + text(&response, "isrc").trim(), + &final_path, + &check, + ) + .map_err(|error| error.to_string())?; + } + Ok(DownloadOutcome::Final(response)) + } + + fn enrich_download_extended( + &self, + request: &mut DownloadRequest, + check: &Check<'_>, + ) -> Result<(), String> { + if request.isrc.is_empty() { + return Ok(()); + } + let needs_artist = request.album_artist.trim().is_empty(); + let needs_tags = + request.genre.is_empty() || request.label.is_empty() || request.copyright.is_empty(); + let artist = || { + self.fetch_music_brainz_album_artist_by_isrc(&request.isrc, &request.album_name, check) + }; + let tags = || { + self.metadata_operation(10, check, |check| { + self.deezer + .get_extended_metadata_by_isrc(request.isrc.trim(), check) + }) + }; + // These lookups are independent. Join the extra worker before returning + // so cancellation and owner shutdown still cover both requests. + let (artist, metadata) = if needs_artist && needs_tags && !request.isrc.trim().is_empty() { + std::thread::scope(|scope| { + let worker = std::thread::Builder::new() + .name("download-album-artist".into()) + .spawn_scoped(scope, artist); + let Ok(worker) = worker else { + let artist = artist(); + check()?; + return Ok((Some(artist), Some(tags()))); + }; + let metadata = tags(); + let artist = worker + .join() + .map_err(|_| "album artist lookup panicked".to_owned())?; + Ok::<_, String>((Some(artist), Some(metadata))) + })? + } else { + let artist = needs_artist.then(artist); + check()?; + let metadata = (needs_tags && !request.isrc.trim().is_empty()).then(tags); + (artist, metadata) + }; + check()?; + if let Some(Ok(artist)) = artist + && !artist.trim().is_empty() + { + request.album_artist = artist.trim().into(); + } + if let Some(metadata) = metadata { + if let Ok(metadata) = metadata { + for (target, value) in [ + (&mut request.genre, &metadata.genre), + (&mut request.label, &metadata.label), + (&mut request.copyright, &metadata.copyright), + ] { + if target.is_empty() { + *target = value.clone(); + } + } + } + if request.genre.is_empty() { + let genre = self.fetch_music_brainz_genre_by_isrc(request.isrc.trim(), check); + check()?; + if let Ok(genre) = genre { + request.genre = genre; + } + } + } + check() + } + + fn cache_prepared(&self, key: &str, request: &DownloadRequest, metadata_ready: bool) { + if request.item_id.trim().is_empty() { + return; + } + let mut cache = self + .prepared_downloads + .lock() + .expect("prepared downloads lock"); + prune(&mut cache); + cache.insert( + request.item_id.trim().into(), + Prepared { + key: key.into(), + request: request.clone(), + metadata_ready, + created: Instant::now(), + }, + ); + } + + fn take_prepared(&self, key: &str, request: &mut DownloadRequest) -> Option<bool> { + let mut cache = self + .prepared_downloads + .lock() + .expect("prepared downloads lock"); + prune(&mut cache); + let prepared = cache + .remove(request.item_id.trim()) + .filter(|entry| entry.key == key)?; + let mut fresh = serde_json::to_value(&*request).expect("download request serialization"); + let old = serde_json::to_value(prepared.request).expect("download request serialization"); + for key in [ + "isrc", + "spotify_id", + "track_name", + "artist_name", + "album_name", + "album_artist", + "cover_url", + "track_number", + "disc_number", + "total_tracks", + "total_discs", + "release_date", + "duration_ms", + "genre", + "label", + "copyright", + "composer", + "tidal_id", + "qobuz_id", + "deezer_id", + ] { + fresh[key] = old[key].clone(); + } + *request = serde_json::from_value(fresh).expect("typed download request"); + Some(prepared.metadata_ready) + } +} + +fn prune(cache: &mut PreparedDownloads) { + cache.retain(|_, entry| entry.created.elapsed() < Duration::from_secs(300)); + while cache.len() >= 128 { + let oldest = cache + .iter() + .min_by_key(|(_, entry)| entry.created) + .map(|(id, _)| id.clone()) + .unwrap(); + cache.remove(&oldest); + } +} + +fn preparation_key(request: &DownloadRequest) -> String { + [ + &request.item_id, + &request.service, + &request.source, + &request.spotify_id, + &request.tidal_id, + &request.qobuz_id, + &request.deezer_id, + &request.track_name, + &request.artist_name, + ] + .into_iter() + .enumerate() + .map(|(index, value)| { + if matches!(index, 1 | 2 | 7 | 8) { + value.trim().to_lowercase() + } else { + value.trim().to_owned() + } + }) + .collect::<Vec<_>>() + .join("\n") +} + +fn host_track(request: &DownloadRequest) -> Value { + let mut track = serde_json::to_value(request).expect("download request serialization"); + let object = track.as_object_mut().unwrap(); + object.retain(|key, _| { + matches!( + key.as_str(), + "album_name" + | "album_artist" + | "cover_url" + | "release_date" + | "track_number" + | "total_tracks" + | "disc_number" + | "total_discs" + | "duration_ms" + | "isrc" + | "genre" + | "label" + | "copyright" + | "composer" + | "comment" + | "explicit" + | "album_type" + | "upc" + ) + }); + object.insert("id".into(), request.provider_track_id.clone().into()); + object.insert("name".into(), request.track_name.clone().into()); + object.insert("artists".into(), request.artist_name.clone().into()); + track +} + +fn download_metadata_fields( + request: &DownloadRequest, + response: &Value, +) -> BTreeMap<String, String> { + let mut fields = BTreeMap::new(); + for (tag, field, fallback) in [ + ("TITLE", "title", &request.track_name), + ("ARTIST", "artist", &request.artist_name), + ("ALBUM", "album", &request.album_name), + ("ALBUMARTIST", "album_artist", &request.album_artist), + ("DATE", "release_date", &request.release_date), + ("ISRC", "isrc", &request.isrc), + ("GENRE", "genre", &request.genre), + ("ORGANIZATION", "label", &request.label), + ("COPYRIGHT", "copyright", &request.copyright), + ("COMPOSER", "composer", &request.composer), + ("COMMENT", "comment", &request.comment), + ("BARCODE", "upc", &request.upc), + ] { + let value = [text(response, field), fallback.as_str()] + .into_iter() + .map(str::trim) + .find(|value| !value.is_empty()) + .unwrap_or_default(); + if !value.is_empty() { + fields.insert(tag.into(), value.into()); + } + } + let positive = |field: &str, fallback: i64| { + response[field] + .as_i64() + .filter(|value| *value > 0) + .unwrap_or(fallback.max(0)) + }; + for (tag, number, total) in [ + ( + "TRACKNUMBER", + positive("track_number", request.track_number), + positive("total_tracks", request.total_tracks), + ), + ( + "DISCNUMBER", + positive("disc_number", request.disc_number), + positive("total_discs", request.total_discs), + ), + ] { + if number > 0 { + fields.insert( + tag.into(), + if total > 0 { + format!("{number}/{total}") + } else { + number.to_string() + }, + ); + } + } + if request.embed_lyrics && !text(response, "lyrics_lrc").is_empty() { + for tag in ["LYRICS", "UNSYNCEDLYRICS"] { + fields.insert(tag.into(), text(response, "lyrics_lrc").into()); + } + } + if response["explicit"] == true || request.explicit { + fields.insert("ITUNESADVISORY".into(), "1".into()); + } + let album_type = [text(response, "album_type"), &request.album_type] + .into_iter() + .map(str::trim) + .find(|value| !value.is_empty()) + .unwrap_or_default(); + if !album_type.is_empty() { + fields.insert( + "RELEASETYPE".into(), + spotiflac_core::matching::lowercase(album_type), + ); + if album_type.eq_ignore_ascii_case("compilation") { + fields.insert("COMPILATION".into(), "1".into()); + } + } + fields +} + +fn apply_metadata(request: &mut DownloadRequest, track: &Value, source: bool) { + let mut value = serde_json::to_value(&*request).expect("download request serialization"); + for (target, key) in [ + ("track_name", "name"), + ("artist_name", "artists"), + ("album_name", "album_name"), + ("album_artist", "album_artist"), + ("spotify_id", "id"), + ("isrc", "isrc"), + ("cover_url", "cover_url"), + ("release_date", "release_date"), + ("genre", "genre"), + ("label", "label"), + ("copyright", "copyright"), + ("composer", "composer"), + ("comment", "comment"), + ("album_type", "album_type"), + ("upc", "upc"), + ("tidal_id", "tidal_id"), + ("qobuz_id", "qobuz_id"), + ("deezer_id", "deezer_id"), + ] { + if !source + && matches!( + target, + "track_name" + | "artist_name" + | "spotify_id" + | "comment" + | "tidal_id" + | "qobuz_id" + | "deezer_id" + ) + { + continue; + } + if (text(&value, target).is_empty() + || (source && matches!(target, "isrc" | "tidal_id" | "qobuz_id" | "deezer_id"))) + && !text(track, key).is_empty() + { + value[target] = track[key].clone(); + } + } + for key in [ + "duration_ms", + "track_number", + "total_tracks", + "disc_number", + "total_discs", + ] { + if !source && key == "duration_ms" { + continue; + } + if value[key] == 0 && track[key].as_i64().is_some_and(|number| number > 0) { + value[key] = track[key].clone(); + } + } + if track["explicit"] == true { + value["explicit"] = true.into(); + } + *request = serde_json::from_value(value).expect("typed download metadata"); +} + +fn preferred_track_id( + request: &DownloadRequest, + manifest: &ExtensionManifest, + explicit: &str, +) -> String { + if !explicit.trim().is_empty() { + return explicit.trim().into(); + } + let replaces = manifest + .capabilities + .get("replacesBuiltInProviders") + .and_then(Value::as_array); + for (provider, native) in [ + ("tidal", &request.tidal_id), + ("qobuz", &request.qobuz_id), + ("deezer", &request.deezer_id), + ("spotify", &request.spotify_id), + ] { + if replaces.is_some_and(|values| { + values.iter().any(|value| { + value + .as_str() + .is_some_and(|id| id.trim().eq_ignore_ascii_case(provider)) + }) + }) { + if provider != "spotify" && !native.trim().is_empty() { + return native.trim().into(); + } + let id = request.spotify_id.trim(); + if !id.is_empty() { + let prefix = format!("{provider}:"); + return if id.to_lowercase().starts_with(&prefix) { + id[prefix.len()..].into() + } else { + id.into() + }; + } + } + } + [ + &request.spotify_id, + &request.tidal_id, + &request.qobuz_id, + &request.deezer_id, + ] + .into_iter() + .map(|value| value.trim()) + .find(|value| !value.is_empty()) + .unwrap_or_default() + .into() +} + +fn filename( + request: &DownloadRequest, + extension: &str, + check: &Check<'_>, +) -> Result<String, String> { + let mut metadata = host_track(request).as_object().cloned().unwrap(); + for (key, value) in [ + ("title", json!(request.track_name)), + ("artist", json!(request.artist_name)), + ("album", json!(request.album_name)), + ("track", json!(request.track_number)), + ("disc", json!(request.disc_number)), + ("date", json!(request.release_date)), + ("playlist_position", json!(request.playlist_position)), + ("provider", json!(request.download_provider)), + ("provider_id", json!(request.provider_track_id)), + ("quality", json!(request.quality)), + ("quality_variant", json!(request.quality_variant)), + ] { + metadata.insert(key.into(), value); + } + let name = build_filename_checked(&request.filename_format, &metadata, 1024 * 1024, check)?; + let name = if name.trim().is_empty() { + format!("{} - {}", request.artist_name, request.track_name) + } else { + name + }; + let extension = if extension.trim().is_empty() { + ".flac" + } else { + extension.trim() + }; + Ok(format!( + "{}{}{}", + sanitize_filename_preserving_token(&name, &request.quality_variant), + if extension.starts_with('.') { "" } else { "." }, + extension + )) +} + +fn album_folder(request: &DownloadRequest, album: &str) -> String { + if !request.album_folder_template.contains("{album}") || album.trim().is_empty() { + return String::new(); + } + let name: String = request + .album_folder_template + .replace("{album}", album) + .chars() + .filter_map(|ch| { + if ch < ' ' || ch == '\u{7f}' { + None + } else if "<>:\"/\\|?*".contains(ch) { + Some(' ') + } else { + Some(ch) + } + }) + .collect(); + let mut name = name + .trim_matches(['.', ' ']) + .split_whitespace() + .collect::<Vec<_>>() + .join(" "); + while name.contains("__") { + name = name.replace("__", "_"); + } + let name = name.trim_matches(['_', ' ']); + let mut end = name.len().min(120); + while !name.is_char_boundary(end) { + end -= 1; + } + name[..end].trim_matches(['.', '_', ' ']).into() +} + +fn output_path( + request: &DownloadRequest, + temporary: &Path, + album: &str, + check: &Check<'_>, +) -> Result<String, String> { + if !request.output_path.is_empty() { + return Ok(request.output_path.clone()); + } + let mut directory = if request.output_dir.is_empty() { + temporary.to_owned() + } else { + PathBuf::from(&request.output_dir) + }; + let folder = album_folder(request, album); + if !request.output_dir.is_empty() && !folder.is_empty() { + directory = crate::files::clean(&directory) + .parent() + .unwrap_or(Path::new(".")) + .join(folder); + } + Ok(directory + .join(filename(request, &request.output_ext, check)?) + .to_string_lossy() + .into_owned()) +} + +fn probe_result( + files: &crate::files::ExtensionFiles, + result: &mut Value, + album_pending: bool, + check: &Check<'_>, +) -> Result<(), String> { + let path = text(result, "file_path").to_owned(); + let file = files + .resolve_legacy(&path)? + .open(OpenOptions::new().read(true)); + if let Ok(mut file) = file { + let mut header = [0; 8]; + // Quality is probed only for FLAC/MP4. Missing album folders may use + // tags from any supported container, including undecoded audio. + let quality = file.read_exact(&mut header).is_ok() + && (&header[..4] == b"fLaC" || &header[4..] == b"ftyp"); + let album = album_pending && text(result, "album").trim().is_empty(); + if (quality || album) + && file.seek(SeekFrom::Start(0)).is_ok() + && let Ok(metadata) = + spotiflac_core::tags::read_file_metadata(&mut file, &path, "", check) + { + if quality { + for key in ["bit_depth", "sample_rate", "audio_codec"] { + result[key] = metadata[key].clone(); + } + } + if album { + result["album"] = text(&metadata, "album").trim().into(); + } + } + } + check() +} + +fn matches_track(request: &DownloadRequest, result: &Value, skip_names: bool) -> bool { + let exact = !request.isrc.is_empty() + && !text(result, "isrc").is_empty() + && request + .isrc + .trim() + .eq_ignore_ascii_case(text(result, "isrc").trim()); + let identity = reenrich::Request { + track_name: request.track_name.clone(), + artist_name: request.artist_name.clone(), + isrc: request.isrc.clone(), + duration_ms: request.duration_ms, + ..Default::default() + }; + let mut candidate = result.clone(); + candidate["name"] = result.get("title").cloned().unwrap_or(json!("")); + candidate["artists"] = result.get("artist").cloned().unwrap_or(json!("")); + // Skipping names does not permit truncated previews or a duration mismatch. + if skip_names && request.duration_ms / 1000 <= 0 { + return true; + } + if skip_names + && (result["duration_ms"].as_i64().unwrap_or_default() / 1000 <= 0 + || (request.duration_ms / 1000) + .abs_diff(result["duration_ms"].as_i64().unwrap_or_default() / 1000) + <= 10) + { + return true; + } + if !identity.verified(&candidate) { + return false; + } + if !exact + && !skip_names + && !request.album_name.is_empty() + && !text(result, "album").is_empty() + && !titles_match(&request.album_name, text(result, "album")) + { + if !request.isrc.is_empty() && !text(result, "isrc").is_empty() { + return false; + } + return strong_identity(request, result); + } + true +} + +fn exact_identity(expected: &str, found: &str, normalize: fn(&str) -> String) -> bool { + let (left, right) = (normalize(expected), normalize(found)); + if !left.is_empty() && !right.is_empty() { + left == right + } else { + lowercase(expected.trim()) == lowercase(found.trim()) + } +} + +fn duration_matches(request: &DownloadRequest, track: &Value) -> bool { + let expected = request.duration_ms / 1000; + let actual = track["duration_ms"].as_i64().unwrap_or_default() / 1000; + expected > 0 && actual > 0 && expected.abs_diff(actual) <= 10 +} + +fn strong_identity(request: &DownloadRequest, track: &Value) -> bool { + let title = text(track, "title"); + let artist = text(track, "artist"); + !request.track_name.is_empty() + && !title.is_empty() + && !request.artist_name.is_empty() + && !artist.is_empty() + && exact_identity(&request.track_name, title, track_identity_title) + && (exact_identity(&request.artist_name, artist, normalize_loose_artist) + || (artists_match(&request.artist_name, artist) && duration_matches(request, track))) +} + +fn select_download_metadata<'a>( + request: &DownloadRequest, + tracks: &'a [Value], +) -> Option<&'a Value> { + let mut best = None; + let mut best_score = i32::MIN; + for track in tracks { + let isrc = text(track, "isrc").trim(); + let expected = request.isrc.trim(); + let exact = !expected.is_empty() && !isrc.is_empty() && expected.eq_ignore_ascii_case(isrc); + if !expected.is_empty() && !isrc.is_empty() && !exact { + continue; + } + let mut candidate = track.clone(); + for (target, source) in [ + ("title", "name"), + ("artist", "artists"), + ("album", "album_name"), + ] { + candidate[target] = text(track, source).into(); + } + if !matches_track(request, &candidate, false) + || (!exact && !strong_identity(request, &candidate)) + { + continue; + } + let score = 2000 + + i32::from(exact) * 10000 + + i32::from(exact_identity( + &request.track_name, + text(track, "name"), + normalize_title, + )) * 400 + + i32::from(exact_identity( + &request.artist_name, + text(track, "artists"), + normalize_loose_artist, + )) * 320 + + i32::from( + !request.album_name.is_empty() + && !text(track, "album_name").is_empty() + && titles_match(&request.album_name, text(track, "album_name")), + ) * 120 + + i32::from(duration_matches(request, track)) * 80 + + i32::from(!text(track, "isrc").is_empty()) * 40 + + i32::from(!text(track, "album_name").is_empty()) * 30 + + i32::from(!text(track, "release_date").is_empty()) * 30 + + i32::from( + track["track_number"] + .as_i64() + .is_some_and(|number| number > 0), + ) * 10; + if score > best_score { + best_score = score; + best = Some(track); + } + } + best +} + +fn success( + request: &DownloadRequest, + result: &Value, + path: &str, + exists: bool, + manifest: &ExtensionManifest, + check: &Check<'_>, +) -> Result<Value, String> { + let mut response = json!({"success":true,"message":if exists { "File already exists".into() } else { format!("Downloaded from {}", manifest.name) }, + "file_path":path,"provider_track_id":request.provider_track_id,"already_exists":exists,"service":manifest.name, + "actual_bit_depth":result["bit_depth"],"actual_sample_rate":result["sample_rate"], + "audio_codec":text(result, "audio_codec").trim(),"actual_container":text(result, "actual_container").trim(), + "requires_container_conversion":result["requires_container_conversion"] == true, + "skip_metadata_enrichment":manifest.skip_metadata_enrichment, + "explicit":request.explicit || result["explicit"] == true}); + for (key, requested, prefer_request) in [ + ("title", request.track_name.trim(), true), + ("artist", request.artist_name.as_str(), false), + ("album", request.album_name.trim(), true), + ("album_artist", request.album_artist.as_str(), true), + ("release_date", request.release_date.trim(), true), + ("cover_url", request.cover_url.trim(), true), + ("isrc", request.isrc.as_str(), false), + ("genre", request.genre.as_str(), false), + ("label", request.label.as_str(), false), + ("copyright", request.copyright.as_str(), false), + ("composer", request.composer.as_str(), false), + ("comment", request.comment.trim(), true), + ("album_type", request.album_type.as_str(), false), + ("upc", request.upc.as_str(), false), + ] { + let provided = text(result, key); + response[key] = if (prefer_request && !requested.is_empty()) || provided.is_empty() { + requested + } else { + provided + } + .into(); + } + for (key, value) in [ + ("track_number", request.track_number), + ("disc_number", request.disc_number), + ("total_tracks", request.total_tracks), + ("total_discs", request.total_discs), + ] { + response[key] = if value != 0 { + json!(value) + } else { + result.get(key).cloned().unwrap_or(json!(0)) + }; + } + for key in ["lyrics_lrc", "decryption_key", "decryption"] { + if let Some(value) = result.get(key) { + response[key] = value.clone(); + } + } + let extension = [ + text(result, "actual_extension"), + text(result, "output_extension"), + ] + .into_iter() + .map(|value| value.trim().to_lowercase()) + .find(|value| !value.is_empty()) + .unwrap_or_default(); + if !extension.is_empty() { + let extension = format!(".{}", extension.trim_start_matches('.')); + response["actual_extension"] = if extension == ".mp4" { + ".m4a".into() + } else { + extension.into() + }; + } + let mut named = request.clone(); + if !text(result, "isrc").trim().is_empty() { + named.isrc = text(result, "isrc").trim().into(); + } + let extension = if text(&response, "actual_extension").is_empty() { + Path::new(path) + .extension() + .unwrap_or_default() + .to_string_lossy() + .into_owned() + } else { + text(&response, "actual_extension").into() + }; + response["resolved_file_name"] = filename(&named, &extension, check)?.into(); + response["resolved_album_folder"] = album_folder(request, text(&response, "album")).into(); + Ok(response) +} + +fn compact(mut value: Value) -> Value { + value.as_object_mut().unwrap().retain(|key, value| { + matches!(key.as_str(), "success" | "message") + || !(value.is_null() || value == "" || value == false || value == 0) + }); + value +} + +fn failure(service: &str, message: &str, kind: &str, retry: i64) -> Value { + let mut response = native_error_response(message); + if !kind.is_empty() { + response["error_type"] = kind.into(); + } + response["service"] = service.into(); + response["retry_after_seconds"] = retry.into(); + response +} + +fn stopped(service: &str, availability: &Value, message: &str) -> Value { + let reason = if !text(availability, "reason").trim().is_empty() { + text(availability, "reason").trim() + } else if !message.is_empty() { + message + } else { + "extension requested no further fallback" + }; + let kind = native_error_response(reason); + failure( + service, + &format!("Fallback stopped by {service}: {reason}"), + if kind["error_type"] == "unknown" { + "extension_error" + } else { + text(&kind, "error_type") + }, + 0, + ) +} + +fn storage_failure(kind: &str, message: &str) -> bool { + kind.eq_ignore_ascii_case("permission") + || [ + "operation not permitted", + "permission denied", + "read-only file system", + "failed to create file", + "failed to create directory", + ] + .into_iter() + .any(|part| message.to_lowercase().contains(part)) +} + +#[cfg(test)] +#[path = "download_latency_tests.rs"] +mod tests; diff --git a/rust_backend/crates/extensions/src/backend/health.rs b/rust_backend/crates/extensions/src/backend/health.rs new file mode 100644 index 00000000..57388248 --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/health.rs @@ -0,0 +1,685 @@ +//! Health snapshots share the installed manager's revision and HTTP transport. + +use super::Backend; +use crate::manager::ExtensionManager; +use crate::manifest::{ExtensionManifest, HealthCheck}; +use serde::Serialize; +use serde_json::Value; +use spotiflac_core::app_version::AppVersion; +use spotiflac_network::{ + HttpRequest, NetworkService, policy::NetworkPermissions, policy::private_literal_or_local, + url::UrlParts, +}; +use spotiflac_providers::resolver::Check; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +#[derive(Clone, Serialize)] +struct ResultSnapshot { + extension_id: String, + status: String, + checked_at: String, + checks: Vec<CheckResult>, +} + +#[derive(Clone, Serialize)] +struct CheckResult { + id: String, + #[serde(skip_serializing_if = "String::is_empty")] + label: String, + url: String, + method: String, + #[serde(skip_serializing_if = "String::is_empty")] + service_key: String, + required: bool, + status: String, + #[serde(skip_serializing_if = "is_zero")] + http_status: u16, + latency_ms: u128, + #[serde(skip_serializing_if = "String::is_empty")] + message: String, + #[serde(skip_serializing_if = "String::is_empty")] + error: String, + checked_at: String, +} + +fn is_zero(value: &u16) -> bool { + *value == 0 +} +fn now() -> String { + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) +} + +struct Cached { + revision: u64, + result: ResultSnapshot, + expires: Instant, +} + +#[derive(Default)] +struct State { + entries: BTreeMap<String, Cached>, + refreshing: BTreeMap<String, u64>, + workers: Vec<JoinHandle<()>>, +} + +pub(super) struct Health { + network: Arc<NetworkService>, + version: AppVersion, + state: Mutex<State>, + closed: AtomicBool, +} + +impl Health { + pub fn new(network: &Arc<NetworkService>, version: AppVersion) -> Self { + Self { + network: network.clone(), + version, + state: Mutex::default(), + closed: AtomicBool::new(false), + } + } + + pub fn clear_memory_cache(&self) { + self.state + .lock() + .expect("health state lock") + .entries + .clear(); + } + + pub fn shutdown(&self) { + self.closed.store(true, Ordering::Release); + let workers = { + let mut state = self.state.lock().expect("health state lock"); + state.entries.clear(); + std::mem::take(&mut state.workers) + }; + for worker in workers { + let _ = worker.join(); + } + } + + fn fetch( + &self, + manager: &ExtensionManager, + manifest: &ExtensionManifest, + revision: u64, + check: &Check<'_>, + ) -> Result<ResultSnapshot, String> { + let check = || { + check()?; + if self.closed.load(Ordering::Acquire) || manager.environment().is_closed() { + return Err("health service closed".into()); + } + if manager.metadata_revision() != revision { + return Err("extension changed during health check".into()); + } + Ok(()) + }; + check()?; + let mut result = ResultSnapshot { + extension_id: manifest.name.clone(), + status: "unsupported".into(), + checked_at: now(), + checks: Vec::new(), + }; + if manifest.service_health.is_empty() { + return Ok(result); + } + result.status = "online".into(); + for health_check in &manifest.service_health { + let entry = self.run_check(manifest, health_check, &check)?; + result.status = match ( + entry.status.as_str(), + entry.required, + result.status.as_str(), + ) { + ("offline", true, _) => "offline", + ("offline", false, "online") | ("degraded", _, "online") => "degraded", + ("unknown", _, "online") => "unknown", + _ => &result.status, + } + .into(); + result.checks.push(entry); + } + check()?; + let ttl = manifest + .service_health + .iter() + .filter(|check| check.cache_ttl_seconds > 0) + .map(|check| (check.cache_ttl_seconds as u64).clamp(60, 600)) + .min() + .unwrap_or(600); + let ttl = if result.status == "unknown" { + ttl.min(120) + } else { + ttl + }; + let mut state = self.state.lock().expect("health state lock"); + check()?; + state.entries.retain(|_, entry| entry.revision == revision); + state.entries.insert( + manifest.name.clone(), + Cached { + revision, + result: result.clone(), + expires: Instant::now() + Duration::from_secs(ttl), + }, + ); + Ok(result) + } + + fn fallback_status(self: &Arc<Self>, manager: &Arc<ExtensionManager>, id: &str) -> String { + let revision = manager.metadata_revision(); + let Ok(manifest) = manager.health_manifest(id) else { + return "unknown".into(); + }; + if manifest.service_health.is_empty() { + return "unknown".into(); + } + let mut state = self.state.lock().expect("health state lock"); + if self.closed.load(Ordering::Acquire) { + return "unknown".into(); + } + state.entries.retain(|_, entry| entry.revision == revision); + let mut status = "unknown".to_owned(); + if let Some(entry) = state.entries.get(id) { + if Instant::now() < entry.expires { + return entry.result.status.clone(); + } + if entry.result.status != "offline" { + status.clone_from(&entry.result.status); + } + } + if state.refreshing.get(id) == Some(&revision) { + return status; + } + // Reap completed native threads, retaining handles for shutdown to join. + let mut index = 0; + while index < state.workers.len() { + if state.workers[index].is_finished() { + let _ = state.workers.swap_remove(index).join(); + } else { + index += 1; + } + } + let health = self.clone(); + let manager = manager.clone(); + let id = id.to_owned(); + let running_id = id.clone(); + state.refreshing.insert(id.clone(), revision); + match std::thread::Builder::new() + .name("extension-health".into()) + .spawn(move || { + let _ = health.fetch(&manager, &manifest, revision, &|| Ok(())); + let mut state = health.state.lock().expect("health state lock"); + if state.refreshing.get(&running_id) == Some(&revision) { + state.refreshing.remove(&running_id); + } + }) { + Ok(worker) => state.workers.push(worker), + Err(_) => { + state.refreshing.remove(&id); + } + } + status + } + + fn run_check( + &self, + manifest: &ExtensionManifest, + input: &HealthCheck, + check: &Check<'_>, + ) -> Result<CheckResult, String> { + check()?; + let mut method = input.method.trim().to_uppercase(); + if method.is_empty() { + method = "GET".into(); + } + let mut result = CheckResult { + id: input.id.clone(), + label: input.label.clone(), + url: input.url.clone(), + method: method.clone(), + service_key: input.service_key.trim().into(), + required: input.required, + status: "unknown".into(), + http_status: 0, + latency_ms: 0, + message: String::new(), + error: String::new(), + checked_at: now(), + }; + let permissions = NetworkPermissions { + domains: manifest.permissions.network.clone().unwrap_or_default(), + allow_http: false, + }; + let parsed = UrlParts::parse(&input.url); + let invalid = match &parsed { + None => Some("invalid health URL".to_owned()), + Some(url) if url.scheme != "https" => Some("health check must use https".into()), + Some(url) if url.hostname.is_empty() => { + Some("health check URL hostname is required".into()) + } + Some(url) if private_literal_or_local(&url.hostname) => { + Some("private/local health check host is not allowed".into()) + } + Some(url) if !permissions.allows_domain(&url.hostname) => Some(format!( + "health check host '{}' is not in extension network permissions", + url.hostname + )), + _ if method != "GET" && method != "HEAD" => { + Some("health check method must be GET or HEAD".into()) + } + _ => None, + }; + if let Some(error) = invalid { + result.status = "offline".into(); + result.error = error; + return Ok(result); + } + let timeout = Duration::from_millis(if input.timeout_ms > 0 { + input.timeout_ms as u64 + } else { + 4000 + }); + let start = Instant::now(); + let network_check = || { + check()?; + if start.elapsed() >= timeout { + Err("HTTP request timeout exceeded".into()) + } else { + Ok(()) + } + }; + let session = self.network.session(permissions, timeout); + let response = session.open_response_stream( + HttpRequest { + url: input.url.clone(), + method: method.clone(), + body: String::new(), + headers: BTreeMap::from([("Accept".into(), "application/json".into())]), + default_json: false, + user_agent: if parsed + .as_ref() + .is_some_and(|url| url.hostname.eq_ignore_ascii_case("api.zarz.moe")) + { + self.version.user_agent() + } else { + crate::utility_host::random_user_agent() + }, + }, + network_check, + ); + result.latency_ms = start.elapsed().as_millis(); + check()?; + let mut response = match response { + Ok(response) => response, + Err(error) => { + result.status = if transient_transport(&error) { + "unknown" + } else { + "offline" + } + .into(); + result.error = error; + return Ok(result); + } + }; + result.http_status = response.response.status; + result.message = format!( + "{} {}", + response.response.status, response.response.status_text + ); + if !(200..300).contains(&result.http_status) { + result.status = "offline".into(); + return Ok(result); + } + if method == "HEAD" { + result.status = "online".into(); + return Ok(result); + } + let mut body = Vec::new(); + let mut buffer = [0; 8192]; + while body.len() < 64 * 1024 { + let limit = buffer.len().min(64 * 1024 - body.len()); + match response.read(&mut buffer[..limit], network_check) { + Ok(0) => break, + Ok(count) => body.extend_from_slice(&buffer[..count]), + Err(error) => { + check()?; + result.status = "degraded".into(); + result.error = error; + result.message.clear(); + return Ok(result); + } + } + } + check()?; + let (status, message) = classify_body(&body, &input.service_key); + result.status = status.into(); + if !message.is_empty() { + result.message = message; + } + Ok(result) + } +} + +impl Backend { + pub fn check_extension_health_json( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + let _operation = self.enter()?; + let revision = self.manager.metadata_revision(); + let manifest = self + .manager + .health_manifest(id) + .map_err(|error| error.to_string())?; + let result = self.health.fetch(&self.manager, &manifest, revision, &|| { + self.check().and_then(|()| check()) + })?; + serde_json::to_string(&result).map_err(|error| error.to_string()) + } + + pub(super) fn prioritize_healthy_downloads( + &self, + priority: Vec<String>, + protected: &str, + selected: &str, + ) -> Result<Vec<String>, String> { + let mut groups = [Vec::new(), Vec::new(), Vec::new()]; + for id in priority { + let id = id.trim(); + if id.is_empty() { + continue; + } + let status = if id.eq_ignore_ascii_case(protected) + || !self + .fallback_allowed(id) + .map_err(|error| error.to_string())? + || !self + .download_manifest(id) + .is_ok_and(|manifest| manifest.has_type("download_provider")) + { + "unknown".into() + } else { + self.health.fallback_status(&self.manager, id) + }; + let index = match status.as_str() { + "online" => 0, + "degraded" => 1, + "offline" => continue, + _ => 2, + }; + groups[index].push(id.to_owned()); + } + let mut ordered: Vec<_> = groups.into_iter().flatten().collect(); + if let Some(index) = ordered + .iter() + .position(|id| id.eq_ignore_ascii_case(selected)) + { + let id = ordered.remove(index); + ordered.insert(0, id); + } + Ok(ordered) + } +} + +fn transient_message(value: &str) -> bool { + let value = value.trim().to_lowercase(); + [ + "deadline exceeded", + "timeout", + "timed out", + "temporarily unavailable", + "try again", + ] + .iter() + .any(|part| value.contains(part)) +} + +fn transient_transport(value: &str) -> bool { + let value = value.to_lowercase(); + transient_message(&value) + || [ + "dns", + "resolve", + "connect", + "unreachable", + "certificate", + "tls", + "unexpected eof", + "incomplete message", + ] + .iter() + .any(|part| value.contains(part)) +} + +fn classify_body(body: &[u8], key: &str) -> (&'static str, String) { + let Ok(payload) = serde_json::from_slice::<Value>(body) else { + return ("online", String::new()); + }; + let key = key.trim(); + if !key.is_empty() + && let Some(services) = payload["services"].as_object() + { + let Some(service) = services.get(key) else { + return ("unknown", format!("service '{key}' not found")); + }; + if !service.is_object() { + return ( + "unknown", + format!("service '{key}' has invalid health payload"), + ); + } + let text = |field: &str| service[field].as_str().unwrap_or_default(); + let message = [text("label"), text("detail"), text("error")] + .into_iter() + .map(str::trim) + .filter(|part| !part.is_empty()) + .collect::<Vec<_>>() + .join(": "); + let transient = [text("detail"), text("error"), text("label")] + .into_iter() + .any(transient_message); + let ok = service["ok"].as_bool(); + let status = if let Some(code) = service["status"].as_f64() { + let code = code as i64; + if (200..300).contains(&code) || (code == 500 && ok == Some(true)) { + "online" + } else if matches!(code, 401 | 403) { + "degraded" + } else if transient || matches!(code, 408 | 429 | 502 | 503 | 504) { + "unknown" + } else { + "offline" + } + } else if matches!( + text("detail").trim().to_lowercase().as_str(), + "auth_required" | "authorization_required" | "login_required" | "unauthorized" + ) { + "degraded" + } else if transient { + "unknown" + } else if let Some(ok) = ok { + if ok { "online" } else { "offline" } + } else { + match text("status").trim().to_lowercase().as_str() { + "ok" | "up" | "online" | "healthy" | "operational" => "online", + "degraded" | "partial" | "warning" | "warn" => "degraded", + "down" | "offline" | "error" | "failed" | "fail" | "unhealthy" => "offline", + _ => "unknown", + } + }; + return (status, message); + } + let raw = payload["status"].as_str().unwrap_or_default(); + let status = match raw.trim().to_lowercase().as_str() { + "degraded" | "partial" | "warning" | "warn" => "degraded", + "down" | "offline" | "error" | "failed" | "fail" | "unhealthy" => { + if transient_message(&String::from_utf8_lossy(body)) { + "unknown" + } else { + "offline" + } + } + _ => "online", + }; + (status, raw.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::RuntimeLimits; + use crate::backend::MetadataOptions; + use crate::environment::ExtensionEnvironment; + use serde_json::json; + use spotiflac_network::{Lookup, LookupFuture, NetworkOptions}; + use std::sync::atomic::AtomicUsize; + + struct PendingLookup(Arc<AtomicUsize>); + impl Lookup for PendingLookup { + fn lookup(&self, _: &str) -> LookupFuture { + self.0.fetch_add(1, Ordering::AcqRel); + Box::pin(std::future::pending()) + } + } + + #[test] + fn health_ordering_recovers_stale_offline_coalesces_and_joins_refresh() { + let root = tempfile::tempdir().unwrap(); + let lookups = Arc::new(AtomicUsize::new(0)); + let network = NetworkService::with_options(NetworkOptions { + lookup: Arc::new(PendingLookup(lookups.clone())), + doh_upstreams: Vec::new(), + ..Default::default() + }) + .unwrap(); + let environment = ExtensionEnvironment::with_network( + &root.path().join("data"), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "1", + network, + ) + .unwrap(); + let backend = Backend::with_environment( + &root.path().join("sources"), + environment, + RuntimeLimits::default(), + MetadataOptions::default(), + ) + .unwrap(); + let ids = [ + "example.selected", + "example.offline", + "example.unknown", + "example.degraded", + "example.online", + ]; + for id in ids { + let source = root.path().join("sources").join(id); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join("manifest.json"), json!({"name":id,"version":"1","description":"Generic health fixture","type":["download_provider"], + "permissions":{"network":["example.com"]},"serviceHealth":[{"id":"status","url":"https://example.com/status","required":true}]}).to_string()).unwrap(); + std::fs::write(source.join("index.js"), "registerExtension({});").unwrap(); + } + backend.load_all().unwrap(); + for id in ids { + backend.set_enabled(id, true).unwrap(); + } + let revision = backend.metadata_revision(); + { + let mut state = backend.health.state.lock().unwrap(); + for (id, status) in [ + (ids[0], "offline"), + (ids[1], "offline"), + (ids[3], "degraded"), + (ids[4], "online"), + ] { + state.entries.insert( + id.into(), + Cached { + revision, + expires: Instant::now() + Duration::from_secs(60), + result: ResultSnapshot { + extension_id: id.into(), + status: status.into(), + checked_at: now(), + checks: Vec::new(), + }, + }, + ); + } + } + let priority = || ids.iter().map(|id| (*id).to_owned()).collect(); + let first = backend + .prioritize_healthy_downloads(priority(), ids[0], ids[0]) + .unwrap(); + assert_eq!(first, [ids[0], ids[4], ids[3], ids[2]]); + backend + .health + .state + .lock() + .unwrap() + .entries + .get_mut(ids[1]) + .unwrap() + .expires = Instant::now() - Duration::from_secs(1); + let started = Instant::now(); + for _ in 0..4 { + let order = backend + .prioritize_healthy_downloads(priority(), ids[0], ids[0]) + .unwrap(); + assert_eq!(order, [ids[0], ids[4], ids[3], ids[1], ids[2]]); + } + assert!( + started.elapsed() < Duration::from_secs(1), + "fallback waited for unresolved health network work" + ); + assert_eq!(backend.health.state.lock().unwrap().workers.len(), 2); + while lookups.load(Ordering::Acquire) == 0 { + assert!(started.elapsed() < Duration::from_secs(3)); + std::thread::sleep(Duration::from_millis(5)); + } + let cached = backend.health.state.lock().unwrap().entries.len(); + assert!(cached > 0); + backend.release_memory(false).unwrap(); + assert_eq!(backend.health.state.lock().unwrap().entries.len(), cached); + backend.release_memory(true).unwrap(); + { + let state = backend.health.state.lock().unwrap(); + assert!(state.entries.is_empty()); + assert_eq!(state.workers.len(), 2); + assert!(state.workers.iter().all(|worker| !worker.is_finished())); + } + // A manager mutation invalidates all old snapshots and late refreshes. + backend.set_enabled(ids[1], false).unwrap(); + backend.set_enabled(ids[1], true).unwrap(); + assert_eq!( + backend + .prioritize_healthy_downloads(priority(), ids[0], ids[0]) + .unwrap(), + ids + ); + let closed = Instant::now(); + backend.shutdown(); + assert!( + closed.elapsed() < Duration::from_secs(1), + "shutdown did not interrupt pending health DNS" + ); + assert!(backend.health.state.lock().unwrap().workers.is_empty()); + assert!(backend.health.state.lock().unwrap().entries.is_empty()); + assert!( + backend + .check_extension_health_json(ids[0], &|| Ok(())) + .is_err() + ); + } +} diff --git a/rust_backend/crates/extensions/src/backend/library.rs b/rust_backend/crates/extensions/src/backend/library.rs new file mode 100644 index 00000000..dbfed384 --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/library.rs @@ -0,0 +1,431 @@ +use super::Backend; +use cap_std::fs::OpenOptions; +use chrono::{SecondsFormat, Utc}; +use serde_json::{Value, json}; +use spotiflac_core::cue::{self, CueSheet}; +use spotiflac_core::tags::{ + extract_cover, library_extension, library_id, library_metadata, read_library_metadata, + read_library_metadata_with_cover, +}; +use std::cell::RefCell; +use std::path::Path; +use std::time::UNIX_EPOCH; + +mod scan; +pub(super) use scan::ScanState; + +impl Backend { + pub fn set_library_cover_cache_directory(&self, directory: &str) -> Result<(), String> { + let _operation = self.enter()?; + if !directory.is_empty() { + self.environment() + .native_files()? + .resolve_legacy(directory)? + .native_display()?; + } + *self + .library_cover_directory + .lock() + .expect("library cover directory lock") = directory.into(); + Ok(()) + } + + pub fn read_audio_metadata( + &self, + path: &str, + hint: &str, + cache_key: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<Value, String> { + let _operation = self.enter()?; + let failure = RefCell::new(None::<String>); + let check = || { + if let Some(error) = failure.borrow().as_ref() { + return Err(error.clone()); + } + self.check().and_then(|()| check()).inspect_err(|error| { + *failure.borrow_mut() = Some(error.clone()); + }) + }; + check()?; + let files = self.environment().native_files()?; + self.scan_audio_file(&files, path, hint, cache_key, &scan_time(), 0, &check) + } + + #[allow(clippy::too_many_arguments)] + fn scan_audio_file( + &self, + files: &crate::files::ExtensionFiles, + path: &str, + hint: &str, + cache_key: &str, + scan_time: &str, + mod_time: i64, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<Value, String> { + check()?; + let input = files.resolve_legacy(path)?; + let mut file = match input.open_native_read() { + Ok(file) => Some(file), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.to_string()), + }; + let mod_time = if mod_time > 0 { + mod_time + } else { + modified(&input) + }; + let mut metadata = None; + let directory = self + .library_cover_directory + .lock() + .expect("library cover directory lock") + .clone(); + let cover = if directory.is_empty() { + None + } else { + self.save_cover_to_cache_for_scan( + path, + hint, + &directory, + cache_key, + |_, format, check| { + let file = file.as_mut().ok_or("audio file unavailable")?; + if format == "flac" { + let (tags, cover) = read_library_metadata_with_cover( + file, path, hint, scan_time, mod_time, check, + )?; + metadata = Some(tags); + cover.ok_or_else(|| "no cover art found in file".into()) + } else { + extract_cover(file, format, check) + } + }, + check, + ) + .ok() + }; + check()?; + let mut result = match metadata { + Some(metadata) => metadata, + None => match file.as_mut() { + Some(file) => read_library_metadata(file, path, hint, scan_time, mod_time, check)?, + None => library_metadata(path, hint, scan_time, mod_time), + }, + }; + if let Some(cover) = cover.filter(|cover| !cover.is_empty()) { + result["coverPath"] = cover.into(); + } + check()?; + if result["hasLyrics"] == false { + let open = |path: &str| { + let input = files.resolve_legacy(path)?; + input.open_native_read().map_err(|error| error.to_string()) + }; + if let Ok(lyrics) = spotiflac_core::lyrics::file::sidecar(path, &open, check) { + result["hasLyrics"] = + spotiflac_core::lyrics::lrc::has_usable_content(&lyrics).into(); + } + } + check()?; + Ok(result) + } + + pub fn parse_cue_file( + &self, + path: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<CueSheet, String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + check()?; + let input = self.environment().native_files()?.resolve_legacy(path)?; + input.native_display()?; + let mut file = input + .open(OpenOptions::new().read(true)) + .map_err(|error| format!("failed to open cue file: {error}"))?; + cue::parse(&mut file, &check) + } + + fn resolve_cue_audio( + &self, + path: &str, + name: &str, + audio_directory: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<String, String> { + let base = Path::new(path).parent().unwrap_or(Path::new(".")); + let directory = if audio_directory.is_empty() { + base + } else { + Path::new(audio_directory) + }; + let files = self.environment().native_files()?; + let exists = |name: &str| -> Result<Option<String>, String> { + check()?; + let candidate = crate::files::clean(&directory.join(name.trim_start_matches('/'))); + let candidate = candidate.to_string_lossy(); + let input = files.resolve_legacy(&candidate)?; + input.native_display()?; + Ok(input.metadata().is_ok().then(|| candidate.into_owned())) + }; + if let Some(path) = exists(name)? { + return Ok(path); + } + let suffix = library_extension(name, ""); + let stem = if suffix.is_empty() { + name + } else { + &name[..name.len() - suffix.len() - 1] + }; + let extensions = [ + "flac", "wav", "aiff", "aif", "ape", "mp3", "ogg", "wv", "m4a", + ]; + for extension in extensions { + for extension in [extension.to_owned(), extension.to_uppercase()] { + if let Some(path) = exists(&format!("{stem}.{extension}"))? { + return Ok(path); + } + } + } + let stem = Path::new(path) + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or_default(); + for extension in extensions { + if let Some(path) = exists(&format!("{stem}.{extension}"))? { + return Ok(path); + } + } + let directory_path = files.resolve_legacy(&directory.to_string_lossy())?; + directory_path.native_display()?; + if let Ok(entries) = directory_path.entries() { + let mut candidates = entries.into_iter().filter(|(name, directory)| { + !directory + && matches!( + library_extension(name, "").as_str(), + "flac" | "wav" | "ape" | "mp3" | "ogg" | "wv" | "m4a" | "aiff" + ) + }); + if let Some((name, _)) = candidates.next() + && candidates.next().is_none() + && let Some(path) = exists(&name)? + { + return Ok(path); + } + } + check()?; + Err(format!( + "audio file not found for cue: {path} (referenced: {name})" + )) + } + + pub fn parse_cue_file_json( + &self, + path: &str, + audio_directory: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<Value, String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + let sheet = self + .parse_cue_file(path, &check) + .map_err(|error| format!("failed to parse cue file: {error}"))?; + let audio = self + .resolve_cue_audio(path, &sheet.file_name, audio_directory, &check) + .map_err(|error| { + error.replace( + "audio file not found for cue:", + "audio file not found for cue sheet:", + ) + })?; + let tracks: Vec<Value> = sheet + .tracks + .iter() + .enumerate() + .map(|(index, track)| { + let mut value = json!({"number":track.number,"title":track.title, + "artist":prefer(&track.performer, &sheet.performer),"start_sec":track.start_time, + "end_sec":next_start(&sheet, index).unwrap_or(-1.0)}); + optional(&mut value, "isrc", &track.isrc); + optional( + &mut value, + "composer", + prefer(&track.composer, &sheet.composer), + ); + value + }) + .collect(); + check()?; + let mut result = json!({"cue_path":path,"audio_path":audio,"album":sheet.title,"artist":sheet.performer,"tracks":tracks}); + optional(&mut result, "genre", &sheet.genre); + optional(&mut result, "date", &sheet.date); + Ok(result) + } + + #[allow(clippy::too_many_arguments)] + pub fn scan_cue_file_for_library( + &self, + path: &str, + audio_directory: &str, + virtual_prefix: &str, + mod_time: i64, + cache_key: &str, + scan_time: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<Value, String> { + let _operation = self.enter()?; + let failure = RefCell::new(None::<String>); + let check = || { + if let Some(error) = failure.borrow().as_ref() { + return Err(error.clone()); + } + self.check().and_then(|()| check()).inspect_err(|error| { + *failure.borrow_mut() = Some(error.clone()); + }) + }; + let sheet = self.parse_cue_file(path, &check)?; + let audio = self.resolve_cue_audio(path, &sheet.file_name, audio_directory, &check)?; + self.scan_cue_sheet( + path, + &sheet, + &audio, + virtual_prefix, + mod_time, + cache_key, + scan_time, + &check, + ) + } + + #[allow(clippy::too_many_arguments)] + fn scan_cue_sheet( + &self, + path: &str, + sheet: &CueSheet, + audio: &str, + virtual_prefix: &str, + mod_time: i64, + cache_key: &str, + scan_time: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<Value, String> { + let files = self.environment().native_files()?; + let input = files.resolve_legacy(audio)?; + input.native_display()?; + let format = library_extension(audio, ""); + let mut depth = 0; + let mut rate = 0; + let mut duration = 0.0; + if let Ok(mut file) = input.open(OpenOptions::new().read(true)) { + if format == "flac" { + if let Ok(quality) = spotiflac_core::media::probe_quality(&mut file, check) { + depth = quality.bit_depth; + rate = quality.sample_rate; + if rate > 0 && quality.total_samples > 0 { + duration = quality.total_samples as f64 / rate as f64; + } + } + } else if format == "mp3" + && let Ok(metadata) = + spotiflac_core::tags::read_file_metadata(&mut file, audio, "", check) + { + rate = metadata["sample_rate"].as_i64().unwrap_or_default(); + duration = metadata["duration"].as_f64().unwrap_or_default(); + } + } + check()?; + let directory = self + .library_cover_directory + .lock() + .expect("library cover directory lock") + .clone(); + let cover = if directory.is_empty() { + String::new() + } else { + self.save_cover_to_cache_with_hint_and_key(audio, "", &directory, cache_key, check) + .unwrap_or_default() + }; + check()?; + let mod_time = if mod_time > 0 { + mod_time + } else { + modified(&files.resolve_legacy(path)?) + }; + let prefix = prefer(virtual_prefix, path); + let mut results = Vec::with_capacity(sheet.tracks.len()); + for (index, track) in sheet.tracks.iter().enumerate() { + check()?; + let end = next_start(sheet, index).or_else(|| (duration > 0.0).then_some(duration)); + let mut result = json!({ + "id":library_id(&format!("{prefix}#track{}", track.number)), + "filePath":format!("{prefix}#track{:02}", track.number), + "trackName":if track.title.is_empty() { format!("Track {:02}", track.number) } else { track.title.clone() }, + "artistName":prefer(prefer(&track.performer, &sheet.performer), "Unknown Artist"), + "albumName":prefer(&sheet.title, "Unknown Album"), "scannedAt":scan_time, + "hasLyrics":false,"totalTracks":sheet.tracks.len(),"discNumber":1,"totalDiscs":1, + "format":format!("cue+{format}"), + }); + for (key, value) in [ + ("fileModTime", mod_time), + ("trackNumber", track.number), + ( + "duration", + end.map_or(0, |end| (end - track.start_time) as i64), + ), + ("bitDepth", depth), + ("sampleRate", rate), + ] { + if value != 0 { + result[key] = value.into(); + } + } + for (key, value) in [ + ("albumArtist", sheet.performer.as_str()), + ("coverPath", &cover), + ("isrc", &track.isrc), + ("releaseDate", &sheet.date), + ("genre", &sheet.genre), + ("composer", prefer(&track.composer, &sheet.composer)), + ] { + optional(&mut result, key, value); + } + results.push(result); + } + Ok(results.into()) + } +} + +fn scan_time() -> String { + Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true) +} + +fn modified(path: &crate::files::FilePath) -> i64 { + path.metadata() + .and_then(|metadata| metadata.modified()) + .map(|time| match time.into_std().duration_since(UNIX_EPOCH) { + Ok(duration) => duration.as_millis() as i64, + Err(error) => -((error.duration().as_nanos().div_ceil(1_000_000)) as i64), + }) + .unwrap_or_default() +} + +fn prefer<'a>(value: &'a str, fallback: &'a str) -> &'a str { + if value.is_empty() { fallback } else { value } +} + +fn optional(result: &mut Value, key: &str, value: &str) { + if !value.is_empty() { + result[key] = value.into(); + } +} + +fn next_start(sheet: &CueSheet, index: usize) -> Option<f64> { + sheet.tracks.get(index + 1).map(|track| { + if track.pre_gap >= 0.0 { + track.pre_gap + } else { + track.start_time + } + }) +} diff --git a/rust_backend/crates/extensions/src/backend/library/scan.rs b/rust_backend/crates/extensions/src/backend/library/scan.rs new file mode 100644 index 00000000..bb4d593d --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/library/scan.rs @@ -0,0 +1,592 @@ +//! One scan owner, bounded worker results and shared full/incremental traversal. + +use super::{Backend, library_extension, modified, scan_time}; +use cap_std::fs::OpenOptions; +use serde::Serialize; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{BufRead, BufReader, BufWriter, Read, Write}; +use std::path::Path; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, TryLockError, mpsc}; +use std::thread; +use std::time::Duration; + +type Check<'a> = &'a (dyn Fn() -> Result<(), String> + Sync); + +#[derive(Default)] +pub(in crate::backend) struct ScanState { + current: Mutex<Option<Arc<Run>>>, + owner: Mutex<()>, +} + +#[derive(Default)] +struct Run { + cancelled: AtomicBool, + failure: Mutex<Option<String>>, + progress: Mutex<Progress>, +} + +#[derive(Default, Serialize)] +struct Progress { + total_files: usize, + scanned_files: usize, + current_file: String, + error_count: usize, + progress_pct: f64, + is_complete: bool, +} + +struct AudioFile { + path: String, + modified: i64, + size: u64, +} + +struct ScanSummary { + deleted: Vec<String>, + skipped: usize, + total: usize, + run: Arc<Run>, +} + +impl Backend { + pub fn get_library_scan_progress(&self) -> Result<Value, String> { + let _operation = self.enter()?; + let current = self.library_scan.current.lock().expect("library scan lock"); + let value = match current.as_ref() { + Some(run) => serde_json::to_value(&*run.progress.lock().expect("scan progress lock")), + None => serde_json::to_value(Progress::default()), + }; + value.map_err(|error| error.to_string()) + } + + pub fn cancel_library_scan(&self) -> Result<(), String> { + let _operation = self.enter()?; + if let Some(run) = self + .library_scan + .current + .lock() + .expect("library scan lock") + .as_ref() + { + run.cancelled.store(true, Ordering::Release); + } + Ok(()) + } + + pub fn scan_library_folder(&self, folder: &str, check: Check<'_>) -> Result<Value, String> { + let mut tracks = Vec::new(); + self.scan_library( + folder, + None, + true, + &mut |value| { + tracks.push(value); + Ok(()) + }, + check, + )?; + Ok(tracks.into()) + } + + pub fn scan_library_folder_incremental( + &self, + folder: &str, + existing_json: &str, + check: Check<'_>, + ) -> Result<Value, String> { + // Go tolerates invalid snapshot JSON. A bad value does not discard other + // valid paths from an otherwise well-formed object. + let value: Value = serde_json::from_str(existing_json).unwrap_or(Value::Null); + let existing = value + .as_object() + .map(|object| { + object + .iter() + .filter_map(|(path, value)| { + value + .as_i64() + .or_else(|| value.is_null().then_some(0)) + .map(|value| (path.clone(), value)) + }) + .collect() + }) + .unwrap_or_default(); + self.incremental(folder, existing, check) + } + + pub fn scan_library_folder_incremental_from_snapshot( + &self, + folder: &str, + snapshot: &str, + check: Check<'_>, + ) -> Result<Value, String> { + let _operation = self.enter()?; + let mut existing = BTreeMap::new(); + if !snapshot.is_empty() { + self.check()?; + check()?; + let input = self + .environment() + .native_files()? + .resolve_legacy(snapshot)?; + input.native_display()?; + let file = input + .open(OpenOptions::new().read(true)) + .map_err(|error| format!("failed to load incremental snapshot: {error}"))?; + let mut reader = BufReader::new(file); + let mut line = Vec::new(); + loop { + self.check()?; + check()?; + line.clear(); + let read = (&mut reader) + .take(64 * 1024) + .read_until(b'\n', &mut line) + .map_err(|error| format!("failed to load incremental snapshot: {error}"))?; + if read == 0 { + break; + } + if read == 64 * 1024 && line.last() != Some(&b'\n') { + return Err( + "failed to load incremental snapshot: bufio.Scanner: token too long".into(), + ); + } + if line.last() == Some(&b'\n') { + line.pop(); + } + if line.last() == Some(&b'\r') { + line.pop(); + } + let line = String::from_utf8_lossy(&line); + if let Some((time, path)) = line.split_once('\t') + && let Ok(time) = time.parse::<i64>() + { + existing.insert(path.into(), time); + } + } + } + self.incremental(folder, existing, check) + } + + fn incremental( + &self, + folder: &str, + existing: BTreeMap<String, i64>, + check: Check<'_>, + ) -> Result<Value, String> { + let mut tracks = Vec::new(); + let summary = self.scan_library( + folder, + Some(&existing), + true, + &mut |value| { + tracks.push(value); + Ok(()) + }, + check, + )?; + Ok( + json!({"scanned":tracks,"deletedPaths":if summary.deleted.is_empty() { Value::Null } else { json!(summary.deleted) },"skippedCount":summary.skipped,"totalFiles":summary.total}), + ) + } + + pub fn scan_library_folder_to_ndjson_file( + &self, + folder: &str, + output: &str, + check: Check<'_>, + ) -> Result<u64, String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + check()?; + if output.is_empty() { + return Err("output path is empty".into()); + } + if supported(output) { + return Err("scan output must not overwrite an audio or CUE file".into()); + } + let files = self.environment().native_files()?; + let output = files.resolve_legacy(output)?; + output.native_display()?; + let _lock = files.lock(&output, &check)?; + let mut stage = output + .stage_existing_parent() + .map_err(|error| format!("create scan output: {error}"))?; + let mut count = 0; + let summary; + { + let mut writer = BufWriter::with_capacity(64 * 1024, &mut stage.file); + summary = self.scan_library( + folder, + None, + false, + &mut |value| { + serde_json::to_writer(&mut writer, &value) + .map_err(|error| error.to_string())?; + writer.write_all(b"\n").map_err(|error| error.to_string())?; + count += 1; + Ok(()) + }, + &check, + )?; + writer + .flush() + .map_err(|error| format!("flush scan output: {error}"))?; + } + stage.publish(&|| { + check()?; + if summary.run.cancelled.load(Ordering::Acquire) { + Err("scan cancelled".into()) + } else { + Ok(()) + } + })?; + Ok(count) + } + + fn collect_library_files( + &self, + folder: &str, + check: Check<'_>, + ) -> Result<Vec<AudioFile>, String> { + let files = self.environment().native_files()?; + let mut pending = vec![folder.to_owned()]; + let mut collected = Vec::new(); + while let Some(path) = pending.pop() { + check()?; + let input = files.resolve_legacy(&path)?; + input.native_display()?; + let metadata = input + .metadata() + .map_err(|error| format!("walk library path {path}: {error}"))?; + if metadata.is_dir() { + let entries = input + .entries() + .map_err(|error| format!("walk library path {path}: {error}"))?; + for (name, directory) in entries.into_iter().rev() { + if !directory && (!supported(&name) || staging(&name)) { + continue; + } + pending.push( + crate::files::clean(&Path::new(&path).join(name)) + .to_string_lossy() + .into_owned(), + ); + } + } else if supported(&path) && !staging(&path) { + collected.push(AudioFile { + path, + modified: modified(&input), + size: metadata.len(), + }); + } + } + Ok(collected) + } + + fn scan_library( + &self, + folder: &str, + existing: Option<&BTreeMap<String, i64>>, + ordered: bool, + sink: &mut dyn FnMut(Value) -> Result<(), String>, + check: Check<'_>, + ) -> Result<ScanSummary, String> { + let _operation = self.enter()?; + check()?; + if folder.is_empty() { + return Err("folder path is empty".into()); + } + // Retain the app data capability for this scan instead of reopening its + // directory for every track. Path resolution still checks live grants. + let files = self.environment().native_files()?; + let input = files.resolve_legacy(folder)?; + input.native_display()?; + let metadata = input + .metadata() + .map_err(|error| format!("folder not found: {error}"))?; + if !metadata.is_dir() { + return Err(format!("path is not a folder: {folder}")); + } + let run = Arc::new(Run::default()); + if let Some(previous) = self + .library_scan + .current + .lock() + .expect("library scan lock") + .replace(run.clone()) + { + previous.cancelled.store(true, Ordering::Release); + } + let check = || { + let mut failure = run.failure.lock().expect("scan failure lock"); + if let Some(error) = failure.as_ref() { + return Err(error.clone()); + } + let result = self.check().and_then(|()| { + if run.cancelled.load(Ordering::Acquire) { + Err("scan cancelled".into()) + } else { + check() + } + }); + if let Err(error) = &result { + *failure = Some(error.clone()); + } + result + }; + let _owner = loop { + check()?; + match self.library_scan.owner.try_lock() { + Ok(owner) => break owner, + Err(TryLockError::WouldBlock) => thread::sleep(Duration::from_millis(5)), + Err(TryLockError::Poisoned(_)) => return Err("library scan owner poisoned".into()), + } + }; + let current = self.collect_library_files(folder, &check)?; + let total = current.len(); + run.progress.lock().expect("scan progress lock").total_files = total; + let paths: BTreeSet<_> = current.iter().map(|file| file.path.as_str()).collect(); + let mut cue_times = BTreeMap::new(); + let mut deleted = Vec::new(); + if let Some(existing) = existing { + for (path, time) in existing { + let base = cue_base(path); + if let Some(base) = base { + cue_times.entry(base).or_insert(*time); + } + if !paths.contains(base.unwrap_or(path)) { + deleted.push(path.clone()); + } + } + } + let mut skipped = 0; + let tasks: Vec<_> = current + .iter() + .filter(|file| { + let old = existing + .and_then(|existing| existing.get(&file.path)) + .copied() + .or_else(|| { + (library_extension(&file.path, "") == "cue") + .then(|| cue_times.get(file.path.as_str()).copied()) + .flatten() + }); + if old == Some(file.modified) { + skipped += 1; + false + } else { + true + } + }) + .collect(); + let time = scan_time(); + let mut referenced = BTreeSet::new(); + let mut parsed = BTreeMap::new(); + // An unchanged CUE still owns its backing audio. Inspect all current + // sheets so incremental scans never re-add that audio as a full album. + for file in ¤t { + check()?; + if library_extension(&file.path, "") == "cue" + && let Ok(sheet) = self.parse_cue_file(&file.path, &check) + && !sheet.file_name.is_empty() + && let Ok(audio) = self.resolve_cue_audio(&file.path, &sheet.file_name, "", &check) + { + referenced.insert(audio.clone()); + parsed.insert(file.path.as_str(), (sheet, audio)); + } + } + check()?; + let mut results: BTreeMap<usize, Vec<Value>> = BTreeMap::new(); + let mut audio = Vec::new(); + let mut completed = skipped; + for (index, file) in tasks.iter().enumerate() { + check()?; + if library_extension(&file.path, "") == "cue" { + let result = match parsed.get(file.path.as_str()) { + Some((sheet, audio)) => self.scan_cue_sheet( + &file.path, + sheet, + audio, + "", + file.modified, + "", + &time, + &check, + ), + None => self.scan_cue_file_for_library( + &file.path, + "", + "", + file.modified, + "", + &time, + &check, + ), + } + .map(|value| value.as_array().expect("cue track array").clone()); + check()?; + completed += 1; + update(&run, completed, &file.path, result.is_err()); + if let Ok(tracks) = result { + if ordered { + results.insert(index, tracks); + } else { + for value in tracks { + sink(value)?; + } + } + } + } else if referenced.contains(&file.path) { + completed += 1; + update(&run, completed, &file.path, false); + } else { + audio.push((index, *file)); + } + } + // Allow short bursts without parking workers after each result. At most + // 64 completed tracks wait; NDJSON still uses bounded memory. + let workers = if audio.len() < 16 { + 1 + } else { + thread::available_parallelism() + .map_or(2, usize::from) + .clamp(2, 4) + }; + let next = AtomicUsize::new(0); + let stop = AtomicBool::new(false); + thread::scope(|scope| -> Result<(), String> { + let (sender, receiver) = mpsc::sync_channel(workers * 16); + for _ in 0..workers { + let sender = sender.clone(); + let (audio, next, stop, check, time, files) = + (&audio, &next, &stop, &check, &time, &files); + scope.spawn(move || { + while !stop.load(Ordering::Acquire) { + let Some((index, file)) = audio.get(next.fetch_add(1, Ordering::AcqRel)) + else { + break; + }; + let result = check().and_then(|()| { + self.scan_audio_file( + files, + &file.path, + "", + &format!("{}|{}|{}", file.path, file.size, file.modified), + time, + file.modified, + check, + ) + }); + if sender.send((*index, file.path.as_str(), result)).is_err() { + break; + } + } + }); + } + drop(sender); + let mut failure = None; + for (index, path, result) in receiver { + completed += 1; + update(&run, completed, path, result.is_err()); + if failure.is_none() { + let emit = check().and_then(|()| { + if let Ok(value) = result { + if ordered { + results.insert(index, vec![value]); + } else { + sink(value)?; + } + } + Ok(()) + }); + if let Err(error) = emit { + failure = Some(error); + stop.store(true, Ordering::Release); + } + } + } + failure.map_or(Ok(()), Err) + })?; + check()?; + for (_, values) in results { + for value in values { + check()?; + sink(value)?; + } + } + check()?; + // Share the existing duplicate index. A scan may observe edits/deletes + // after a previously cached lookup; never retain that stale snapshot. + if existing.is_none() || !tasks.is_empty() || !deleted.is_empty() { + self.environment() + .invalidate_isrc_cache(folder) + .map_err(|error| error.to_string())?; + } + let mut progress = run.progress.lock().expect("scan progress lock"); + progress.scanned_files = total; + progress.is_complete = true; + progress.progress_pct = 100.0; + drop(progress); + Ok(ScanSummary { + deleted, + skipped, + total, + run, + }) + } +} + +fn cue_base(path: &str) -> Option<&str> { + path.rfind("#track") + .filter(|index| *index > 0) + .map(|index| &path[..index]) +} + +fn supported(path: &str) -> bool { + matches!( + library_extension(path, "").as_str(), + "flac" + | "m4a" + | "mp4" + | "aac" + | "mp3" + | "opus" + | "ogg" + | "ape" + | "wv" + | "mpc" + | "wav" + | "aiff" + | "aif" + | "cue" + ) +} + +fn staging(path: &str) -> bool { + let name = Path::new(path) + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(); + name.ends_with(".partial") + || name + .rsplit_once('.') + .is_some_and(|(stem, _)| stem.ends_with(".partial")) +} + +fn update(run: &Run, completed: usize, path: &str, error: bool) { + let mut progress = run.progress.lock().expect("scan progress lock"); + progress.scanned_files = completed; + progress.current_file = Path::new(path) + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(); + progress.error_count += usize::from(error); + if progress.total_files > 0 { + progress.progress_pct = completed as f64 / progress.total_files as f64 * 100.0; + } +} diff --git a/rust_backend/crates/extensions/src/backend/lyrics.rs b/rust_backend/crates/extensions/src/backend/lyrics.rs new file mode 100644 index 00000000..1306214e --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/lyrics.rs @@ -0,0 +1,263 @@ +use super::Backend; +use crate::files::ExtensionFiles; +use cap_std::fs::OpenOptions; +use serde::{Deserialize, Serialize}; +use spotiflac_core::lyrics::{LyricsResponse, config, file, lrc}; +use spotiflac_providers::lyrics::{Check, SearchRequest}; +use std::collections::BTreeMap; +use std::io::Write; + +/// Native export arguments. Unlike the private JS SDK, file_path is not trimmed. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct LyricsRequest { + pub spotify_id: String, + pub track: String, + pub artist: String, + pub file_path: String, + pub duration_ms: i64, +} + +#[derive(Default, Serialize)] +struct LrcResult { + lyrics: String, + source: String, + sync_type: String, + instrumental: bool, +} + +impl Backend { + pub fn embed_lyrics_to_file( + &self, + path: &str, + lyrics: &str, + check: &Check<'_>, + ) -> Result<String, String> { + let fields = BTreeMap::from([ + ("LYRICS".into(), lyrics.into()), + ("UNSYNCEDLYRICS".into(), lyrics.into()), + ]); + self.embed_flac_fields_response( + path, + &fields, + "", + ("Failed to embed lyrics", "Lyrics embedded successfully"), + check, + ) + } + + pub fn get_lyrics_lrc( + &self, + request: &LyricsRequest, + check: &Check<'_>, + ) -> Result<String, String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + Ok(self.lrc_result(request, &check)?.lyrics) + } + + pub fn get_lyrics_lrc_with_source( + &self, + request: &LyricsRequest, + check: &Check<'_>, + ) -> Result<String, String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + serde_json::to_string(&self.lrc_result(request, &check)?).map_err(|error| error.to_string()) + } + + fn lrc_result(&self, request: &LyricsRequest, check: &Check<'_>) -> Result<LrcResult, String> { + check()?; + if !request.file_path.is_empty() { + let files = self.manager.environment().native_files()?; + return Ok(match existing(&files, &request.file_path, check)? { + Some(lyrics) => { + let source = lrc::extract_source(&lyrics); + LrcResult { + source: if source.is_empty() { + "Embedded".into() + } else { + source + }, + sync_type: "EMBEDDED".into(), + instrumental: lrc::is_instrumental_marker(&lyrics), + lyrics, + } + } + None => LrcResult::default(), + }); + } + let response = self.fetch_lyrics(request, check)?; + Ok(LrcResult { + lyrics: render(&response, request), + source: response.source, + sync_type: response.sync_type, + instrumental: response.instrumental, + }) + } + + pub(super) fn fetch_lyrics( + &self, + request: &LyricsRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, String> { + let response = self + .lyrics + .fetch( + SearchRequest { + spotify_id: request.spotify_id.clone(), + track: request.track.clone(), + artist: request.artist.clone(), + duration: request.duration_ms as f64 / 1000.0, + ..SearchRequest::default() + }, + check, + ) + .map_err(|error| error.to_string())?; + check()?; + Ok(response) + } + + pub fn get_lyrics_providers_json(&self) -> Result<String, String> { + let _operation = self.enter()?; + serde_json::to_string(&self.lyrics.providers()).map_err(|error| error.to_string()) + } + + pub fn set_lyrics_providers_json(&self, raw: &str) -> Result<(), String> { + let _operation = self.enter()?; + let providers = config::decode_providers(raw).map_err(|error| error.to_string())?; + self.lyrics + .set_providers(&providers) + .map_err(|error| error.to_string()) + } + + pub fn get_available_lyrics_providers_json(&self) -> Result<String, String> { + let _operation = self.enter()?; + serde_json::to_string(&config::available_providers()).map_err(|error| error.to_string()) + } + + pub fn get_lyrics_fetch_options_json(&self) -> Result<String, String> { + let _operation = self.enter()?; + serde_json::to_string(&self.lyrics.options()).map_err(|error| error.to_string()) + } + + pub fn set_lyrics_fetch_options_json(&self, raw: &str) -> Result<(), String> { + let _operation = self.enter()?; + let _settings = self.lyrics_settings.lock().expect("lyrics settings lock"); + let mut options = self.lyrics.options(); + options + .update_json(raw) + .map_err(|error| error.to_string())?; + self.lyrics + .set_options(options) + .map_err(|error| error.to_string()) + } + + pub fn fetch_and_save_lyrics( + &self, + request: &LyricsRequest, + output_path: &str, + check: &Check<'_>, + ) -> Result<(), String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + check()?; + let files = self.manager.environment().native_files()?; + let previous = if request.file_path.is_empty() { + None + } else { + existing(&files, &request.file_path, &check)? + }; + let (content, message) = if let Some(content) = previous { + ( + content, + format!("Saved LRC from embedded/sidecar to: {output_path}"), + ) + } else { + let response = self + .fetch_lyrics(request, &check) + .map_err(|error| format!("lyrics not found: {error}"))?; + if response.instrumental { + return Err("track is instrumental, no lyrics available".into()); + } + let content = render(&response, request); + if content.is_empty() { + return Err("failed to generate LRC content".into()); + } + let message = format!( + "Saved LRC to: {output_path} ({} lines)", + response.lines().len() + ); + (content, message) + }; + let write = || -> Result<(), String> { + let output = files.resolve_legacy(output_path)?; + let _output = files.lock(&output, &check)?; + check()?; + // Go WriteFile does not create missing parents. Reject symlinks and + // non-regular destinations before an atomic replacement as well. + output.native_display()?; + output.require_parent()?; + let permissions = match output.open(OpenOptions::new().write(true)) { + Ok(file) => Some( + file.metadata() + .map_err(|error| error.to_string())? + .permissions(), + ), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.to_string()), + }; + let mut stage = output + .stage_existing_parent() + .map_err(|error| error.to_string())?; + if let Some(permissions) = permissions { + stage + .file + .set_permissions(permissions) + .map_err(|error| error.to_string())?; + } + for chunk in content.as_bytes().chunks(16 * 1024) { + check()?; + stage + .file + .write_all(chunk) + .map_err(|error| error.to_string())?; + } + stage.publish(&check) + }; + write().map_err(|error| format!("failed to write LRC file: {error}"))?; + let _ = self + .manager + .environment() + .log_buffer() + .add("INFO", "Lyrics", &message); + Ok(()) + } +} + +fn render(response: &LyricsResponse, request: &LyricsRequest) -> String { + if response.instrumental { + "[instrumental:true]".into() + } else { + lrc::with_metadata(response, &request.track, &request.artist) + } +} + +fn existing( + files: &ExtensionFiles, + path: &str, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Option<String>, String> { + let result = file::extract( + path, + &|path| { + files + .resolve_legacy(path)? + .open(OpenOptions::new().read(true)) + .map_err(|error| error.to_string()) + }, + check, + ); + check()?; + Ok(result.ok().filter(|text| lrc::has_usable_content(text))) +} diff --git a/rust_backend/crates/extensions/src/backend/metadata.rs b/rust_backend/crates/extensions/src/backend/metadata.rs new file mode 100644 index 00000000..b150f536 --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/metadata.rs @@ -0,0 +1,591 @@ +use super::Backend; +use serde::Serialize; +use serde_json::{Value, json}; +use spotiflac_core::metadata::TrackMetadata; +use spotiflac_providers::deezer::{self, MetadataLookup}; +use spotiflac_providers::musicbrainz::MusicBrainzOptions; +use spotiflac_providers::resolver::{Check, ResolverError, ResolverOptions}; +use std::time::{Duration, Instant}; + +/// Trusted native configuration, never extension arguments or persisted settings. +pub struct MetadataOptions { + pub deezer_endpoint: String, + pub resolver: ResolverOptions, + pub musicbrainz: MusicBrainzOptions, +} + +impl Default for MetadataOptions { + fn default() -> Self { + Self { + deezer_endpoint: "https://api.deezer.com".into(), + resolver: ResolverOptions::default(), + musicbrainz: MusicBrainzOptions::default(), + } + } +} + +fn encode(value: &impl Serialize) -> Result<String, ResolverError> { + serde_json::to_string(value).map_err(|error| ResolverError::Failed(error.to_string())) +} + +fn context(error: ResolverError, prefix: &str) -> ResolverError { + match error { + ResolverError::Cancelled(_) | ResolverError::Closed | ResolverError::Busy => error, + _ => ResolverError::Failed(format!("{prefix}: {error}")), + } +} + +fn isrc_result(track: &TrackMetadata) -> Value { + // Go's public map includes fields omitted by the ordinary track serializer. + let mut result = json!({ + "spotify_id": track.spotify_id, "artists": track.artists, "name": track.name, + "album_name": track.album_name, "album_artist": track.album_artist, + "duration_ms": track.duration_ms, "images": track.images, + "release_date": track.release_date, "track_number": track.track_number, + "total_tracks": track.total_tracks, "disc_number": track.disc_number, + "total_discs": track.total_discs, "external_urls": track.external_urls, + "isrc": track.isrc, "album_id": track.album_id, "artist_id": track.artist_id, + "album_type": track.album_type, "composer": track.composer + }); + let id = track + .spotify_id + .strip_prefix("deezer:") + .unwrap_or(&track.spotify_id) + .trim(); + if !id.is_empty() { + result["id"] = json!(id); + result["track_id"] = json!(id); + result["success"] = json!(true); + } + result +} + +impl Backend { + pub(super) fn metadata_operation<T>( + &self, + seconds: u64, + check: &Check<'_>, + work: impl FnOnce(&Check<'_>) -> Result<T, ResolverError>, + ) -> Result<T, String> { + let _operation = self.enter()?; + let started = Instant::now(); + let check = || { + self.check()?; + check()?; + if started.elapsed() >= Duration::from_secs(seconds) { + Err("context deadline exceeded".into()) + } else { + Ok(()) + } + }; + check()?; + let result = work(&check); + check()?; + result.map_err(|error| error.to_string()) + } + + pub fn set_metadata_language(&self, tag: &str) -> Result<(), String> { + let _operation = self.enter()?; + self.deezer.set_language(tag); + Ok(()) + } + + pub fn set_song_link_region(&self, region: &str) -> Result<(), String> { + let _operation = self.enter()?; + self.availability + .set_region(region) + .map_err(|error| error.to_string()) + } + + pub fn get_song_link_region(&self) -> Result<String, String> { + let _operation = self.enter()?; + self.availability + .region() + .map_err(|error| error.to_string()) + } + + // The retired Settings track-ID cache is distinct from catalog caches. + pub fn get_track_cache_size(&self) -> Result<u64, String> { + let _operation = self.enter()?; + Ok(0) + } + + pub fn clear_track_id_cache(&self) -> Result<(), String> { + let _operation = self.enter()?; + Ok(()) + } + + pub fn parse_deezer_url(&self, url: &str) -> Result<(String, String), String> { + let _operation = self.enter()?; + deezer::parse_url(url).map_err(|error| error.to_string()) + } + + pub fn search_deezer( + &self, + query: &str, + track_limit: isize, + artist_limit: isize, + filter: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + encode( + self.deezer + .search_all(query, track_limit, artist_limit, filter, check)? + .as_ref(), + ) + }) + } + + pub fn get_deezer_metadata( + &self, + kind: &str, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| self.deezer_metadata(kind, id, check)) + } + + pub(super) fn deezer_metadata( + &self, + kind: &str, + id: &str, + check: &Check<'_>, + ) -> Result<String, ResolverError> { + match kind { + "track" => encode(&json!({"track": self.deezer.get_track(id, check)?})), + "album" => encode(self.deezer.get_album(id, check)?.as_ref()), + "artist" => encode(self.deezer.get_artist(id, check)?.as_ref()), + "playlist" => encode(&self.deezer.get_playlist(id, check)?), + _ => Err(ResolverError::Failed(format!( + "unsupported Deezer resource type: {kind}" + ))), + } + } + + pub fn get_deezer_extended_metadata( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(15, check, |check| { + if id.is_empty() { return Err(ResolverError::Failed("empty track ID".into())); } + let metadata = self.deezer.get_extended_metadata_by_track_id(id, check)?; + encode(&json!({"genre":metadata.genre,"label":metadata.label,"copyright":metadata.copyright})) + }) + } + + pub fn search_deezer_by_isrc(&self, isrc: &str, check: &Check<'_>) -> Result<String, String> { + self.search_deezer_by_isrc_for_item_id(isrc, "", check) + } + + pub fn search_deezer_by_isrc_for_item_id( + &self, + isrc: &str, + item_id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(10, check, |check| { + let state = self.manager.environment().download_state(); + let lease = if item_id.is_empty() { + None + } else { + Some( + state + .cancellation + .acquire(item_id) + .map_err(|error| ResolverError::Cancelled(error.to_string()))?, + ) + }; + let check = || { + check()?; + lease.as_ref().map_or(Ok(()), |lease| { + lease.check_active().map_err(|error| error.to_string()) + }) + }; + check().map_err(ResolverError::Cancelled)?; + let track = self.deezer.search_by_isrc(isrc, &check); + check().map_err(ResolverError::Cancelled)?; + encode(&isrc_result(&track?)) + }) + } + + pub fn convert_spotify_to_deezer( + &self, + kind: &str, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + let (deezer_id, fetch_error) = match kind { + "track" => (self.deezer_id_from_spotify(id, check).map_err(|error| context(error, "could not find Deezer equivalent"))?, "failed to fetch Deezer metadata"), + "album" => (self.availability.deezer_album_id(id, check).map_err(|error| context(error, "could not find Deezer album"))?, "failed to fetch Deezer album metadata"), + _ => return Err(ResolverError::Failed(format!("spotify to Deezer conversion only supported for tracks and albums: please search by name for {kind}"))), + }; + self.deezer_metadata(kind, &deezer_id, check).map_err(|error| context(error, fetch_error)) + }) + } + + fn deezer_id_from_spotify(&self, id: &str, check: &Check<'_>) -> Result<String, ResolverError> { + let track = self.availability.check_track(id, "", check)?; + if track.deezer && !track.deezer_id.is_empty() { + Ok(track.deezer_id) + } else { + Err(ResolverError::Failed("track not found on Deezer".into())) + } + } + + pub fn get_track_platform_links_json( + &self, + id: &str, + isrc: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + encode(&json!({"platforms":self.availability.track_platform_links(id, isrc, check)?})) + }) + } + + pub fn check_track_availability_json( + &self, + id: &str, + isrc: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + encode(&self.availability.check_track(id, isrc, check)?) + }) + } + + pub fn check_album_availability_json( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + encode(&self.availability.check_album(id, check)?) + }) + } + + pub fn check_availability_from_deezer_json( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + encode(&self.availability.check_from_deezer(id, check)?) + }) + } + + pub fn check_availability_by_platform_json( + &self, + platform: &str, + kind: &str, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + encode( + &self + .availability + .check_by_platform(platform, kind, id, check)?, + ) + }) + } + + pub fn check_availability_from_url_json( + &self, + url: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + encode(&self.availability.check_from_url(url, check)?) + }) + } + + pub fn get_streaming_urls_json(&self, id: &str, check: &Check<'_>) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + encode(&self.availability.streaming_urls(id, check)?) + }) + } + + pub fn get_deezer_id_from_spotify( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| self.deezer_id_from_spotify(id, check)) + } + + pub fn get_deezer_album_id_from_spotify( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + self.availability.deezer_album_id(id, check) + }) + } + + pub fn get_youtube_url_from_spotify( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + self.availability.youtube_url_from_spotify(id, check) + }) + } + + pub fn get_spotify_id_from_deezer_track( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + self.availability.platform_from_deezer(id, "spotify", check) + }) + } + + pub fn get_tidal_url_from_deezer_track( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + self.availability.platform_from_deezer(id, "tidal", check) + }) + } + + pub fn get_amazon_url_from_deezer_track( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + self.availability.platform_from_deezer(id, "amazon", check) + }) + } + + pub fn get_youtube_url_from_deezer_track( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + self.availability.platform_from_deezer(id, "youtube", check) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::RuntimeLimits; + use crate::environment::ExtensionEnvironment; + use spotiflac_network::{NetworkOptions, NetworkService}; + use std::io::{BufRead, BufReader, ErrorKind, Write}; + use std::net::TcpListener; + use std::sync::{Arc, mpsc}; + use std::thread; + + #[test] + fn isrc_http_cancellation_does_not_cancel_another_item() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let certificate = rcgen::generate_simple_self_signed(vec!["127.0.0.1".into()]).unwrap(); + let config = Arc::new( + rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert( + vec![certificate.cert.der().clone()], + rustls::pki_types::PrivatePkcs8KeyDer::from( + certificate.signing_key.serialize_der(), + ) + .into(), + ) + .unwrap(), + ); + let network = NetworkService::with_options(NetworkOptions { + extra_root_pem: certificate.cert.pem().into_bytes(), + doh_upstreams: vec![], + ..Default::default() + }) + .unwrap(); + network.set_allow_private_network(true); + let root = tempfile::tempdir().unwrap(); + let environment = ExtensionEnvironment::with_network( + &root.path().join("data"), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "1", + network, + ) + .unwrap(); + let backend = Arc::new( + Backend::with_environment( + &root.path().join("sources"), + environment, + RuntimeLimits::default(), + MetadataOptions { + deezer_endpoint: format!("https://{}", listener.local_addr().unwrap()), + ..Default::default() + }, + ) + .unwrap(), + ); + let start = |isrc: &'static str, item: &'static str| { + let owner = backend.clone(); + let (finished, completion) = mpsc::channel(); + let worker = thread::spawn(move || { + finished + .send(owner.search_deezer_by_isrc_for_item_id(isrc, item, &|| Ok(()))) + .unwrap(); + }); + (worker, completion) + }; + let accept = |isrc: &str| { + let deadline = Instant::now() + Duration::from_secs(3); + let stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) + if error.kind() == ErrorKind::WouldBlock && Instant::now() < deadline => + { + thread::sleep(Duration::from_millis(1)); + } + Err(error) => panic!("ISRC request did not arrive: {error}"), + } + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(3))) + .unwrap(); + let mut stream = rustls::StreamOwned::new( + rustls::ServerConnection::new(config.clone()).unwrap(), + stream, + ); + let mut reader = BufReader::new(&mut stream); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + assert!(line.starts_with(&format!("GET /2.0/track/isrc:{isrc} "))); + loop { + line.clear(); + assert!(reader.read_line(&mut line).unwrap() > 0); + if line == "\r\n" { + break; + } + } + stream + }; + let (cancelled_worker, cancelled) = start("USAAA2600001", "item-a"); + let cancelled_stream = accept("USAAA2600001"); + let (surviving_worker, surviving) = start("USAAA2600002", "item-b"); + let mut surviving_stream = accept("USAAA2600002"); + + backend.release_memory(true).unwrap(); + assert!(cancelled.try_recv().is_err()); + assert!(surviving.try_recv().is_err()); + backend + .environment() + .download_state() + .cancel("item-a") + .unwrap(); + assert_eq!( + cancelled.recv_timeout(Duration::from_secs(3)).unwrap(), + Err("download cancelled".into()) + ); + assert!(surviving.try_recv().is_err()); + // Neither response has been sent: cancellation must interrupt active IO. + drop(cancelled_stream); + let body = r#"{"id":7,"title":"Survivor","duration":3,"isrc":"USAAA2600002","artist":{"id":2,"name":"Artist"},"album":{"id":3,"title":"Album"}}"#; + write!( + surviving_stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .unwrap(); + surviving_stream.flush().unwrap(); + drop(surviving_stream); + let result = surviving + .recv_timeout(Duration::from_secs(3)) + .unwrap() + .unwrap(); + let result: Value = serde_json::from_str(&result).unwrap(); + assert_eq!(result["success"], true); + assert_eq!(result["track_id"], "7"); + assert_eq!(result["isrc"], "USAAA2600002"); + cancelled_worker.join().unwrap(); + surviving_worker.join().unwrap(); + backend.shutdown_checked().unwrap(); + } + + #[test] + fn shutdown_cancels_metadata_and_keeps_directory_ownership_until_work_releases() { + let root = tempfile::tempdir().unwrap(); + let create = || { + Backend::new( + &root.path().join("sources"), + &root.path().join("data"), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "1", + RuntimeLimits::default(), + ) + .unwrap() + }; + let backend = Arc::new(create()); + let availability = backend.availability(); + let (entered, started) = mpsc::channel(); + let (cancelled, cancellation) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let (finished, completion) = mpsc::channel(); + let owner = backend.clone(); + let worker = thread::spawn(move || { + owner.metadata_operation(30, &|| Ok(()), |check| -> Result<(), ResolverError> { + entered.send(()).unwrap(); + loop { + if let Err(error) = check() { + cancelled.send(()).unwrap(); + released.recv_timeout(Duration::from_secs(3)).unwrap(); + return Err(ResolverError::Cancelled(error)); + } + thread::sleep(Duration::from_millis(1)); + } + }) + }); + started.recv_timeout(Duration::from_secs(3)).unwrap(); + let owner = backend.clone(); + let shutdown = thread::spawn(move || { + owner.shutdown_checked().unwrap(); + finished.send(()).unwrap(); + }); + cancellation.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!(completion.try_recv().is_err()); + assert!( + Backend::new( + &root.path().join("sources"), + &root.path().join("data"), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "1", + RuntimeLimits::default() + ) + .is_err() + ); + release.send(()).unwrap(); + assert_eq!(worker.join().unwrap(), Err("backend is closed".into())); + shutdown.join().unwrap(); + completion.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!(availability.region().is_err()); + assert_eq!( + backend.get_track_cache_size(), + Err("backend is closed".into()) + ); + assert_eq!(create().get_track_cache_size().unwrap(), 0); + } +} diff --git a/rust_backend/crates/extensions/src/backend/musicbrainz.rs b/rust_backend/crates/extensions/src/backend/musicbrainz.rs new file mode 100644 index 00000000..2f90de2f --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/musicbrainz.rs @@ -0,0 +1,25 @@ +use super::Backend; +use spotiflac_providers::resolver::Check; + +impl Backend { + pub fn fetch_music_brainz_genre_by_isrc( + &self, + isrc: &str, + check: &Check<'_>, + ) -> Result<String, String> { + // Three 10-second HTTP attempts plus two 2-second retry waits fit + // within this root deadline. Cancellation still applies per caller. + self.metadata_operation(35, check, |check| self.musicbrainz.genre(isrc, check)) + } + + pub fn fetch_music_brainz_album_artist_by_isrc( + &self, + isrc: &str, + album_name: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(35, check, |check| { + self.musicbrainz.album_artist(isrc, album_name, check) + }) + } +} diff --git a/rust_backend/crates/extensions/src/backend/provider_metadata.rs b/rust_backend/crates/extensions/src/backend/provider_metadata.rs new file mode 100644 index 00000000..5098903f --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/provider_metadata.rs @@ -0,0 +1,536 @@ +use super::{Backend, ManagerError}; +use serde_json::{Map, Value, json}; +use spotiflac_core::cancellation::{CancellationDomain, CancellationRegistry, RequestLease}; +use spotiflac_core::matching::lowercase; +use spotiflac_providers::resolver::{Check, ResolverError}; +use std::sync::{Arc, mpsc}; +use std::time::Duration; + +impl Backend { + pub fn enrich_track_json(&self, id: &str, track_json: &str) -> Result<String, String> { + let _operation = self.enter()?; + self.manager + .enrich_track_export(id, track_json) + .map_err(|error| error.to_string()) + } + + pub fn handle_url_json(&self, url: &str) -> Result<String, String> { + let _operation = self.enter()?; + let id = self + .manager + .find_url_handler(url) + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("no extension found to handle URL: {url}"))?; + let raw = self + .manager + .provider_call(&id, "handleUrl", &json!([url]).to_string(), None, 30_000) + .map_err(|error| error.to_string())?; + let value: Value = serde_json::from_str(&raw).map_err(|error| error.to_string())?; + let mut result = strings( + &value, + &[ + "type", + "id", + "name", + "cover_url", + "header_image", + "header_video", + ], + ); + result.insert("extension_id".into(), id.into()); + if let Some(value) = value.get("track").filter(|value| !value.is_null()) { + result.insert("track".into(), track(value, "", 0)); + } + let values = array(&value, "tracks"); + if !values.is_empty() { + result.insert("tracks".into(), url_tracks(values)); + } + if let Some(value) = value.get("album").filter(|value| !value.is_null()) { + let mut info = strings( + value, + &[ + "id", + "name", + "artists", + "cover_url", + "header_image", + "header_video", + "release_date", + "album_type", + "provider_id", + ], + ); + info.insert("audio_traits".into(), value["audio_traits"].clone()); + info.insert( + "total_tracks".into(), + json!(value["total_tracks"].as_i64().unwrap_or_default()), + ); + result.insert("album".into(), info.into()); + } + if let Some(value) = value.get("artist").filter(|value| !value.is_null()) { + let mut info = strings( + value, + &[ + "id", + "name", + "image_url", + "header_image", + "header_video", + "provider_id", + ], + ); + info.insert( + "listeners".into(), + json!(value["listeners"].as_i64().unwrap_or_default()), + ); + for key in ["albums", "releases"] { + let values = array(value, key); + if !values.is_empty() { + info.insert( + key.into(), + values + .iter() + .map(|value| { + let mut value = album(value, false); + if value["album_type"] == "" { + value["album_type"] = "album".into(); + } + value + }) + .collect::<Vec<_>>() + .into(), + ); + } + } + let values = array(value, "top_tracks"); + if !values.is_empty() { + info.insert("top_tracks".into(), url_tracks(values)); + } + result.insert("artist".into(), info.into()); + } + self.check()?; + serde_json::to_string(&result).map_err(|error| error.to_string()) + } + + pub fn custom_search_json( + &self, + id: &str, + query: &str, + options_json: &str, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, String> { + let _operation = self.enter()?; + let options = serde_json::from_str::<Map<String, Value>>(options_json).unwrap_or_default(); + let raw = self + .manager + .provider_call( + id, + "customSearch", + &json!([query, options]).to_string(), + lease.clone(), + 30_000, + ) + .map_err(|error| error.to_string())?; + let values: Vec<Value> = serde_json::from_str(&raw).map_err(|error| error.to_string())?; + let normalized = values + .iter() + .map(|value| { + self.check()?; + if let Some(lease) = &lease { + lease.check_active().map_err(|error| error.to_string())?; + } + Ok(track(value, "", 0)) + }) + .collect::<Result<Vec<_>, String>>()?; + serde_json::to_string(&normalized).map_err(|error| error.to_string()) + } + + pub fn get_extension_home_feed_json( + &self, + id: &str, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, String> { + let _operation = self.enter()?; + self.manager + .provider_call(id, "getHomeFeed", "[]", lease, 60_000) + .map_err(|error| error.to_string()) + } + + pub fn get_provider_metadata_json( + &self, + provider_id: &str, + kind: &str, + resource_id: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(30, check, |check| { + let id = provider_id.trim(); + if id.is_empty() { + return Err(ResolverError::Failed("empty provider ID".into())); + } + let available = self.manager.require_metadata_provider(id); + // Only built-in dispatch folds case; installed IDs retain their case. + if lowercase(id) == "deezer" && available.is_err() { + return self.deezer_metadata(kind, resource_id, check); + } + available.map_err(|error| ResolverError::Failed(error.to_string()))?; + let method = match kind { + "track" => "getTrack", + "album" => "getAlbum", + "playlist" => "getPlaylist", + "artist" => "getArtist", + _ => { + return Err(ResolverError::Failed(format!( + "unsupported provider resource type: {kind}" + ))); + } + }; + let arguments = json!([resource_id]).to_string(); + let result = self.provider_metadata_call(id, method, &arguments, check)?; + let result = response(kind, &result, check)?; + serde_json::to_string(&result).map_err(|error| ResolverError::Failed(error.to_string())) + }) + } + + pub(super) fn provider_metadata_call( + &self, + id: &str, + method: &str, + arguments: &str, + check: &Check<'_>, + ) -> Result<Value, ResolverError> { + self.metadata_provider_work(check, |lease| { + self.manager + .provider_call(id, method, arguments, Some(lease), 30_000) + }) + } + + pub(super) fn metadata_provider_work( + &self, + check: &Check<'_>, + work: impl FnOnce(Arc<RequestLease>) -> Result<String, ManagerError> + Send, + ) -> Result<Value, ResolverError> { + let cancellation = CancellationRegistry::new(CancellationDomain::ExtensionRequest); + let lease = Arc::new( + cancellation + .acquire("") + .expect("new metadata cancellation lease"), + ); + std::thread::scope(|scope| { + let (done, completion) = mpsc::channel::<()>(); + let worker_lease = &lease; + let worker = std::thread::Builder::new() + .name("provider-metadata".into()) + .spawn_scoped(scope, move || { + let _done = done; + work(worker_lease.clone()) + }) + .map_err(|error| ResolverError::Failed(error.to_string()))?; + let mut cancelled = None; + loop { + if let Err(message) = check() { + cancelled = Some(message); + lease.release(); + break; + } + if !matches!( + completion.recv_timeout(Duration::from_millis(5)), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + break; + } + } + let result = worker + .join() + .map_err(|_| ResolverError::Failed("metadata provider panicked".into()))?; + lease.release(); + if let Some(message) = cancelled { + return Err(ResolverError::Cancelled(message)); + } + check().map_err(ResolverError::Cancelled)?; + let result = result.map_err(|error| ResolverError::Failed(error.to_string()))?; + serde_json::from_str(&result).map_err(|error| ResolverError::Failed(error.to_string())) + }) + } +} + +// Inputs have already passed the installed provider's typed decoder. Fill the +// fields that Go's internal omitempty serializer drops before the public map. +fn text<'a>(value: &'a Value, key: &str) -> &'a str { + value[key].as_str().unwrap_or_default() +} + +fn strings(value: &Value, fields: &[&str]) -> Map<String, Value> { + fields + .iter() + .map(|key| ((*key).into(), json!(text(value, key)))) + .collect() +} + +fn array<'a>(value: &'a Value, key: &str) -> &'a [Value] { + value[key].as_array().map(Vec::as_slice).unwrap_or_default() +} + +fn url_tracks(values: &[Value]) -> Value { + values.iter().map(|value| track(value, "", 0)).collect() +} + +fn track(value: &Value, cover: &str, index: usize) -> Value { + let mut result = strings( + value, + &[ + "id", + "name", + "artists", + "album_name", + "album_artist", + "album_id", + "album_url", + "artist_id", + "artist_url", + "external_urls", + "preview_url", + "release_date", + "isrc", + "provider_id", + "item_type", + "album_type", + "spotify_id", + "genre", + "label", + "copyright", + "composer", + "comment", + "audio_quality", + "audio_modes", + "upc", + ], + ); + let cover = [text(value, "cover_url"), text(value, "images"), cover] + .into_iter() + .find(|value| !value.is_empty()) + .unwrap_or_default(); + result.insert("images".into(), json!(cover)); + result.insert("cover_url".into(), json!(cover)); + for key in [ + "duration_ms", + "track_number", + "total_tracks", + "disc_number", + "total_discs", + ] { + result.insert(key.into(), json!(value[key].as_i64().unwrap_or_default())); + } + if result["track_number"] == 0 && index > 0 { + result.insert("track_number".into(), json!(index)); + } + result.insert( + "explicit".into(), + json!(value["explicit"].as_bool().unwrap_or(false)), + ); + result.insert("external_links".into(), value["external_links"].clone()); + result.into() +} + +fn tracks(values: &[Value], cover: &str, check: &Check<'_>) -> Result<Value, ResolverError> { + values + .iter() + .enumerate() + .map(|(index, value)| { + check().map_err(ResolverError::Cancelled)?; + Ok(track(value, cover, index + 1)) + }) + .collect::<Result<Vec<_>, _>>() + .map(Value::Array) +} + +fn album(value: &Value, full: bool) -> Value { + let mut result = strings( + value, + &[ + "id", + "name", + "artists", + "cover_url", + "release_date", + "album_type", + "provider_id", + ], + ); + result.insert("images".into(), json!(text(value, "cover_url"))); + result.insert( + "total_tracks".into(), + json!(value["total_tracks"].as_i64().unwrap_or_default()), + ); + if full { + result.extend(strings( + value, + &["artist_id", "header_image", "header_video"], + )); + result.insert("audio_traits".into(), json!(array(value, "audio_traits"))); + } + result.into() +} + +fn response(kind: &str, value: &Value, check: &Check<'_>) -> Result<Value, ResolverError> { + check().map_err(ResolverError::Cancelled)?; + let cover = text(value, "cover_url"); + Ok(match kind { + "track" => json!({"track":track(value, "", 0)}), + "album" => { + json!({"album_info":album(value, true),"track_list":tracks(array(value,"tracks"),cover,check)?}) + } + "playlist" => { + let mut info = strings( + value, + &[ + "id", + "name", + "cover_url", + "header_image", + "header_video", + "provider_id", + ], + ); + info.insert("images".into(), json!(cover)); + info.insert( + "owner".into(), + json!({"name":text(value,"artists"),"images":cover}), + ); + json!({"playlist_info":info,"track_list":tracks(array(value,"tracks"),cover,check)?}) + } + "artist" => { + let cover = text(value, "image_url"); + let image = [text(value, "header_image"), cover] + .into_iter() + .map(str::trim) + .find(|value| !value.is_empty()) + .unwrap_or_default(); + let mut info = strings( + value, + &["id", "name", "header_image", "header_video", "provider_id"], + ); + info.insert("images".into(), json!(image)); + info.insert("cover_url".into(), json!(cover)); + if value["listeners"].as_i64().unwrap_or_default() > 0 { + info.insert("listeners".into(), value["listeners"].clone()); + } + let mut result = json!({"artist_info":info}); + for key in ["albums", "releases"] { + let values = array(value, key); + if key == "albums" || !values.is_empty() { + result[key] = values + .iter() + .map(|value| { + check().map_err(ResolverError::Cancelled)?; + Ok(album(value, false)) + }) + .collect::<Result<Vec<_>, ResolverError>>()? + .into(); + } + } + let top = array(value, "top_tracks"); + if !top.is_empty() { + result["top_tracks"] = tracks(top, cover, check)?; + } + result + } + _ => unreachable!("validated provider resource type"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::RuntimeLimits; + use std::path::Path; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Barrier}; + use std::thread; + + fn backend(root: &Path) -> Backend { + Backend::new( + &root.join("sources"), + &root.join("data"), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "1", + RuntimeLimits::default(), + ) + .unwrap() + } + + #[test] + fn metadata_provider_work_returns_immediate_json() { + let root = tempfile::tempdir().unwrap(); + let backend = backend(root.path()); + let result = backend + .metadata_provider_work(&|| Ok(()), |_| Ok(r#"{"immediate":true}"#.into())) + .unwrap(); + assert_eq!(result["immediate"], true); + } + + #[test] + fn metadata_provider_work_waits_for_delayed_worker_completion() { + let root = tempfile::tempdir().unwrap(); + let backend = Arc::new(backend(root.path())); + let started = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + let worker_backend = Arc::clone(&backend); + let worker_started = Arc::clone(&started); + let worker_release = Arc::clone(&release); + let worker = thread::spawn(move || { + worker_backend.metadata_provider_work(&|| Ok(()), move |_| { + worker_started.wait(); + worker_release.wait(); + Ok(r#"{"delayed":true}"#.into()) + }) + }); + started.wait(); + release.wait(); + assert_eq!(worker.join().unwrap().unwrap()["delayed"], true); + } + + #[test] + fn metadata_provider_work_reports_worker_panic() { + let root = tempfile::tempdir().unwrap(); + let backend = backend(root.path()); + let result = backend + .metadata_provider_work(&|| Ok(()), |_| -> Result<String, ManagerError> { + panic!("metadata worker failed") + }); + assert_eq!( + result, + Err(ResolverError::Failed("metadata provider panicked".into())) + ); + } + + #[test] + fn metadata_provider_work_releases_lease_and_joins_cancelled_worker() { + let root = tempfile::tempdir().unwrap(); + let backend = backend(root.path()); + let started = Arc::new(AtomicBool::new(false)); + let joined = Arc::new(AtomicBool::new(false)); + let worker_started = Arc::clone(&started); + let worker_joined = Arc::clone(&joined); + let result = backend.metadata_provider_work( + &|| { + if started.load(Ordering::Acquire) { + Err("cancelled by test".into()) + } else { + Ok(()) + } + }, + move |lease| { + worker_started.store(true, Ordering::Release); + assert!(lease.wait_cancelled(60_000).is_err()); + worker_joined.store(true, Ordering::Release); + Ok("{}".into()) + }, + ); + assert_eq!( + result, + Err(ResolverError::Cancelled("cancelled by test".into())) + ); + assert!(joined.load(Ordering::Acquire)); + } +} diff --git a/rust_backend/crates/extensions/src/backend/reenrich.rs b/rust_backend/crates/extensions/src/backend/reenrich.rs new file mode 100644 index 00000000..fb458da5 --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/reenrich.rs @@ -0,0 +1,359 @@ +use super::{Backend, LyricsRequest}; +use cap_std::fs::OpenOptions; +use serde_json::json; +use spotiflac_core::lyrics::{file, lrc}; +use spotiflac_core::metadata::TrackMetadata; +use spotiflac_core::metadata::reenrich::{self, Request}; +use spotiflac_core::tags::{embed_flac_metadata, extract_cover}; +use spotiflac_providers::deezer::MetadataLookup; +use spotiflac_providers::resolver::{Check, availability}; +use std::io::Write; +use std::sync::Mutex; + +impl Backend { + /// Resolve proposed tags without reading audio, downloading covers or writing files. + pub fn preview_reenrich_file( + &self, + request_json: &str, + check: &Check<'_>, + ) -> Result<String, String> { + let _operation = self.enter()?; + let check = || { + self.check()?; + check() + }; + check()?; + let request = self.resolve_reenrich_request(request_json, &check)?; + serde_json::to_string(&json!({"method":"preview","success":true, + "enriched_metadata":request.result_metadata()})) + .map_err(|error| error.to_string()) + } + + /// Execute FLAC enrichment or return the existing native FFmpeg plan. + /// Returned cover files belong to the caller, which removes them after use. + pub fn reenrich_file(&self, request_json: &str, check: &Check<'_>) -> Result<String, String> { + let _operation = self.enter()?; + let failure = Mutex::new(None::<String>); + let check = || { + if let Some(error) = failure.lock().expect("reenrich cancellation lock").as_ref() { + return Err(error.clone()); + } + self.check().and_then(|()| check()).inspect_err(|error| { + *failure.lock().expect("reenrich cancellation lock") = Some(error.clone()); + }) + }; + check()?; + let request = self.resolve_reenrich_request(request_json, &check)?; + let enriched = request.result_metadata(); + if request.preview_only { + return Ok(json!({"method":"preview","success":true, + "enriched_metadata":enriched}) + .to_string()); + } + let files = self.environment().native_files()?; + files.resolve_legacy(&request.file_path)?.native_display()?; + let is_flac = request.file_path.to_ascii_lowercase().ends_with(".flac"); + let wants_cover = !request.cover_url.is_empty() && request.selected("cover", "cover"); + let fetch_cover = || { + if wants_cover { + self.cover + .download( + &request.cover_url, + request.cover_max_dimension as i64, + &check, + ) + .ok() + } else { + None + } + }; + let fetch_lyrics = || -> Result<String, String> { + // Fetches and existing lyrics are best-effort; cancellation is not. + check()?; + let mut lyrics = String::new(); + if request.selected("lyrics", "lyrics") { + if let Ok(existing) = file::extract( + &request.file_path, + &|path| { + files + .resolve_legacy(path)? + .open(OpenOptions::new().read(true)) + .map_err(|error| error.to_string()) + }, + &check, + ) { + lyrics = existing; + } + check()?; + if request.embed_lyrics { + let query = LyricsRequest { + spotify_id: request.spotify_id.clone(), + track: request.track_name.clone(), + artist: request.artist_name.clone(), + duration_ms: request.duration_ms, + ..LyricsRequest::default() + }; + if let Ok(response) = self.fetch_lyrics(&query, &check) + && !response.instrumental + { + lyrics = lrc::with_metadata(&response, &query.track, &query.artist); + } + check()?; + } + } + Ok(lyrics) + }; + let (cover, lyrics) = std::thread::scope(|scope| { + let worker = (wants_cover && request.selected("lyrics", "lyrics")).then(|| { + std::thread::Builder::new() + .name("reenrich-cover".into()) + .spawn_scoped(scope, fetch_cover) + }); + let lyrics = fetch_lyrics(); + let cover = match worker { + Some(Ok(worker)) => worker + .join() + .map_err(|_| "cover worker panicked".to_owned())?, + _ => fetch_cover(), + }; + check()?; + Ok::<_, String>((cover, lyrics?)) + })?; + let metadata = request.write_metadata(&lyrics); + let external = request.write_external_lrc(&lyrics); + if is_flac { + let cover = cover.as_deref().filter(|data| !data.is_empty()); + let prefix = if cover.is_some() { + "failed to embed metadata with cover" + } else { + "failed to embed metadata" + }; + self.rewrite_tag_file(&request.file_path, &check, |source, output, _| { + embed_flac_metadata( + source, + output, + &metadata, + &request.artist_tag_mode, + cover, + &check, + ) + .map_err(|error| format!("{prefix}: {error}"))?; + // Verify the staged result before replacing the user's audio. + if cover.is_some() { + let picture = extract_cover(output, "flac", &check).map_err(|error| { + format!("metadata embedded but cover verification failed: {error}") + })?; + if picture.data.is_empty() { + return Err( + "metadata embedded but cover verification failed: empty embedded cover" + .into(), + ); + } + } + Ok(true) + }) + .map_err(|error| { + if error.starts_with("failed to embed metadata") + || error.starts_with("metadata embedded") + { + error + } else { + format!("{prefix}: {error}") + } + })?; + return Ok( + json!({"method":"native","success":true,"enriched_metadata":enriched, + "lyrics":lyrics,"write_external_lrc":external}) + .to_string(), + ); + } + + let temporary = cover.as_deref().and_then(|data| { + let write = || -> Result<tempfile::NamedTempFile, String> { + let mut file = tempfile::Builder::new() + .prefix("reenrich_cover_") + .suffix(".jpg") + .tempfile_in(self.environment().data_directory()) + .map_err(|error| error.to_string())?; + for chunk in data.chunks(65536) { + check()?; + file.write_all(chunk).map_err(|error| error.to_string())?; + } + file.as_file() + .sync_all() + .map_err(|error| error.to_string())?; + Ok(file) + }; + write().ok() + }); + check()?; + let cover_path = if let Some(file) = &temporary { + files + .resolve_legacy(&file.path().to_string_lossy())? + .native_display()? + } else { + String::new() + }; + let result = json!({"method":"ffmpeg","cover_path":cover_path,"lyrics":lyrics, + "enriched_metadata":enriched,"metadata":metadata,"write_external_lrc":external}) + .to_string(); + check()?; + if let Some(file) = temporary { + file.keep().map_err(|error| error.to_string())?; + } + Ok(result) + } + + fn resolve_reenrich_request( + &self, + request_json: &str, + check: &Check<'_>, + ) -> Result<Request, String> { + let mut request = Request::parse(request_json) + .map_err(|error| format!("failed to parse request: {error}"))?; + if request.file_path.is_empty() { + return Err("file_path is required".into()); + } + if request.search_online { + let mut found = false; + let catalog_track = self.reenrich_identifiers(&request, &check); + if let Some(track) = &catalog_track { + request.apply(&reenrich::from_catalog(track)); + found = true; + } + check()?; + let query = request.query(); + if !query.is_empty() { + let tracks = self.metadata_provider_work(&check, |lease| { + self.manager.search_metadata_providers_with_lease( + &query, + 5, + true, + "", + 30_000, + Some(lease), + ) + }); + check()?; + if let Ok(tracks) = tracks + && let Some(tracks) = tracks.as_array() + && let Some(track) = request.select(tracks) + { + request.apply(track); + found = true; + } + } + if request.selected("basic_tags", "album_artist") + && request.album_artist.is_empty() + && !request.isrc.is_empty() + { + let artist = self.fetch_music_brainz_album_artist_by_isrc( + &request.isrc, + &request.album_name, + &check, + ); + check()?; + if let Ok(artist) = artist + && !artist.trim().is_empty() + { + request.album_artist = artist.trim().into(); + found = true; + } + } + if found + && !request.isrc.is_empty() + && request.any_selected("extra", &["genre", "label", "copyright"]) + && (request.genre.is_empty() + || request.label.is_empty() + || request.copyright.is_empty()) + { + let isrc = request.isrc.trim(); + if !isrc.is_empty() { + let extended = self.metadata_operation(10, &check, |check| { + if let Some(track) = catalog_track + .as_ref() + .filter(|track| track.isrc.trim().eq_ignore_ascii_case(isrc)) + { + return self.deezer.get_extended_metadata_for_track(track, check); + } + self.deezer.get_extended_metadata_by_isrc(isrc, check) + }); + check()?; + if let Ok(extended) = extended { + for (target, value) in [ + (&mut request.genre, &extended.genre), + (&mut request.label, &extended.label), + (&mut request.copyright, &extended.copyright), + ] { + if target.is_empty() && !value.is_empty() { + *target = value.clone(); + } + } + } + if request.genre.is_empty() { + let genre = self.fetch_music_brainz_genre_by_isrc(isrc, &check); + check()?; + if let Ok(genre) = genre + && !genre.is_empty() + { + request.genre = genre; + } + } + } + } + } + check()?; + Ok(request) + } + + fn reenrich_identifiers(&self, request: &Request, check: &Check<'_>) -> Option<TrackMetadata> { + let isrc = request.isrc.trim(); + if !isrc.is_empty() { + let track = + self.metadata_operation(10, check, |check| self.deezer.search_by_isrc(isrc, check)); + if check().is_err() { + return None; + } + if let Ok(track) = track + && request.verified(&reenrich::from_catalog(&track)) + { + return Some(track); + } + } + let source = request.spotify_id.trim(); + if source.is_empty() { + return None; + } + let mut deezer_id = source + .strip_prefix("deezer:") + .unwrap_or(source) + .trim() + .to_owned(); + if deezer_id == source { + deezer_id = availability::deezer_id_from_url(source); + } + if deezer_id.is_empty() { + let mut spotify_id = availability::spotify_id_from_url(source); + if spotify_id.is_empty() && source.len() == 22 && !source.contains([':', '/']) { + spotify_id = source.into(); + } + if !spotify_id.is_empty() { + if let Ok(id) = self.get_deezer_id_from_spotify(&spotify_id, check) { + deezer_id = id.trim().into(); + } + if check().is_err() { + return None; + } + } + } + if deezer_id.is_empty() { + return None; + } + let track = self + .metadata_operation(15, check, |check| self.deezer.get_track(&deezer_id, check)) + .ok()?; + request + .verified(&reenrich::from_catalog(&track)) + .then_some(track) + } +} diff --git a/rust_backend/crates/extensions/src/backend/share.rs b/rust_backend/crates/extensions/src/backend/share.rs new file mode 100644 index 00000000..7f3e75fa --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/share.rs @@ -0,0 +1,156 @@ +use super::Backend; +use serde_json::Value; +use spotiflac_core::metadata::share::{self, Provider, Request, ShareResult}; +use spotiflac_providers::resolver::{Check, ResolverError}; +use std::collections::{BTreeMap, VecDeque}; + +#[derive(Default)] +pub(super) struct Cache { + values: BTreeMap<(u64, String), String>, + order: VecDeque<(u64, String)>, +} + +impl Cache { + pub fn clear(&mut self) { + self.values.clear(); + self.order.clear(); + } + + fn put(&mut self, key: (u64, String), value: String) { + if !self.values.contains_key(&key) { + self.order.push_back(key.clone()); + } + self.values.insert(key, value); + while self.order.len() > 128 { + if let Some(oldest) = self.order.pop_front() { + self.values.remove(&oldest); + } + } + } +} + +impl Backend { + pub fn find_collection_across_extensions_json( + &self, + request_json: &str, + check: &Check<'_>, + ) -> Result<String, String> { + self.metadata_operation(60, check, |check| { + let request = Request::parse(request_json) + .map_err(|error| ResolverError::Failed(error.to_string()))?; + if request.name.is_empty() { + return Ok("[]".into()); + } + let revision = self.manager.metadata_revision(); + let providers = self + .manager + .share_providers(&request.source_extension_id) + .map_err(|error| ResolverError::Failed(error.to_string()))?; + let key = (revision, request.cache_key(&providers)); + if let Some(cached) = self + .share_cache + .lock() + .expect("share cache lock") + .values + .get(&key) + { + return Ok(cached.clone()); + } + let query = request.query(); + // Go searches all providers concurrently; retain snapshot order on join. + let results = std::thread::scope(|scope| { + let workers: Vec<_> = providers + .iter() + .map(|provider| { + let request = &request; + let query = &query; + scope + .spawn(move || self.share_for_provider(provider, request, query, check)) + }) + .collect(); + workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| ResolverError::Failed("collection search panicked".into())) + }) + .collect::<Result<Vec<_>, _>>() + })?; + check().map_err(ResolverError::Cancelled)?; + let response = serde_json::to_string(&results) + .map_err(|error| ResolverError::Failed(error.to_string()))?; + if revision == self.manager.metadata_revision() + && results.iter().all(ShareResult::cacheable) + { + self.share_cache + .lock() + .expect("share cache lock") + .put(key, response.clone()); + } + Ok(response) + }) + } + + fn share_for_provider( + &self, + provider: &Provider, + request: &Request, + query: &str, + check: &Check<'_>, + ) -> ShareResult { + let fetch = || -> Result<Vec<Value>, ResolverError> { + let filter = match request.kind.as_str() { + "album" => "albums", + "artist" => "artists", + _ => "", + }; + if !filter.is_empty() { + let arguments = + serde_json::json!([query, {"filter":filter,"limit":10}]).to_string(); + let custom = + self.provider_metadata_call(&provider.id, "customSearch", &arguments, check); + check().map_err(ResolverError::Cancelled)?; + if let Ok(Value::Array(tracks)) = custom + && !tracks.is_empty() + { + return Ok(tracks); + } + } + let arguments = serde_json::json!([query, 10]).to_string(); + let result = + self.provider_metadata_call(&provider.id, "searchTracks", &arguments, check)?; + Ok(result["tracks"].as_array().cloned().unwrap_or_default()) + }; + match fetch() { + Ok(tracks) => share::select(provider, request, &tracks), + Err(error) => { + let mut result = ShareResult::new(provider); + result.error = error.to_string(); + result + } + } + } +} + +#[cfg(test)] +mod tests { + use super::Cache; + + #[test] + fn collection_share_cache_is_bounded_fifo_and_revisions_are_distinct() { + let mut cache = Cache::default(); + for index in 0..128 { + cache.put((0, index.to_string()), "original".into()); + } + assert!(cache.values.contains_key(&(0, "0".into()))); + cache.put((0, "0".into()), "updated".into()); + cache.put((1, "0".into()), "new revision".into()); + assert_eq!(cache.values.len(), 128); + assert!(!cache.values.contains_key(&(0, "0".into()))); + assert!(cache.values.contains_key(&(0, "1".into()))); + assert_eq!(cache.values[&(1, "0".into())], "new revision"); + cache.clear(); + assert!(cache.values.is_empty() && cache.order.is_empty()); + } +} diff --git a/rust_backend/crates/extensions/src/backend/tags.rs b/rust_backend/crates/extensions/src/backend/tags.rs new file mode 100644 index 00000000..e6de33b9 --- /dev/null +++ b/rust_backend/crates/extensions/src/backend/tags.rs @@ -0,0 +1,370 @@ +use super::Backend; +use crate::files::ExtensionFiles; +use cap_std::fs::{File, OpenOptions}; +use serde_json::{Value, json}; +use spotiflac_core::tags::{ + embed_flac_metadata, rewrite_ac4_config, rewrite_ac4_metadata, rewrite_audio_tags, + rewrite_m4a_freeform, +}; +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::io::Read; + +impl Backend { + pub fn rewrite_split_artist_tags( + &self, + path: &str, + artist: &str, + album_artist: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<String, String> { + let fields = BTreeMap::from([ + ("ARTIST".into(), artist.into()), + ("ALBUMARTIST".into(), album_artist.into()), + ]); + self.embed_flac_fields_response( + path, + &fields, + "split_vorbis", + ( + "Failed to rewrite artist tags", + "Split artist tags written successfully", + ), + check, + ) + } + + pub(super) fn embed_flac_fields_response( + &self, + path: &str, + fields: &BTreeMap<String, String>, + artist_mode: &str, + messages: (&str, &str), + check: &dyn Fn() -> Result<(), String>, + ) -> Result<String, String> { + let _operation = self.enter()?; + let failure = RefCell::new(None::<String>); + let check = || { + if let Some(error) = failure.borrow().as_ref() { + return Err(error.clone()); + } + self.check().and_then(|()| check()).inspect_err(|error| { + *failure.borrow_mut() = Some(error.clone()); + }) + }; + check()?; + self.environment() + .native_files()? + .resolve_legacy(path)? + .native_display()?; + if let Err(error) = self.rewrite_tag_file(path, &check, |source, output, _| { + embed_flac_metadata(source, output, fields, artist_mode, None, &check)?; + Ok(true) + }) { + check()?; + return Ok( + crate::download::native_error_response(&format!("{}: {error}", messages.0)) + .to_string(), + ); + } + Ok(json!({"success":true,"message":messages.1}).to_string()) + } + + /// Preserve the application facade, including native-editor fallback to + /// FFmpeg. Permission and lifecycle failures must never become success. + pub fn edit_file_metadata( + &self, + path: &str, + metadata_json: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<Value, String> { + let _operation = self.enter()?; + let failure = RefCell::new(None::<String>); + let check = || { + if let Some(error) = failure.borrow().as_ref() { + return Err(error.clone()); + } + self.check().and_then(|()| check()).inspect_err(|error| { + *failure.borrow_mut() = Some(error.clone()); + }) + }; + check()?; + let decoded: Option<BTreeMap<String, Option<String>>> = serde_json::from_str(metadata_json) + .map_err(|error| format!("invalid metadata JSON: {error}"))?; + let null_fields = decoded.is_none(); + let fields: BTreeMap<String, String> = decoded + .unwrap_or_default() + .into_iter() + .map(|(key, value)| (key, value.unwrap_or_default())) + .collect(); + let files = self.manager.environment().native_files()?; + let target = files.resolve_legacy(path)?; + target.native_display()?; + if let Some(cover) = fields.get("cover_path").filter(|p| !p.trim().is_empty()) { + files.resolve_legacy(cover.trim())?.native_display()?; + } + let lower = path.to_lowercase(); + let suffix = |extensions: &[&str]| extensions.iter().any(|ext| lower.ends_with(ext)); + let m4a = suffix(&[".m4a", ".mp4", ".m4b"]); + let mp4 = target + .open(OpenOptions::new().read(true)) + .ok() + .is_some_and(|mut file| { + let mut header = [0; 12]; + file.read(&mut header) + .is_ok_and(|count| count >= 8 && &header[4..8] == b"ftyp") + }); + check()?; + let mut replay_gain = false; + let only_replay_gain = fields + .iter() + .filter(|(_, value)| !value.trim().is_empty()) + .all(|(key, _)| { + let allowed = matches!( + key.trim().to_lowercase().as_str(), + "replaygain_track_gain" + | "replaygain_track_peak" + | "replaygain_album_gain" + | "replaygain_album_peak" + ); + replay_gain |= allowed; + allowed + }); + let success = |method: &str| json!({"success":true,"method":method}); + if only_replay_gain && replay_gain && (m4a || mp4) { + self.edit_m4a_freeform(path, &fields, true, &check) + .map_err(|error| format!("failed to write M4A metadata: {error}"))?; + return Ok(success("native_m4a_replaygain")); + } + let required = if suffix(&[".flac"]) { + if mp4 { + return Err("failed to write FLAC metadata: file is an MP4/M4A stream under a .flac name; rename it to .m4a".into()); + } + Some(("flac", "native", "failed to write FLAC metadata")) + } else if suffix(&[".wav"]) { + Some(("wav", "native_wav", "failed to write WAV metadata")) + } else if suffix(&[".aiff", ".aif", ".aifc"]) { + Some(("aiff", "native_aiff", "failed to write AIFF metadata")) + } else if suffix(&[".ape", ".wv", ".mpc"]) { + Some(("ape", "native_ape", "failed to write APE tags")) + } else { + None + }; + if let Some((format, method, prefix)) = required { + self.edit_audio_tags(path, format, &fields, &check) + .map_err(|error| format!("{prefix}: {error}"))?; + return Ok(success(method)); + } + for (applicable, format, method) in [ + (suffix(&[".mp3"]), "mp3", "native_mp3"), + (suffix(&[".ogg", ".opus"]), "ogg", "native_ogg"), + (m4a || mp4, "m4a", "native_m4a"), + ] { + if applicable { + if self.edit_audio_tags(path, format, &fields, &check).is_ok() { + return Ok(success(method)); + } + // Recheck capabilities after a failed attempt too: a revoked + // grant or replaced symlink must not be handed to FFmpeg. + let files = self.manager.environment().native_files()?; + files.resolve_legacy(path)?.native_display()?; + if let Some(cover) = fields.get("cover_path").filter(|p| !p.trim().is_empty()) { + files.resolve_legacy(cover.trim())?.native_display()?; + } + } + check()?; + } + Ok( + json!({"success":true,"method":"ffmpeg","fields":if null_fields { Value::Null } else { json!(fields) }}), + ) + } + + /// Edit supported tag containers through the same file grants and output + /// locks as downloads/lyrics. Platform descriptors are staged by the native + /// adapter before calling this path-based, atomic publication operation. + pub fn edit_audio_tags( + &self, + path: &str, + format: &str, + fields: &BTreeMap<String, String>, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<(), String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + check()?; + if !matches!( + format, + "flac" + | "mp3" + | "m4a" + | "mp4" + | "m4b" + | "ogg" + | "opus" + | "wav" + | "aiff" + | "aif" + | "aifc" + | "ape" + | "wv" + | "mpc" + ) { + return Err(format!("unsupported tag writer: {format}")); + } + self.rewrite_tag_file(path, &check, |source, output, files| { + let cover = read_cover( + files, + fields.get("cover_path").map(|p| p.trim()), + matches!(format, "wav" | "aiff" | "aif" | "aifc"), + &check, + )?; + rewrite_audio_tags(source, output, format, fields, cover.as_deref(), &check)?; + Ok(true) + }) + .map(|_| ()) + } + + pub fn edit_m4a_freeform( + &self, + path: &str, + fields: &BTreeMap<String, String>, + replay_gain_only: bool, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<bool, String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + check()?; + let applicable = if replay_gain_only { + [ + "replaygain_track_gain", + "replaygain_track_peak", + "replaygain_album_gain", + "replaygain_album_peak", + ] + .iter() + .any(|key| fields.get(*key).is_some_and(|v| !v.trim().is_empty())) + } else { + fields.contains_key("isrc") || fields.contains_key("label") + }; + if !applicable { + return Ok(false); + } + self.rewrite_tag_file(path, &check, |source, output, _| { + rewrite_m4a_freeform(source, output, fields, replay_gain_only, &check) + }) + } + + pub fn ensure_ac4_config( + &self, + path: &str, + reference: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<(), String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + check()?; + self.rewrite_tag_file(path, &check, |source, output, files| { + rewrite_ac4_config( + source, + output, + || { + files + .resolve_legacy(reference)? + .open(OpenOptions::new().read(true)) + .map_err(|error| error.to_string()) + }, + &check, + ) + }) + .map(|_| ()) + } + + pub fn write_ac4_metadata( + &self, + path: &str, + metadata_json: &str, + cover_path: &str, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<bool, String> { + let _operation = self.enter()?; + let check = || self.check().and_then(|()| check()); + check()?; + self.rewrite_tag_file(path, &check, |source, output, files| { + rewrite_ac4_metadata( + source, + output, + metadata_json, + || read_cover(files, Some(cover_path), false, &check), + &check, + ) + }) + } + + // Callers hold the root operation guard and supply the combined root/lease check. + pub(super) fn rewrite_tag_file( + &self, + path: &str, + check: &dyn Fn() -> Result<(), String>, + edit: impl FnOnce(&mut File, &mut File, &ExtensionFiles) -> Result<bool, String>, + ) -> Result<bool, String> { + let files = self.manager.environment().native_files()?; + let target = files.resolve_legacy(path)?; + let _lock = files.lock(&target, check)?; + target.native_display()?; + target.require_parent()?; + let mut source = target + .open(OpenOptions::new().read(true).write(true)) + .map_err(|e| e.to_string())?; + let permissions = source.metadata().map_err(|e| e.to_string())?.permissions(); + let mut stage = target.stage_existing_parent().map_err(|e| e.to_string())?; + stage + .file + .set_permissions(permissions) + .map_err(|e| e.to_string())?; + let changed = edit(&mut source, &mut stage.file, &files)?; + check()?; + drop(source); + if changed { + stage.publish(check)?; + } + Ok(changed) + } +} + +fn read_cover( + files: &ExtensionFiles, + path: Option<&str>, + required: bool, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Option<Vec<u8>>, String> { + let Some(path) = path.filter(|path| !path.trim().is_empty()) else { + return Ok(None); + }; + let path = files.resolve_legacy(path)?; + // Go ignores a missing/unreadable cover while applying tags. + let mut file = match path.open(OpenOptions::new().read(true)) { + Ok(file) => file, + Err(error) if required => return Err(format!("read cover art: {error}")), + Err(_) => return Ok(None), + }; + if file.metadata().map_err(|e| e.to_string())?.len() > 64 * 1024 * 1024 { + return Err("cover exceeds 64 MiB".into()); + } + let mut data = Vec::new(); + let mut buffer = [0; 65536]; + loop { + check()?; + let count = file.read(&mut buffer).map_err(|e| e.to_string())?; + if count == 0 { + break; + } + if data.len() + count > 64 * 1024 * 1024 { + return Err("cover exceeds 64 MiB".into()); + } + data.extend_from_slice(&buffer[..count]); + } + if required && data.is_empty() { + return Err("cover art is empty".into()); + } + Ok(Some(data)) +} diff --git a/rust_backend/crates/extensions/src/binary.rs b/rust_backend/crates/extensions/src/binary.rs new file mode 100644 index 00000000..cf6211e0 --- /dev/null +++ b/rust_backend/crates/extensions/src/binary.rs @@ -0,0 +1,227 @@ +use crate::host::decode_go_utf8; +use base64::{ + Engine, alphabet, + engine::general_purpose::{GeneralPurpose, GeneralPurposeConfig, STANDARD}, +}; +use rquickjs::{Object, TypedArray, Value}; + +pub(crate) fn option_string( + options: Option<&Object<'_>>, + key: &str, + fallback: &str, +) -> Result<String, String> { + let Some(options) = options else { + return Ok(fallback.into()); + }; + let value: Value = options.get(key).map_err(|error| error.to_string())?; + if let Some(value) = value.as_string() { + let value = value.to_string().map_err(|error| error.to_string())?; + if !value.trim().is_empty() { + return Ok(value.trim().into()); + } + } else if let Ok(value) = TypedArray::<u8>::from_value(value) { + let bytes = copy_typed(&value); + if !bytes.is_empty() { + return Ok(decode_go_utf8(&bytes)); + } + } + Ok(fallback.into()) +} + +pub(crate) fn option_i64(options: &Object<'_>, key: &str, fallback: i64) -> Result<i64, String> { + let value: Value = options.get(key).map_err(|error| error.to_string())?; + if let Some(number) = value.as_number() { + return Ok(go_float_i64(number)); + } + if let Some(value) = value.as_string() { + let value = value.to_string().map_err(|error| error.to_string())?; + let value = value.trim(); + let prefix: String = value + .chars() + .enumerate() + .take_while(|(index, ch)| { + ch.is_ascii_digit() || (*index == 0 && (*ch == '+' || *ch == '-')) + }) + .map(|(_, ch)| ch) + .collect(); + return Ok(prefix.parse().unwrap_or(fallback)); + } + Ok(fallback) +} + +pub(crate) fn option_bool(options: &Object<'_>, key: &str, fallback: bool) -> Result<bool, String> { + let value: Value = options.get(key).map_err(|error| error.to_string())?; + if let Some(value) = value.as_bool() { + return Ok(value); + } + if let Some(value) = value.as_number() { + return Ok(value != 0.0); + } + if let Some(value) = value.as_string() { + return Ok( + match value + .to_string() + .map_err(|error| error.to_string())? + .trim() + .to_lowercase() + .as_str() + { + "1" | "true" | "yes" | "on" => true, + "0" | "false" | "no" | "off" => false, + _ => fallback, + }, + ); + } + Ok(fallback) +} + +pub(crate) fn decode_string(value: &str, encoding: &str) -> Result<Vec<u8>, String> { + match encoding.trim().to_lowercase().as_str() { + "" | "utf8" | "utf-8" | "text" => Ok(value.as_bytes().to_vec()), + "base64" => { + decode_base64(value.trim()).map_err(|error| format!("invalid base64 data: {error}")) + } + "hex" => { + let value = value.trim().as_bytes(); + let mut bytes = Vec::with_capacity(value.len() / 2); + let digit = |byte: u8| { + (byte as char) + .to_digit(16) + .map(|value| value as u8) + .ok_or_else(|| format!("invalid hex data: invalid byte 0x{byte:02x}")) + }; + for pair in value.chunks(2) { + let first = digit(pair[0])?; + if pair.len() < 2 { + return Err("invalid hex data: encoding/hex: odd length hex string".into()); + } + bytes.push(first * 16 + digit(pair[1])?); + } + Ok(bytes) + } + _ => Err(format!("unsupported byte encoding: {encoding}")), + } +} + +pub(crate) fn decode_base64(value: &str) -> Result<Vec<u8>, base64::DecodeError> { + GeneralPurpose::new( + &alphabet::STANDARD, + GeneralPurposeConfig::new().with_decode_allow_trailing_bits(true), + ) + .decode(value.replace(['\r', '\n'], "")) +} + +pub(crate) fn decode_option_bytes( + options: Option<&Object<'_>>, + key: &str, + encoding: &str, +) -> Result<Vec<u8>, String> { + if let Some(options) = options { + let value: Value = options.get(key).map_err(|error| error.to_string())?; + if let Ok(value) = TypedArray::<u8>::from_value(value) { + // runtimeOptionString converts []byte to a Go string without UTF-8 + // replacement or trimming. Preserve arbitrary binary keys and IVs. + if matches!( + encoding.trim().to_lowercase().as_str(), + "" | "utf8" | "utf-8" | "text" + ) { + return Ok(copy_typed(&value)); + } + } + } + decode_string(&option_string(options, key, "")?, encoding) +} + +pub(crate) fn encode(bytes: &[u8], encoding: &str) -> Result<String, String> { + match encoding.trim().to_lowercase().as_str() { + "" | "base64" => Ok(STANDARD.encode(bytes)), + "hex" => { + use std::fmt::Write; + let mut value = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(value, "{byte:02x}"); + } + Ok(value) + } + "utf8" | "utf-8" | "text" => Ok(decode_go_utf8(bytes)), + _ => Err(format!("unsupported byte encoding: {encoding}")), + } +} + +pub(crate) fn decode_value(value: Value<'_>, encoding: &str) -> Result<Vec<u8>, String> { + if let Some(text) = value.as_string() { + return decode_string( + &text.to_string().map_err(|error| error.to_string())?, + encoding, + ); + } + if let Ok(bytes) = TypedArray::<u8>::from_value(value.clone()) { + return Ok(copy_typed(&bytes)); + } + if let Some(array) = value.as_array() { + return array + .iter::<Value>() + .enumerate() + .map(|(index, item)| { + let value = item.map_err(|error| error.to_string())?; + let number = value + .as_number() + .ok_or_else(|| format!("unsupported byte array item at index {index}"))?; + // Goja exports integral values within int64 as int64. Other + // numbers use Go's float64 -> native int conversion first. + if number.fract() == 0.0 + && (-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&number) + { + return Ok(number as i64 as u8); + } + #[cfg(any(target_arch = "aarch64", target_arch = "arm"))] + return Ok(number as isize as u8); + #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] + Ok(go_float_i64(number) as u8) + }) + .collect(); + } + Err("unsupported byte payload type".into()) +} + +#[allow(unsafe_code)] +fn copy_typed(bytes: &TypedArray<'_, u8>) -> Vec<u8> { + // SAFETY: only this worker can access the VM. Finish the owned copy before + // any engine call, getter, callback, or operation that can detach a buffer. + unsafe { bytes.as_bytes() }.unwrap_or_default().to_vec() +} + +fn go_float_i64(number: f64) -> i64 { + #[cfg(target_arch = "aarch64")] + return number as i64; + #[cfg(target_arch = "arm")] + { + // Go ARM32 uses runtime._d2v for int64 conversions, while byte-array + // conversion above uses the hardware's native int32 instruction. + let bits = number.to_bits(); + let shift = ((bits >> 52) & 0x7ff) as i32 - 1075; + let mantissa = (bits & ((1_u64 << 52) - 1)) | (1_u64 << 52); + let magnitude = if shift < -63 { + 0 + } else if shift < 0 { + mantissa >> -shift + } else if shift <= 11 { + mantissa << shift + } else { + u64::from(number as u32) << 32 + }; + return if number.is_sign_negative() { + magnitude.wrapping_neg() as i64 + } else { + magnitude as i64 + }; + } + #[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))] + if number.is_finite() + && (-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&number) + { + number as i64 + } else { + i64::MIN + } +} diff --git a/rust_backend/crates/extensions/src/crypto.rs b/rust_backend/crates/extensions/src/crypto.rs new file mode 100644 index 00000000..73a8540f --- /dev/null +++ b/rust_backend/crates/extensions/src/crypto.rs @@ -0,0 +1,150 @@ +//! Block modes retained for the existing extension API and media transforms. + +use aes::cipher::{Block, BlockDecrypt, BlockEncrypt, KeyInit}; +use zeroize::Zeroizing; + +pub enum BlockCipher { + Aes128(Box<aes::Aes128>), + Aes192(Box<aes::Aes192>), + Aes256(Box<aes::Aes256>), + Blowfish(Box<blowfish::Blowfish>), +} + +impl BlockCipher { + pub fn new(algorithm: &str, key: &[u8]) -> Result<Self, String> { + Ok(match algorithm { + "aes" => match key.len() { + 16 => Self::Aes128(Box::new( + aes::Aes128::new_from_slice(key).expect("AES-128 key"), + )), + 24 => Self::Aes192(Box::new( + aes::Aes192::new_from_slice(key).expect("AES-192 key"), + )), + 32 => Self::Aes256(Box::new( + aes::Aes256::new_from_slice(key).expect("AES-256 key"), + )), + size => return Err(format!("crypto/aes: invalid key size {size}")), + }, + "blowfish" => { + if key.is_empty() || key.len() > 56 { + return Err(format!("crypto/blowfish: invalid key size {}", key.len())); + } + // Go accepts 1–56 bytes; RustCrypto starts at four. Repeating a + // short key preserves Blowfish's cyclic key expansion exactly. + let key = Zeroizing::new(if key.len() < 4 { + key.repeat(4_usize.div_ceil(key.len())) + } else { + key.to_vec() + }); + Self::Blowfish(Box::new( + blowfish::Blowfish::new_from_slice(&key).expect("Blowfish key"), + )) + } + _ => return Err(format!("unsupported block cipher algorithm: {algorithm}")), + }) + } + + pub fn block_size(&self) -> usize { + if matches!(self, Self::Blowfish(_)) { + 8 + } else { + 16 + } + } + + /// CBC requires aligned input; CTR increments the entire big-endian counter + /// and can transform a partial final block. Input is an owned Rust snapshot. + pub fn transform( + &self, + data: &mut [u8], + iv: &[u8], + mode: &str, + decrypt: bool, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<(), String> { + let size = self.block_size(); + if iv.len() != size { + return Err(format!("iv must be {size} bytes")); + } + if mode != "cbc" && mode != "ctr" { + return Err(format!("unsupported block cipher mode: {mode}")); + } + if mode == "cbc" && !data.len().is_multiple_of(size) { + return Err(format!("input length must be a multiple of {size} bytes")); + } + match self { + Self::Aes128(cipher) => transform(cipher.as_ref(), data, iv, mode, decrypt, check), + Self::Aes192(cipher) => transform(cipher.as_ref(), data, iv, mode, decrypt, check), + Self::Aes256(cipher) => transform(cipher.as_ref(), data, iv, mode, decrypt, check), + Self::Blowfish(cipher) => transform(cipher.as_ref(), data, iv, mode, decrypt, check), + } + } +} + +fn transform<C: BlockEncrypt + BlockDecrypt>( + cipher: &C, + data: &mut [u8], + iv: &[u8], + mode: &str, + decrypt: bool, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(), String> { + let mut state = Block::<C>::clone_from_slice(iv); + for (index, chunk) in data.chunks_mut(iv.len()).enumerate() { + if index % 1024 == 0 { + check()?; + } + if mode == "ctr" { + let mut block = state.clone(); + cipher.encrypt_block(&mut block); + for (byte, mask) in chunk.iter_mut().zip(block.iter()) { + *byte ^= mask; + } + for byte in state.iter_mut().rev() { + *byte = byte.wrapping_add(1); + if *byte != 0 { + break; + } + } + } else if decrypt { + let encrypted = Block::<C>::clone_from_slice(chunk); + let mut block = encrypted.clone(); + cipher.decrypt_block(&mut block); + for ((byte, plain), previous) in chunk.iter_mut().zip(block.iter()).zip(state.iter()) { + *byte = plain ^ previous; + } + state = encrypted; + } else { + for (byte, previous) in chunk.iter_mut().zip(state.iter()) { + *byte ^= previous; + } + let block = Block::<C>::from_mut_slice(chunk); + cipher.encrypt_block(block); + state.clone_from(block); + } + } + check() +} + +pub fn pad(data: &mut Vec<u8>, block_size: usize) { + let count = block_size - data.len() % block_size; + data.resize(data.len() + count, count as u8); +} + +pub fn unpad(data: &mut Vec<u8>, block_size: usize) -> Result<(), String> { + if data.is_empty() || !data.len().is_multiple_of(block_size) { + return Err("invalid padded payload length".into()); + } + let count = usize::from(*data.last().expect("nonempty padded data")); + if count == 0 + || count > block_size + || count > data.len() + || data[data.len() - count..] + .iter() + .any(|byte| usize::from(*byte) != count) + { + return Err("invalid PKCS7 padding".into()); + } + data.truncate(data.len() - count); + Ok(()) +} diff --git a/rust_backend/crates/extensions/src/crypto_host.rs b/rust_backend/crates/extensions/src/crypto_host.rs new file mode 100644 index 00000000..4d46f930 --- /dev/null +++ b/rust_backend/crates/extensions/src/crypto_host.rs @@ -0,0 +1,272 @@ +use crate::{ + binary, + crypto::{self, BlockCipher}, + host::decode_go_utf8, + runtime::Control, + storage, +}; +use aes_gcm::aead::{OsRng, rand_core::RngCore}; +use base64::{Engine, engine::general_purpose::STANDARD}; +use rquickjs::{ArrayBuffer, Ctx, Function, Object, Value}; +use sha2::{Digest, Sha256}; +use std::sync::Arc; +use zeroize::Zeroizing; + +pub(crate) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, +) -> rquickjs::Result<()> { + let text_control = Arc::clone(&control); + host.set( + "cryptoText", + Function::new( + ctx.clone(), + move |ctx: Ctx<'js>, decrypt: bool, data: String, key: String| { + let data = Zeroizing::new(data); + let key = Zeroizing::new(key); + let key = Zeroizing::new(<[u8; 32]>::from(Sha256::digest(key.as_bytes()))); + let result = (|| { + text_control.check().map_err(|error| error.to_string())?; + let output = if decrypt { + let bytes = Zeroizing::new( + binary::decode_base64(&data) + .map_err(|_| "invalid base64 ciphertext".to_owned())?, + ); + let plain = storage::decrypt(&bytes, &key) + .map_err(|_| "invalid base64 ciphertext".to_owned())?; + decode_go_utf8(&plain) + } else { + STANDARD.encode( + storage::encrypt(data.as_bytes(), &key) + .map_err(|error| error.to_string())?, + ) + }; + text_control.check().map_err(|error| error.to_string())?; + Ok(output) + })(); + let object = result_object(&ctx, result.as_ref().err())?; + if let Ok(data) = result { + object.set("data", data)?; + } + Ok::<_, rquickjs::Error>(object) + }, + )?, + )?; + host.set( + "generateKey", + Function::new(ctx.clone(), |ctx: Ctx<'js>, length: u32| { + let result = (|| { + if !(1..=4096).contains(&length) { + return Err("key length must be an integer between 1 and 4096 bytes".to_owned()); + } + let mut bytes = Zeroizing::new(vec![0; length as usize]); + OsRng + .try_fill_bytes(&mut bytes) + .map_err(|error| error.to_string())?; + Ok(bytes) + })(); + let object = result_object(&ctx, result.as_ref().err())?; + if let Ok(bytes) = result { + object.set("key", STANDARD.encode(&*bytes))?; + object.set("hex", binary::encode(&bytes, "hex").expect("hex encoding"))?; + } + Ok::<_, rquickjs::Error>(object) + })?, + )?; + host.set( + "blockTransform", + Function::new( + ctx.clone(), + move |ctx: Ctx<'js>, operation: String, data: Value<'js>, options: Value<'js>| { + let check = || control.check().map_err(|error| error.to_string()); + let result = if operation == "segments" { + segments(data, options, &check) + } else { + block(data, options, operation == "decrypt", &check) + }; + let object = result_object(&ctx, result.as_ref().err())?; + if let Ok(result) = result { + object.set(result.count_name, result.count)?; + if result.raw { + // Charge retained output to QuickJS's heap; external Rust-owned + // ArrayBuffers would bypass the runtime's memory limit. + object.set("data", ArrayBuffer::new_copy(ctx.clone(), &*result.bytes)?)?; + } else { + match binary::encode(&result.bytes, &result.encoding) { + Ok(data) => object.set("data", data)?, + Err(error) => return result_object(&ctx, Some(&error)), + } + } + } + Ok(object) + }, + )?, + )?; + Ok(()) +} + +fn result_object<'js>(ctx: &Ctx<'js>, error: Option<&String>) -> rquickjs::Result<Object<'js>> { + let object = Object::new(ctx.clone())?; + object.set("success", error.is_none())?; + if let Some(error) = error { + object.set("error", error.as_str())?; + } + Ok(object) +} + +struct Output { + bytes: Zeroizing<Vec<u8>>, + encoding: String, + raw: bool, + count_name: &'static str, + count: usize, +} + +fn block( + data: Value<'_>, + options: Value<'_>, + decrypt: bool, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Output, String> { + check()?; + let options = options.as_object(); + let text = |key, fallback| binary::option_string(options, key, fallback); + let algorithm = text("algorithm", "")?.to_lowercase(); + let mode = text("mode", "cbc")?.to_lowercase(); + let input_encoding = text("inputEncoding", "base64")?.to_lowercase(); + let encoding = text("outputEncoding", "base64")?.to_lowercase(); + let padding = text("padding", "none")?.to_lowercase(); + if algorithm.is_empty() { + return Err("algorithm is required".into()); + } + let key = Zeroizing::new( + binary::decode_option_bytes(options, "key", &text("keyEncoding", "utf8")?) + .map_err(|error| format!("invalid key: {error}"))?, + ); + if key.is_empty() { + return Err("key is required".into()); + } + let iv = binary::decode_option_bytes(options, "iv", &text("ivEncoding", "utf8")?) + .map_err(|error| format!("invalid iv: {error}"))?; + if mode != "cbc" && mode != "ctr" { + return Err(format!("unsupported block cipher mode: {mode}")); + } + let mut bytes = Zeroizing::new(binary::decode_value(data, &input_encoding)?); + let cipher = BlockCipher::new(&algorithm, &key)?; + let size = cipher.block_size(); + if iv.len() != size { + return Err(format!( + "{} must be {size} bytes for {algorithm}", + if mode == "ctr" { "iv (counter)" } else { "iv" } + )); + } + if mode == "cbc" && !decrypt && padding == "pkcs7" { + crypto::pad(&mut bytes, size); + } + cipher.transform(&mut bytes, &iv, &mode, decrypt, check)?; + if mode == "cbc" && decrypt && padding == "pkcs7" { + crypto::unpad(&mut bytes, size)?; + } + Ok(Output { + bytes, + encoding, + raw: false, + count_name: "block_size", + count: size, + }) +} + +fn segments( + data: Value<'_>, + options: Value<'_>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Output, String> { + check()?; + let options = options.as_object().ok_or("options object is required")?; + let text = |key, fallback| binary::option_string(Some(options), key, fallback); + let algorithm = text("algorithm", "aes")?.to_lowercase(); + let input_encoding = text("inputEncoding", "base64")?.to_lowercase(); + let encoding = text("outputEncoding", "base64")?.to_lowercase(); + let iv_encoding = text("ivEncoding", "base64")?.to_lowercase(); + let key = Zeroizing::new( + binary::decode_option_bytes(Some(options), "key", &text("keyEncoding", "hex")?) + .map_err(|error| format!("invalid key: {error}"))?, + ); + if key.is_empty() { + return Err("key is required".into()); + } + if algorithm != "aes" && algorithm != "blowfish" { + return Err(format!("unsupported algorithm: {algorithm}")); + } + let cipher = BlockCipher::new(&algorithm, &key)?; + let size = cipher.block_size(); + let raw_input = input_encoding == "bytes" || input_encoding == "raw"; + let mut bytes = Zeroizing::new( + binary::decode_value(data, if raw_input { "" } else { &input_encoding }).map_err( + |error| { + if raw_input { + format!("invalid byte payload: {error}") + } else { + error + } + }, + )?, + ); + let segments: Value = options.get("segments").map_err(|error| error.to_string())?; + if segments.is_null() || segments.is_undefined() { + return Err("segments array is required".into()); + } + let segments = segments.as_array().ok_or("segments must be an array")?; + let mut count = 0; + for (index, segment) in segments.iter::<Value>().enumerate() { + check()?; + let segment = segment.map_err(|error| error.to_string())?; + let segment = segment + .as_object() + .ok_or_else(|| format!("segment {index} is not an object"))?; + let offset = binary::option_i64(segment, "offset", -1)?; + let length = binary::option_i64(segment, "size", -1)?; + if offset < 0 || length < 0 { + return Err(format!("segment {index} has invalid offset/size")); + } + if length == 0 { + continue; + } + let offset = offset as u64; + let length = length as u64; + if offset + .checked_add(length) + .is_none_or(|end| end > bytes.len() as u64) + { + return Err(format!( + "segment {index} out of bounds (offset={offset} size={length} len={})", + bytes.len() + )); + } + let mut iv = binary::decode_option_bytes(Some(segment), "iv", &iv_encoding) + .map_err(|error| format!("segment {index} has invalid iv: {error}"))?; + if iv.len() > size { + return Err(format!( + "segment {index} iv longer than block size ({} > {size})", + iv.len() + )); + } + iv.resize(size, 0); + cipher.transform( + &mut bytes[offset as usize..(offset + length) as usize], + &iv, + "ctr", + true, + check, + )?; + count += 1; + } + Ok(Output { + bytes, + raw: encoding == "bytes" || encoding == "raw", + encoding, + count_name: "segments_processed", + count, + }) +} diff --git a/rust_backend/crates/extensions/src/download/chunked.rs b/rust_backend/crates/extensions/src/download/chunked.rs new file mode 100644 index 00000000..09f15e6c --- /dev/null +++ b/rust_backend/crates/extensions/src/download/chunked.rs @@ -0,0 +1,361 @@ +use super::store::DownloadFile; +use super::{ + DownloadOptions, Failure, fingerprint, header, next_delay, retryable, set_header, + status_failure, validator, +}; +use crate::files::FilePath; +use spotiflac_network::{HttpRequest, HttpResponse, HttpStream, NetworkSession}; +use std::io::Write; +use std::time::Duration; + +#[allow(clippy::too_many_arguments)] +pub(super) fn download( + path: &FilePath, + network: &NetworkSession, + url: &str, + options: DownloadOptions, + chunk_size: u64, + check: &dyn Fn() -> Result<(), String>, + wait: &dyn Fn(Duration) -> Result<(), String>, + progress: &mut dyn FnMut(u64, i64), +) -> Result<serde_json::Value, Failure> { + options.item.start(); + let mut attempts = 0; + let mut delay = Duration::from_millis(options.policy.initial_retry_delay_ms as u64); + let probe = loop { + attempts += 1; + check().map_err(|error| Failure::new("cancelled", error, attempts))?; + let response = open(network, url, &options, "bytes=0-1", "", false, check); + let failure = match response { + Ok(stream) if matches!(stream.response.status, 200 | 206) => break stream.response, + Ok(stream) => { + let mut failure = status_failure(&stream.response, &options.policy, attempts); + failure.error = format!("chunked probe HTTP {}", stream.response.status); + failure + } + Err(error) => Failure::new( + "transient_network", + format!("chunked probe failed: {error}"), + attempts, + ), + }; + check().map_err(|error| Failure::new("cancelled", error, attempts))?; + if attempts >= options.policy.max_attempts + || (failure.http_status > 0 && !retryable(failure.http_status)) + { + return Err(failure); + } + wait(retry_delay(&failure, delay)) + .map_err(|error| Failure::new("cancelled", error, attempts))?; + delay = next_delay(delay, &options.policy); + }; + let total = total_length(&probe); + let validator = validator(&probe); + let fingerprint = fingerprint(url); + let mut output = DownloadFile::open( + path, + &fingerprint, + options.persistent && !validator.is_empty() && !fingerprint.is_empty(), + ) + .map_err(|error| { + Failure::new( + "storage_error", + format!("failed to create chunked staged file: {error}"), + 0, + ) + })?; + if output.restored + && (output.state.validator != validator + || (output.state.total > 0 && total > 0 && output.state.total != total)) + { + output + .restart() + .map_err(|error| Failure::new("storage_error", error, 0))?; + } + output.state.validator = validator.clone(); + output.state.total = total; + options.item.initial(output.state.bytes, total); + let reporter = options.item.reporter(output.state.bytes, total); + let mut notified = output.state.bytes; + let mut completed_chunk = false; + let mut buffer = [0; 64 << 10]; + while output.state.total <= 0 || output.state.bytes < output.state.total as u64 { + let mut start = output.state.bytes; + let mut end = start.saturating_add(chunk_size - 1); + if output.state.total > 0 { + end = end.min(output.state.total as u64 - 1); + } + let mut delay = Duration::from_millis(options.policy.initial_retry_delay_ms as u64); + let mut full_response = false; + let mut chunk_complete = false; + let mut failure = Failure::new("transient_network", "chunked transfer failed", attempts); + for attempt in 1..=options.policy.max_attempts { + attempts += 1; + check().map_err(|error| Failure::new("cancelled", error, attempts))?; + let continuation = attempt == 1 && completed_chunk; + let response = open( + network, + url, + &options, + &format!("bytes={start}-{end}"), + &validator, + continuation, + check, + ); + let mut stream = match response { + Ok(stream) => stream, + Err(error) => { + check().map_err(|error| Failure::new("cancelled", error, attempts))?; + failure = Failure::new( + "transient_network", + format!("chunked request at {start} failed: {error}"), + attempts, + ); + if attempt == options.policy.max_attempts { + break; + } + wait(delay).map_err(|error| Failure::new("cancelled", error, attempts))?; + delay = next_delay(delay, &options.policy); + continue; + } + }; + let status = stream.response.status; + if !matches!(status, 200 | 206) { + failure = status_failure(&stream.response, &options.policy, attempts); + failure.error = format!("chunked HTTP {status} at offset {start}"); + drop(stream); + if !retryable(status) || attempt == options.policy.max_attempts { + break; + } + wait(retry_delay(&failure, delay)) + .map_err(|error| Failure::new("cancelled", error, attempts))?; + delay = next_delay(delay, &options.policy); + continue; + } + let range = header(&stream.response, "Content-Range"); + if status == 206 && !range.starts_with(&format!("bytes {start}-")) { + let mut failure = Failure::new( + "integrity_failed", + format!("chunked response has unexpected Content-Range {range:?} at {start}"), + attempts, + ); + failure.http_status = status; + return Err(failure); + } + let next_validator = super::validator(&stream.response); + if !next_validator.is_empty() && !validator.is_empty() && next_validator != validator { + return Err(Failure::new( + "integrity_failed", + "chunked response validator changed during transfer", + attempts, + )); + } + let length = header(&stream.response, "Content-Length") + .parse::<i64>() + .unwrap_or(-1); + if status == 200 && start > 0 { + output.restart().map_err(|error| { + Failure::new( + "storage_error", + format!("failed to restart ignored range: {error}"), + attempts, + ) + })?; + output.state.validator = validator.clone(); + start = 0; + if length > 0 { + output.state.total = length; + } + } + let mut received = continuation; + let body_result = loop { + let guard = options + .budget + .as_ref() + .map(|budget| budget.enter(!received)); + let read = stream.read(&mut buffer, check); + drop(guard); + match read { + Ok(0) => break Ok(()), + Ok(count) => { + received = true; + output.file.write_all(&buffer[..count]).map_err(|error| { + Failure::new( + "storage_error", + format!("failed to write chunked output: {error}"), + attempts, + ) + })?; + output.state.bytes += count as u64; + reporter.report(output.state.bytes as i64, output.state.total); + if output.state.total > 0 + && (output.state.bytes.saturating_sub(notified) >= 128 << 10 + || output.state.bytes >= output.state.total as u64) + { + notified = output.state.bytes; + let _charge = options.budget.as_ref().map(|budget| budget.enter(true)); + progress(output.state.bytes, output.state.total); + } + } + Err(error) => break Err(error), + } + }; + drop(stream); + let written = output.state.bytes - start; + let expected = if status == 206 && length <= 0 { + end.saturating_sub(start).saturating_add(1) as i64 + } else { + length + }; + let body_result = body_result.and_then(|()| { + if expected > 0 && written != expected as u64 { + Err("unexpected EOF".into()) + } else { + Ok(()) + } + }); + if body_result.is_ok() && written > 0 { + full_response = status == 200; + chunk_complete = true; + completed_chunk = true; + break; + } + output.rewind(start).map_err(|error| { + Failure::new( + "storage_error", + format!("failed to roll back incomplete chunk: {error}"), + attempts, + ) + })?; + if output.state.total > 0 { + options.item.set( + start as f64 / output.state.total as f64, + start, + output.state.total, + ); + } + failure = Failure::new( + "transient_network", + body_result.err().map_or_else( + || format!("chunk at {start} was empty"), + |error| format!("failed to read chunk at {start}: {error}"), + ), + attempts, + ); + if let Err(error) = check() { + output.checkpoint(true); + return Err(Failure::new("cancelled", error, attempts)); + } + if attempt == options.policy.max_attempts { + break; + } + wait(delay).map_err(|error| Failure::new("cancelled", error, attempts))?; + delay = next_delay(delay, &options.policy); + } + if !chunk_complete { + output.checkpoint(true); + return Err(failure); + } + output.checkpoint(false); + if full_response || (output.state.total <= 0 && output.state.bytes - start < chunk_size) { + break; + } + } + if output.state.bytes == 0 + || (output.state.total > 0 && output.state.bytes != output.state.total as u64) + { + return Err(Failure::new( + "integrity_failed", + format!( + "chunked transfer size mismatch: expected {} bytes, wrote {}", + output.state.total, output.state.bytes + ), + attempts, + )); + } + output.publish(check).map_err(|error| { + Failure::new( + "storage_error", + format!("failed to publish file: {error}"), + attempts, + ) + })?; + options + .item + .set(1.0, output.state.bytes, output.state.bytes as i64); + Ok( + serde_json::json!({"success":true,"path":path.display(),"size":output.state.bytes,"attempts":attempts}), + ) +} + +#[allow(clippy::too_many_arguments)] +fn open( + network: &NetworkSession, + url: &str, + options: &DownloadOptions, + range: &str, + validator: &str, + continuation: bool, + check: &dyn Fn() -> Result<(), String>, +) -> Result<HttpStream, String> { + let mut headers = options.headers.clone(); + set_header(&mut headers, "Range", range.to_owned()); + if !validator.is_empty() { + set_header(&mut headers, "If-Range", validator.to_owned()); + } + let user_agent = options + .headers + .get("User-Agent") + .filter(|value| !value.is_empty()) + .cloned() + .unwrap_or_else(|| options.app_version.user_agent()); + set_header(&mut headers, "User-Agent", user_agent.clone()); + let _guard = options + .budget + .as_ref() + .map(|budget| budget.enter(!continuation)); + network.open_stream( + HttpRequest { + url: url.to_owned(), + method: "GET".into(), + body: String::new(), + headers, + default_json: false, + user_agent, + }, + Duration::from_secs(24 * 60 * 60), + Duration::from_secs(60), + check, + ) +} + +fn total_length(response: &HttpResponse) -> i64 { + if let Some((_, total)) = header(response, "Content-Range").rsplit_once('/') { + // Go's chunk probe uses Sscanf rather than the ordinary strict parser. + let prefix: String = total + .trim_start() + .chars() + .enumerate() + .take_while(|(index, ch)| { + ch.is_ascii_digit() || (*index == 0 && (*ch == '-' || *ch == '+')) + }) + .map(|(_, ch)| ch) + .collect(); + if let Ok(total) = prefix.parse() { + return total; + } + } + if response.status == 200 { + header(response, "Content-Length").parse().unwrap_or(-1) + } else { + 0 + } +} + +fn retry_delay(failure: &Failure, fallback: Duration) -> Duration { + if failure.retry_after_seconds > 0 { + Duration::from_secs(failure.retry_after_seconds.into()) + } else { + fallback + } +} diff --git a/rust_backend/crates/extensions/src/download/mod.rs b/rust_backend/crates/extensions/src/download/mod.rs new file mode 100644 index 00000000..208403ae --- /dev/null +++ b/rust_backend/crates/extensions/src/download/mod.rs @@ -0,0 +1,494 @@ +mod chunked; +pub(crate) mod progress; +mod segment_store; +pub(crate) mod segments; +mod store; + +use crate::files::{ExtensionFiles, FilePath}; +use crate::transfer_policy::DownloadTransferPolicy; +use aes_gcm::aead::{OsRng, rand_core::RngCore}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use spotiflac_core::app_version::AppVersion; +use spotiflac_network::{HttpRequest, HttpResponse, NetworkSession, url::UrlParts}; +use std::collections::BTreeMap; +use std::io::Write; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use store::DownloadFile; + +#[derive(Serialize)] +pub(crate) struct Failure { + success: bool, + error: String, + error_type: &'static str, + attempts: i64, + #[serde(skip_serializing_if = "is_zero")] + http_status: u16, + #[serde(skip_serializing_if = "is_zero")] + retry_after_seconds: u16, +} + +fn is_zero(value: &u16) -> bool { + *value == 0 +} + +/// Go's native exports share this envelope, including file-only operations. +/// It is distinct from the JS SDK transfer failure payload above. +pub(crate) fn native_error_response(message: &str) -> serde_json::Value { + let lower = message.to_lowercase(); + let kind = [ + ( + "isp_blocked", + &["isp blocking", "try using vpn", "change dns"][..], + ), + ("cancelled", &["cancel"][..]), + ( + "verification_required", + &[ + "verification_required", + "session is not authenticated", + "signed session is not authenticated", + "signed session expired", + ][..], + ), + ( + "provider_reauth_required", + &["byoa_provider_reauth_required", "reauth_provider"][..], + ), + ("request_auth_invalid", &["request_auth_invalid"][..]), + ("provider_auth_failed", &["provider_auth_failed"][..]), + ("provider_unavailable", &["provider_unavailable"][..]), + ( + "rate_limit", + &[ + "rate limit", + "http 429", + "http status 429", + "status 429", + "429 for ", + "429:", + "429;", + "too many requests", + ][..], + ), + ( + "permission", + &[ + "permission", + "operation not permitted", + "access denied", + "failed to create file", + "failed to create directory", + ][..], + ), + ( + "not_found", + &[ + "not found", + "not available", + "no results", + "track not found", + "all services failed", + ][..], + ), + ("network", &["network", "connection", "timeout", "dial"][..]), + ] + .into_iter() + .find(|(_, patterns)| patterns.iter().any(|pattern| lower.contains(pattern))) + .map_or("unknown", |(kind, _)| kind); + serde_json::json!({"success":false,"message":"","error":message,"error_type":kind}) +} + +impl Failure { + pub(crate) fn new(kind: &'static str, error: impl ToString, attempts: i64) -> Self { + Self { + success: false, + error: error.to_string(), + error_type: kind, + attempts, + http_status: 0, + retry_after_seconds: 0, + } + } +} + +pub(crate) struct DownloadOptions { + pub item: progress::ItemProgressTarget, + pub headers: BTreeMap<String, String>, + pub app_version: AppVersion, + pub resume: bool, + pub persistent: bool, + pub policy: DownloadTransferPolicy, + pub budget: Option<Arc<crate::resolution::ResolutionBudget>>, + pub chunk_size: Option<u64>, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn download( + files: &ExtensionFiles, + path: &FilePath, + network: &NetworkSession, + url: &str, + mut options: DownloadOptions, + check: &dyn Fn() -> Result<(), String>, + wait: &dyn Fn(Duration) -> Result<(), String>, + progress: &mut dyn FnMut(u64, i64), +) -> Result<serde_json::Value, Failure> { + let _guard = files + .lock(path, check) + .map_err(|error| Failure::new("cancelled", error, 0))?; + if let Some(chunk_size) = options.chunk_size { + return chunked::download( + path, network, url, options, chunk_size, check, wait, progress, + ); + } + let caller_range = options + .headers + .keys() + .any(|name| name.eq_ignore_ascii_case("Range")); + if caller_range { + options.resume = false; + options.persistent = false; + } + let fingerprint = fingerprint(url); + let mut output = DownloadFile::open( + path, + &fingerprint, + options.resume && options.persistent && !fingerprint.is_empty(), + ) + .map_err(|error| { + Failure::new( + "storage_error", + format!("failed to create staged file: {error}"), + 0, + ) + })?; + options.item.start(); + options.item.initial(output.state.bytes, output.state.total); + let reporter = options + .item + .reporter(output.state.bytes, output.state.total); + if output.restored + && output.state.bytes > 0 + && output.state.total > 0 + && output.state.bytes == output.state.total as u64 + { + output.publish(check).map_err(|error| { + Failure::new( + "storage_error", + format!("failed to publish restored download: {error}"), + 0, + ) + })?; + options + .item + .set(1.0, output.state.bytes, output.state.total); + return Ok( + serde_json::json!({"success":true,"path":path.display(),"size":output.state.bytes,"attempts":0,"resumed":true}), + ); + } + let mut retry_delay = Duration::from_millis(options.policy.initial_retry_delay_ms as u64); + let mut notified = 0_u64; + for attempt in 1..=options.policy.max_attempts { + check().map_err(|error| Failure::new("cancelled", error, attempt))?; + let mut range_from = if options.resume && !output.state.validator.is_empty() { + output.state.bytes + } else { + 0 + }; + let mut headers = options.headers.clone(); + if range_from > 0 { + set_header(&mut headers, "Range", format!("bytes={range_from}-")); + set_header(&mut headers, "If-Range", output.state.validator.clone()); + } + let request = HttpRequest { + url: url.to_owned(), + method: "GET".into(), + body: String::new(), + headers, + default_json: false, + user_agent: options.app_version.user_agent(), + }; + let charge = options.budget.as_ref().map(|budget| budget.enter(true)); + let stream = network.open_stream( + request, + Duration::from_secs(24 * 60 * 60), + Duration::from_secs(60), + check, + ); + drop(charge); + let mut stream = match stream { + Ok(stream) => stream, + Err(error) => { + check().map_err(|error| Failure::new("cancelled", error, attempt))?; + if attempt == options.policy.max_attempts { + return Err(Failure::new("transient_network", error, attempt)); + } + if output.state.bytes > 0 && (!options.resume || output.state.validator.is_empty()) + { + output + .restart() + .map_err(|error| Failure::new("storage_error", error, attempt))?; + } + wait(retry_delay).map_err(|error| Failure::new("cancelled", error, attempt))?; + retry_delay = next_delay(retry_delay, &options.policy); + continue; + } + }; + let response = &stream.response; + let status = response.status; + if !(200..300).contains(&status) { + let failure = status_failure(response, &options.policy, attempt); + let delay = if failure.retry_after_seconds > 0 { + Duration::from_secs(failure.retry_after_seconds.into()) + } else { + retry_delay + }; + drop(stream); + if !retryable(status) || attempt == options.policy.max_attempts { + return Err(failure); + } + wait(delay).map_err(|error| Failure::new("cancelled", error, attempt))?; + retry_delay = next_delay(retry_delay, &options.policy); + continue; + } + if range_from > 0 && status == 206 { + let range = header(response, "Content-Range"); + if !range.starts_with(&format!("bytes {range_from}-")) { + let mut failure = Failure::new( + "integrity_failed", + format!( + "resume failed: unexpected Content-Range {range:?} at {range_from} bytes" + ), + attempt, + ); + failure.http_status = status; + return Err(failure); + } + let validator = validator(response); + if !validator.is_empty() && validator != output.state.validator { + let mut failure = Failure::new( + "integrity_failed", + "resume failed: response validator changed", + attempt, + ); + failure.http_status = status; + return Err(failure); + } + } + if range_from > 0 && status == 200 { + output + .rewind(0) + .map_err(|error| Failure::new("storage_error", error, attempt))?; + range_from = 0; + } + let validator = validator(response); + if !validator.is_empty() { + output.state.validator = validator; + } + let length = header(response, "Content-Length") + .parse::<i64>() + .unwrap_or(-1); + output.state.total = if status == 206 && !caller_range { + header(response, "Content-Range") + .rsplit_once('/') + .and_then(|(_, total)| total.parse().ok()) + .unwrap_or(if length > 0 { + (range_from as i64).saturating_add(length) + } else { + 0 + }) + } else { + length + }; + if output.state.total > 0 { + reporter.report(output.state.bytes as i64, output.state.total); + } + let mut buffer = [0; 64 << 10]; + let mut received_bytes = false; + let body_result = loop { + let budget = options + .budget + .as_ref() + .map(|budget| budget.enter(!received_bytes || !matches!(status, 200 | 206))); + let read = stream.read(&mut buffer, check); + drop(budget); + match read { + Ok(0) => break Ok(()), + Ok(count) => { + received_bytes = true; + if let Err(error) = output.file.write_all(&buffer[..count]) { + return Err(Failure::new( + "storage_error", + format!("failed to write staged file: {error}"), + attempt, + )); + } + output.state.bytes += count as u64; + reporter.report(output.state.bytes as i64, output.state.total); + if output.state.total > 0 + && (output.state.bytes.saturating_sub(notified) >= 128 << 10 + || output.state.bytes >= output.state.total as u64) + { + notified = output.state.bytes; + let _charge = options.budget.as_ref().map(|budget| budget.enter(true)); + progress(output.state.bytes, output.state.total); + } + output.checkpoint(false); + } + Err(error) => break Err(error), + } + }; + drop(stream); + let body_result = body_result.and_then(|()| { + if output.state.total > 0 && output.state.bytes != output.state.total as u64 { + Err("unexpected EOF".into()) + } else { + Ok(()) + } + }); + if let Err(error) = body_result { + output.checkpoint(true); + check().map_err(|error| Failure::new("cancelled", error, attempt))?; + if attempt == options.policy.max_attempts { + return Err(Failure::new( + "transient_network", + format!("failed to read response: {error}"), + attempt, + )); + } + if !options.resume || output.state.validator.is_empty() { + output + .restart() + .map_err(|error| Failure::new("storage_error", error, attempt))?; + } + wait(retry_delay).map_err(|error| Failure::new("cancelled", error, attempt))?; + retry_delay = next_delay(retry_delay, &options.policy); + continue; + } + if output.state.bytes == 0 { + return Err(Failure::new( + "integrity_failed", + "download response was empty", + attempt, + )); + } + output.publish(check).map_err(|error| { + Failure::new( + "storage_error", + format!("failed to publish file: {error}"), + 0, + ) + })?; + if output.state.total > 0 { + options + .item + .set(1.0, output.state.bytes, output.state.total); + } else { + options.item.received(output.state.bytes); + } + return Ok( + serde_json::json!({"success":true,"path":path.display(),"size":output.state.bytes,"attempts":attempt}), + ); + } + unreachable!("policy has at least one attempt") +} + +fn set_header(headers: &mut BTreeMap<String, String>, name: &str, value: String) { + headers.retain(|key, _| !key.eq_ignore_ascii_case(name)); + headers.insert(name.to_owned(), value); +} + +fn header<'a>(response: &'a HttpResponse, name: &str) -> &'a str { + response + .headers + .get(name) + .and_then(|values| values.first()) + .map_or("", String::as_str) +} + +fn validator(response: &HttpResponse) -> String { + let etag = header(response, "Etag").trim(); + if !etag.is_empty() && !etag.to_uppercase().starts_with("W/") { + etag.to_owned() + } else { + header(response, "Last-Modified").trim().to_owned() + } +} + +fn fingerprint(url: &str) -> String { + UrlParts::parse(url).map_or(String::new(), |url| { + format!( + "{:x}", + Sha256::digest(format!( + "{}://{}{}", + url.scheme.to_lowercase(), + url.authority().to_lowercase(), + url.escaped_path() + )) + ) + }) +} + +fn retryable(status: u16) -> bool { + matches!(status, 408 | 425 | 429 | 500..=u16::MAX) +} + +fn status_failure( + response: &HttpResponse, + policy: &DownloadTransferPolicy, + attempt: i64, +) -> Failure { + let status = response.status; + let kind = if status == 429 { + "rate_limited" + } else if policy.refresh_stream_on_status.contains(&i64::from(status)) { + "expired_stream" + } else if retryable(status) { + "transient_network" + } else { + "http_error" + }; + let mut failure = Failure::new(kind, format!("HTTP error: {status}"), attempt); + failure.http_status = status; + let value = header(response, "Retry-After"); + let nanos = if let Ok(seconds) = value.parse::<isize>() { + i128::from((seconds as i64).wrapping_mul(1_000_000_000)) + } else { + httpdate::parse_http_date(value) + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |time| { + time.as_nanos() as i128 + - SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as i128 + }) + }; + if nanos > 0 { + failure.retry_after_seconds = + ((nanos.min(120_000_000_000) + 500_000_000) / 1_000_000_000).max(1) as u16; + } + failure +} + +fn next_delay(current: Duration, policy: &DownloadTransferPolicy) -> Duration { + let ceiling = current + .saturating_mul(2) + .min(Duration::from_millis(policy.max_retry_delay_ms as u64)); + let floor = Duration::from_millis(policy.initial_retry_delay_ms as u64); + if ceiling <= floor { + return ceiling; + } + let mut random = [0; 8]; + if OsRng.try_fill_bytes(&mut random).is_err() { + return ceiling; + } + floor + + Duration::from_nanos( + (u64::from_ne_bytes(random) as f64 / u64::MAX as f64 + * (ceiling - floor).as_nanos() as f64) as u64, + ) +} diff --git a/rust_backend/crates/extensions/src/download/progress.rs b/rust_backend/crates/extensions/src/download/progress.rs new file mode 100644 index 00000000..abfce9a2 --- /dev/null +++ b/rust_backend/crates/extensions/src/download/progress.rs @@ -0,0 +1,55 @@ +use spotiflac_core::progress::{ItemTransferProgressReporter, ProgressRegistry}; +use std::sync::Arc; + +pub(crate) struct ItemProgressTarget { + registry: Arc<ProgressRegistry>, + id: String, + track: bool, +} + +impl ItemProgressTarget { + pub fn new(registry: Arc<ProgressRegistry>, id: String, track: bool) -> Self { + Self { + registry, + id, + track, + } + } + + pub fn start(&self) { + if !self.id.is_empty() { + let _ = self.registry.downloading(&self.id); + } + } + + pub fn set(&self, progress: f64, bytes: u64, total: i64) { + if self.track && !self.id.is_empty() { + let _ = self + .registry + .set_progress(&self.id, progress, bytes as i64, total); + } + } + + pub fn initial(&self, bytes: u64, total: i64) { + if total > 0 { + self.set(bytes as f64 / total as f64, bytes, total); + } else { + self.received(bytes); + } + } + + pub fn received(&self, bytes: u64) { + if self.track && !self.id.is_empty() && bytes > 0 { + let _ = self.registry.set_received(&self.id, bytes as i64); + } + } + + pub fn reporter(&self, bytes: u64, total: i64) -> ItemTransferProgressReporter { + ItemTransferProgressReporter::new( + Arc::clone(&self.registry), + if self.track { &self.id } else { "" }, + bytes as i64, + total, + ) + } +} diff --git a/rust_backend/crates/extensions/src/download/segment_store.rs b/rust_backend/crates/extensions/src/download/segment_store.rs new file mode 100644 index 00000000..c6a6c7a3 --- /dev/null +++ b/rust_backend/crates/extensions/src/download/segment_store.rs @@ -0,0 +1,274 @@ +use super::store::{private_options, remove_if_present}; +use crate::files::FilePath; +use cap_std::fs::{Dir, File}; +use serde::{Deserialize, Serialize}; +use std::ffi::OsString; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::sync::Arc; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +#[derive(Default, Deserialize, Serialize)] +#[serde(default)] +struct Checkpoint { + version: i64, + fingerprint: String, + next_index: usize, + bytes: u64, + updated_at: u64, +} + +pub(super) struct SegmentOutput { + pub parent: Arc<Dir>, + target: OsString, + pub staged: OsString, + checkpoint: OsString, + file: File, + state: Checkpoint, + keep: bool, + promoted: bool, + saved_bytes: u64, + saved_at: Instant, + segments: usize, +} + +impl SegmentOutput { + pub fn open( + path: &FilePath, + fingerprint: String, + segments: usize, + keep: bool, + ) -> io::Result<Self> { + let (parent, target) = path.open_parent()?; + let mut staged = target.clone(); + staged.push(".partial"); + let mut checkpoint = staged.clone(); + checkpoint.push(".checkpoint.json.segments"); + for name in [&staged, &checkpoint] { + match parent.symlink_metadata(name) { + Ok(info) if !info.is_file() => { + return Err(io::Error::other( + "download staging path is not a regular file", + )); + } + Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error), + _ => {} + } + } + let state = keep + .then(|| { + let file = parent + .open_with(&checkpoint, private_options().read(true)) + .ok()?; + let mut bytes = Vec::new(); + file.take((1 << 20) + 1).read_to_end(&mut bytes).ok()?; + if bytes.len() > 1 << 20 { + return None; + } + let state: Checkpoint = serde_json::from_slice(&bytes).ok()?; + (state.version == 1 + && state.fingerprint == fingerprint + && state.next_index <= segments + && (state.next_index == 0) == (state.bytes == 0)) + .then_some(state) + }) + .flatten(); + if state.is_none() { + remove_if_present(&parent, &staged)?; + remove_if_present(&parent, &checkpoint)?; + } + let mut file = parent.open_with( + &staged, + private_options().read(true).write(true).create(true), + )?; + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(io::Error::other( + "download staging path is not a regular file", + )); + } + // A short file loses segment boundaries; the whole assembled prefix + // must be discarded instead of clamping its byte count. + let state = state + .filter(|state| metadata.len() >= state.bytes) + .unwrap_or_else(|| Checkpoint { + fingerprint, + ..Checkpoint::default() + }); + if state.next_index == 0 { + remove_if_present(&parent, &checkpoint)?; + } + file.set_len(state.bytes)?; + file.seek(SeekFrom::Start(state.bytes))?; + Ok(Self { + parent: Arc::new(parent), + target, + staged, + checkpoint, + file, + saved_bytes: state.bytes, + state, + keep, + promoted: false, + saved_at: Instant::now(), + segments, + }) + } + + pub fn next_index(&self) -> usize { + self.state.next_index + } + + pub fn bytes(&self) -> u64 { + self.state.bytes + } + + pub fn append( + &mut self, + segment: &mut SegmentFile, + size: u64, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<(), String> { + segment + .file + .seek(SeekFrom::Start(0)) + .map_err(|error| error.to_string())?; + let mut buffer = [0; 128 << 10]; + let mut copied = 0; + let result = (|| { + loop { + check()?; + let count = segment + .file + .read(&mut buffer) + .map_err(|error| error.to_string())?; + if count == 0 { + break; + } + self.file + .write_all(&buffer[..count]) + .map_err(|error| error.to_string())?; + copied += count as u64; + } + if copied != size { + return Err("short segment copy".into()); + } + Ok(()) + })(); + if result.is_ok() { + self.state.bytes += copied; + self.state.next_index += 1; + } else { + // Do not checkpoint a half-appended segment on cancellation or I/O failure. + let _ = self.file.set_len(self.state.bytes); + let _ = self.file.seek(SeekFrom::Start(self.state.bytes)); + } + result + } + + pub fn checkpoint(&mut self, force: bool) { + if !self.keep + || self.state.next_index == 0 + || self.state.bytes == 0 + || (!force + && self.state.bytes.saturating_sub(self.saved_bytes) < 8 << 20 + && self.saved_at.elapsed().as_secs() < 5) + { + return; + } + if self.save().is_ok() { + self.saved_bytes = self.state.bytes; + self.saved_at = Instant::now(); + } + } + + fn save(&mut self) -> io::Result<()> { + self.file.sync_all()?; + self.state.version = 1; + self.state.updated_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + let bytes = serde_json::to_vec(&self.state)?; + let mut temporary = self.checkpoint.clone(); + temporary.push(".tmp"); + remove_if_present(&self.parent, &temporary)?; + let result = (|| { + let mut file = self + .parent + .open_with(&temporary, private_options().write(true).create_new(true))?; + file.write_all(&bytes)?; + file.sync_all()?; + drop(file); + self.parent + .rename(&temporary, &self.parent, &self.checkpoint) + })(); + if result.is_err() { + let _ = self.parent.remove_file(&temporary); + } + result + } + + pub fn publish(&mut self, check: &dyn Fn() -> Result<(), String>) -> Result<(), String> { + self.file.sync_all().map_err(|error| error.to_string())?; + check()?; + self.parent + .rename(&self.staged, &self.parent, &self.target) + .map_err(|error| error.to_string())?; + self.promoted = true; + let _ = self.parent.remove_file(&self.checkpoint); + let _ = self + .parent + .try_clone() + .and_then(|parent| parent.into_std_file().sync_all()); + Ok(()) + } +} + +impl Drop for SegmentOutput { + fn drop(&mut self) { + // Also clean leftovers from a process that stopped before these workers started. + for index in 0..self.segments { + let _ = self.parent.remove_file(segment_name(&self.staged, index)); + } + if self.promoted || !self.keep { + let _ = self.parent.remove_file(&self.checkpoint); + if !self.promoted { + let _ = self.parent.remove_file(&self.staged); + } + } + } +} + +pub(super) struct SegmentFile { + parent: Arc<Dir>, + name: OsString, + pub file: File, +} + +impl SegmentFile { + pub fn open(parent: &Arc<Dir>, staged: &OsString, index: usize) -> io::Result<Self> { + let name = segment_name(staged, index); + remove_if_present(parent, &name)?; + let file = parent.open_with( + &name, + private_options().read(true).write(true).create_new(true), + )?; + Ok(Self { + parent: Arc::clone(parent), + name, + file, + }) + } +} + +impl Drop for SegmentFile { + fn drop(&mut self) { + let _ = self.parent.remove_file(&self.name); + } +} + +fn segment_name(staged: &OsString, index: usize) -> OsString { + let mut name = staged.clone(); + name.push(format!(".segment.{index:06}")); + name +} diff --git a/rust_backend/crates/extensions/src/download/segments.rs b/rust_backend/crates/extensions/src/download/segments.rs new file mode 100644 index 00000000..ac0ea592 --- /dev/null +++ b/rust_backend/crates/extensions/src/download/segments.rs @@ -0,0 +1,381 @@ +use super::segment_store::{SegmentFile, SegmentOutput}; +use super::{DownloadOptions, Failure, header, next_delay, retryable, status_failure}; +use crate::files::{ExtensionFiles, FilePath}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use spotiflac_network::{HttpRequest, NetworkSession}; +use std::collections::BTreeMap; +use std::io::Write; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Mutex, mpsc}; +use std::thread; +use std::time::{Duration, Instant}; + +#[derive(Deserialize)] +pub(crate) struct Segment { + pub url: String, + pub headers: BTreeMap<String, String>, +} + +struct Completed { + file: SegmentFile, + size: u64, + attempts: i64, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn download( + files: &ExtensionFiles, + path: &FilePath, + network: &NetworkSession, + segments: &[Segment], + options: DownloadOptions, + check: &(dyn Fn() -> Result<(), String> + Sync), + progress: &mut dyn FnMut(u64, usize, usize), +) -> Result<serde_json::Value, Failure> { + let _guard = files + .lock(path, check) + .map_err(|error| Failure::new("cancelled", error, 0))?; + let mut hash = Sha256::new(); + for segment in segments { + hash.update(segment.url.as_bytes()); + hash.update([0]); + } + let mut output = SegmentOutput::open( + path, + format!("{:x}", hash.finalize()), + segments.len(), + options.persistent, + ) + .map_err(|error| { + Failure::new( + "storage_error", + format!("failed to create segmented output: {error}"), + 0, + ) + })?; + let restored = output.next_index() == segments.len() && output.bytes() > 0; + options.item.start(); + options.item.set( + output.next_index() as f64 / segments.len() as f64, + output.bytes(), + 0, + ); + let reporter = options.item.reporter(output.bytes(), 0); + if !restored { + let mut next_job = output.next_index(); + let stop = AtomicBool::new(false); + let completed = AtomicBool::new(false); + let received = AtomicU64::new(output.bytes()); + let worker_count = (options.policy.max_parallel_segments as usize) + .min(segments.len() - output.next_index()); + let mut first_failure = None; + let mut pending = BTreeMap::new(); + let parent = std::sync::Arc::clone(&output.parent); + let staged = output.staged.clone(); + let (jobs, job_receiver) = mpsc::sync_channel(worker_count); + let job_receiver = Mutex::new(job_receiver); + // Scoped workers cannot outlive the JS call, its cancellation lease, or + // its directory capability. Only this VM thread invokes JS callbacks. + thread::scope(|scope| { + let (sender, receiver) = mpsc::sync_channel(worker_count); + for _ in 0..worker_count { + jobs.send(next_job).expect("segment workers not started"); + next_job += 1; + } + let mut jobs = (next_job < segments.len()).then_some(jobs); + for _ in 0..worker_count { + let sender = sender.clone(); + let parent = &parent; + let staged = &staged; + let job_receiver = &job_receiver; + let stop = &stop; + let received = &received; + let completed = &completed; + let options = &options; + let reporter = &reporter; + let spawned = thread::Builder::new() + .name("extension-segment".into()) + .stack_size(2 << 20) + .spawn_scoped(scope, move || { + let check_worker = || { + check()?; + if stop.load(Ordering::Acquire) { + return Err("download cancelled".into()); + } + Ok(()) + }; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + loop { + if check_worker().is_err() { + break; + } + let Ok(index) = job_receiver + .lock() + .unwrap_or_else(|error| error.into_inner()) + .recv() + else { + break; + }; + let segment = &segments[index]; + let result = fetch( + segment, + index, + parent, + staged, + network, + options, + received, + completed, + reporter, + &check_worker, + ); + let failed = result.is_err(); + if sender.send((index, result)).is_err() || failed { + break; + } + } + })); + if let Err(panic) = result { + let _ = sender.send(( + usize::MAX, + Err(Failure::new("storage_error", "segment worker panicked", 0)), + )); + std::panic::resume_unwind(panic); + } + }); + if let Err(error) = spawned { + first_failure = Some(Failure::new( + "storage_error", + format!("failed to start segment worker: {error}"), + 0, + )); + stop.store(true, Ordering::Release); + jobs.take(); + break; + } + } + drop(sender); + loop { + if let Err(error) = check() { + first_failure.get_or_insert_with(|| Failure::new("cancelled", error, 0)); + stop.store(true, Ordering::Release); + jobs.take(); + } + let (index, result) = match receiver.recv_timeout(Duration::from_millis(10)) { + Ok(result) => result, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + match result { + Err(error) => { + first_failure.get_or_insert(error); + stop.store(true, Ordering::Release); + jobs.take(); + } + Ok(result) => { + pending.insert(index, result); + } + } + while let Some(mut ready) = pending.remove(&output.next_index()) { + let index = output.next_index(); + if let Err(error) = output.append(&mut ready.file, ready.size, check) { + first_failure.get_or_insert_with(|| { + Failure::new( + "storage_error", + format!("failed to append segment {index}: {error}"), + ready.attempts, + ) + }); + stop.store(true, Ordering::Release); + jobs.take(); + break; + } + output.checkpoint(false); + drop(ready); + options.item.set( + output.next_index() as f64 / segments.len() as f64, + received.load(Ordering::Acquire), + 0, + ); + let _charge = options.budget.as_ref().map(|budget| budget.enter(true)); + progress( + received.load(Ordering::Acquire), + output.next_index(), + segments.len(), + ); + // Only assembly releases a slot: completed files and active + // requests together never exceed the worker count. + if let Some(sender) = &jobs { + if sender.send(next_job).is_err() { + jobs.take(); + } else { + next_job += 1; + if next_job == segments.len() { + jobs.take(); + } + } + } + } + } + }); + if let Some(error) = first_failure { + output.checkpoint(true); + return Err(error); + } + } + if output.next_index() != segments.len() || output.bytes() == 0 { + return Err(Failure::new( + "integrity_failed", + format!( + "segmented transfer incomplete: assembled {} of {} segments", + output.next_index(), + segments.len() + ), + 0, + )); + } + output.publish(check).map_err(|error| { + Failure::new( + "storage_error", + format!("failed to publish segmented output: {error}"), + 0, + ) + })?; + options.item.set(1.0, output.bytes(), output.bytes() as i64); + let mut result = serde_json::json!({"success":true,"path":path.display(),"size":output.bytes(),"segments":segments.len()}); + if restored { + result["resumed"] = true.into(); + } + Ok(result) +} + +#[allow(clippy::too_many_arguments)] +fn fetch( + segment: &Segment, + index: usize, + parent: &std::sync::Arc<cap_std::fs::Dir>, + staged: &std::ffi::OsString, + network: &NetworkSession, + options: &DownloadOptions, + received: &AtomicU64, + completed: &AtomicBool, + reporter: &spotiflac_core::progress::ItemTransferProgressReporter, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Completed, Failure> { + let mut delay = Duration::from_millis(options.policy.initial_retry_delay_ms as u64); + for attempt in 1..=options.policy.max_attempts { + check().map_err(|error| Failure::new("cancelled", error, attempt))?; + let mut file = SegmentFile::open(parent, staged, index).map_err(|error| { + Failure::new( + "storage_error", + format!("failed to create segment file: {error}"), + attempt, + ) + })?; + let continuation = attempt == 1 && completed.load(Ordering::Acquire); + let charge = options + .budget + .as_ref() + .map(|budget| budget.enter(!continuation)); + let stream = network.open_stream( + HttpRequest { + url: segment.url.clone(), + method: "GET".into(), + body: String::new(), + headers: segment.headers.clone(), + default_json: false, + user_agent: options.app_version.user_agent(), + }, + Duration::from_secs(24 * 60 * 60), + Duration::from_secs(60), + check, + ); + drop(charge); + let mut failure = match stream { + Err(error) => Failure::new("transient_network", error, attempt), + Ok(mut stream) => { + let status = stream.response.status; + if !(200..300).contains(&status) { + let mut failure = status_failure(&stream.response, &options.policy, attempt); + failure.error = format!("segment {index} HTTP error: {status}"); + if !retryable(status) || attempt == options.policy.max_attempts { + return Err(failure); + } + failure + } else { + let expected = header(&stream.response, "Content-Length") + .parse::<i64>() + .unwrap_or(-1); + let mut size = 0; + let mut buffer = [0; 64 << 10]; + let result = loop { + let charge = options.budget.as_ref().map(|budget| { + budget + .enter(!matches!(status, 200 | 206) || (size == 0 && !continuation)) + }); + let read = stream.read(&mut buffer, check); + drop(charge); + match read { + Ok(0) => break Ok(()), + Ok(count) => { + if let Err(error) = file.file.write_all(&buffer[..count]) { + break Err(error.to_string()); + } + size += count as u64; + received.fetch_add(count as u64, Ordering::AcqRel); + reporter.report(received.load(Ordering::Acquire) as i64, 0); + } + Err(error) => break Err(error), + } + }; + let result = result.and_then(|()| { + if expected > 0 && size != expected as u64 { + Err("unexpected EOF".into()) + } else { + Ok(()) + } + }); + if result.is_ok() && size > 0 { + completed.store(true, Ordering::Release); + return Ok(Completed { + file, + size, + attempts: attempt, + }); + } + received.fetch_sub(size, Ordering::AcqRel); + let message = result.err().map_or_else( + || format!("segment {index} response was empty"), + |error| format!("failed to read segment {index}: {error}"), + ); + Failure::new("transient_network", message, attempt) + } + } + }; + drop(file); + if let Err(error) = check() { + failure = Failure::new("cancelled", error, attempt); + } + if attempt == options.policy.max_attempts || failure.error_type == "cancelled" { + return Err(failure); + } + let retry = if failure.retry_after_seconds > 0 { + Duration::from_secs(failure.retry_after_seconds.into()) + } else { + delay + }; + let _charge = options.budget.as_ref().map(|budget| budget.enter(true)); + let began = Instant::now(); + while began.elapsed() < retry { + check().map_err(|error| Failure::new("cancelled", error, attempt))?; + thread::sleep( + retry + .saturating_sub(began.elapsed()) + .min(Duration::from_millis(10)), + ); + } + delay = next_delay(delay, &options.policy); + } + unreachable!("policy has at least one attempt") +} diff --git a/rust_backend/crates/extensions/src/download/store.rs b/rust_backend/crates/extensions/src/download/store.rs new file mode 100644 index 00000000..a4e3a8a5 --- /dev/null +++ b/rust_backend/crates/extensions/src/download/store.rs @@ -0,0 +1,230 @@ +use crate::files::FilePath; +use cap_std::fs::{Dir, File, OpenOptions}; +use serde::{Deserialize, Serialize}; +use std::ffi::OsString; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +#[derive(Default, Deserialize, Serialize)] +#[serde(default)] +pub(super) struct Checkpoint { + version: i64, + fingerprint: String, + pub validator: String, + pub bytes: u64, + pub total: i64, + updated_at: u64, +} + +pub(super) struct DownloadFile { + parent: Dir, + target: OsString, + staged: OsString, + checkpoint_name: OsString, + pub file: File, + pub state: Checkpoint, + pub restored: bool, + keep_partial: bool, + promoted: bool, + saved_bytes: u64, + saved_at: Instant, +} + +impl DownloadFile { + pub fn open(path: &FilePath, fingerprint: &str, keep_partial: bool) -> Result<Self, String> { + let (parent, target) = path.open_parent().map_err(|error| error.to_string())?; + let mut staged = target.clone(); + staged.push(".partial"); + let mut checkpoint_name = staged.clone(); + checkpoint_name.push(".checkpoint.json"); + for name in [&staged, &checkpoint_name] { + match parent.symlink_metadata(name) { + Ok(info) if !info.is_file() => { + return Err("download staging path is not a regular file".into()); + } + Err(error) if error.kind() != io::ErrorKind::NotFound => { + return Err(error.to_string()); + } + _ => {} + } + } + let state = keep_partial + .then(|| { + let mut file = parent + .open_with(&checkpoint_name, private_options().read(true)) + .ok()?; + let mut data = Vec::new(); + (&mut file) + .take((1 << 20) + 1) + .read_to_end(&mut data) + .ok()?; + if data.len() > 1 << 20 { + return None; + } + let state: Checkpoint = serde_json::from_slice(&data).ok()?; + (state.version == 1 + && !state.fingerprint.is_empty() + && state.fingerprint == fingerprint + && !state.validator.is_empty() + && state.bytes > 0) + .then_some(state) + }) + .flatten(); + let restored = state.is_some(); + if !restored { + remove_if_present(&parent, &staged).map_err(|error| error.to_string())?; + remove_if_present(&parent, &checkpoint_name).map_err(|error| error.to_string())?; + } + let file = parent + .open_with( + &staged, + private_options().read(true).write(true).create(true), + ) + .map_err(|error| error.to_string())?; + if !file + .metadata() + .map_err(|error| error.to_string())? + .is_file() + { + return Err("download staging path is not a regular file".into()); + } + let mut result = Self { + parent, + target, + staged, + checkpoint_name, + file, + state: state.unwrap_or_default(), + restored, + keep_partial, + promoted: false, + saved_bytes: 0, + saved_at: Instant::now(), + }; + result.state.fingerprint = fingerprint.to_owned(); + result.state.bytes = result.state.bytes.min( + result + .file + .metadata() + .map_err(|error| error.to_string())? + .len(), + ); + result + .file + .set_len(result.state.bytes) + .map_err(|error| error.to_string())?; + result + .file + .seek(SeekFrom::Start(result.state.bytes)) + .map_err(|error| error.to_string())?; + result.saved_bytes = result.state.bytes; + Ok(result) + } + + pub fn restart(&mut self) -> Result<(), String> { + self.rewind(0)?; + self.state.validator.clear(); + remove_if_present(&self.parent, &self.checkpoint_name).map_err(|error| error.to_string()) + } + + pub fn rewind(&mut self, bytes: u64) -> Result<(), String> { + self.file + .set_len(bytes) + .map_err(|error| error.to_string())?; + self.file + .seek(SeekFrom::Start(bytes)) + .map_err(|error| error.to_string())?; + self.state.bytes = bytes; + Ok(()) + } + + pub fn checkpoint(&mut self, force: bool) { + if !self.keep_partial || self.state.validator.is_empty() || self.state.bytes == 0 { + return; + } + if !force + && self.state.bytes.saturating_sub(self.saved_bytes) < 8 << 20 + && self.saved_at.elapsed().as_secs() < 5 + { + return; + } + self.saved_bytes = self.state.bytes; + self.saved_at = Instant::now(); + // Data reaches stable storage before publishing its checkpoint pointer. + // A failed save is retried on the next interval, not on every read. + let _ = self.save(); + } + + fn save(&mut self) -> io::Result<()> { + self.file.sync_all()?; + self.state.version = 1; + self.state.updated_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + let bytes = serde_json::to_vec(&self.state)?; + let mut temporary = self.checkpoint_name.clone(); + temporary.push(".tmp"); + // Remove the directory entry itself; an old symlink is never followed. + remove_if_present(&self.parent, &temporary)?; + let result = (|| { + let mut file = self + .parent + .open_with(&temporary, private_options().write(true).create_new(true))?; + file.write_all(&bytes)?; + file.sync_all()?; + drop(file); + self.parent + .rename(&temporary, &self.parent, &self.checkpoint_name) + })(); + if result.is_err() { + let _ = self.parent.remove_file(&temporary); + } + result + } + + pub fn publish(&mut self, check: &dyn Fn() -> Result<(), String>) -> Result<(), String> { + self.file.sync_all().map_err(|error| error.to_string())?; + check()?; + self.parent + .rename(&self.staged, &self.parent, &self.target) + .map_err(|error| error.to_string())?; + self.promoted = true; + let _ = self.parent.remove_file(&self.checkpoint_name); + let _ = self + .parent + .try_clone() + .and_then(|parent| parent.into_std_file().sync_all()); + Ok(()) + } +} + +impl Drop for DownloadFile { + fn drop(&mut self) { + if self.promoted || !self.keep_partial { + let _ = self.parent.remove_file(&self.checkpoint_name); + if !self.promoted { + let _ = self.parent.remove_file(&self.staged); + } + } + } +} + +pub(super) fn private_options() -> OpenOptions { + let mut options = OpenOptions::new(); + #[cfg(unix)] + { + use cap_std::fs::OpenOptionsExt; + options.mode(0o600).custom_flags( + (rustix::fs::OFlags::NONBLOCK | rustix::fs::OFlags::NOFOLLOW).bits() as i32, + ); + } + options +} + +pub(super) fn remove_if_present(parent: &Dir, name: &OsString) -> io::Result<()> { + match parent.remove_file(name) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + result => result, + } +} diff --git a/rust_backend/crates/extensions/src/download_host.rs b/rust_backend/crates/extensions/src/download_host.rs new file mode 100644 index 00000000..0a9b0e92 --- /dev/null +++ b/rust_backend/crates/extensions/src/download_host.rs @@ -0,0 +1,257 @@ +use crate::download::{self, DownloadOptions}; +use crate::runtime::{Control, ExtensionServices}; +use rquickjs::{Ctx, Function, Object, Value}; +use std::sync::Arc; + +pub(crate) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + register_segments(ctx, host, Arc::clone(&control), services)?; + let Some(files) = &services.files else { + return Ok(()); + }; + let files = Arc::clone(files); + let downloads = Arc::clone(&services.downloads); + let network = services.network.clone(); + let policy = services.transfer_policy.clone(); + let app_version = services.app_version.clone(); + host.set( + "downloadCall", + Function::new( + ctx.clone(), + move |ctx: Ctx<'js>, + url: String, + path: String, + options: Value<'js>, + headers: String| { + let result: Result<serde_json::Value, String> = (|| { + let check = || control.check().map_err(|error| error.to_string()); + check()?; + let network = network.as_ref().ok_or("network access unavailable")?; + network.validate_url(&url)?; + let path = files.resolve(&path)?; + let options = options.as_object(); + let value = |key: &str| -> Result<Value<'js>, String> { + options.map_or_else( + || Ok(Value::new_undefined(ctx.clone())), + |object| object.get(key).map_err(|error| error.to_string()), + ) + }; + let chunked = value("chunked")?; + let chunk_size = if chunked.as_bool() == Some(true) { + Some(1 << 20) + } else { + chunked + .as_number() + .filter(|value| *value > 0.0) + .map(|value| { + let size = value as i64; + if size <= 0 { 1 << 20 } else { size as u64 } + }) + }; + let mut policy = policy.clone(); + if let Some(attempts) = value("maxAttempts")? + .as_number() + .filter(|number| *number > 0.0) + { + policy.max_attempts = (attempts as i64).clamp(1, 8); + } + let resume = value("resume")? + .as_bool() + .unwrap_or(policy.resume_policy == "validated"); + let has_track = options + .map(|object| object.contains_key("trackItemBytes")) + .transpose() + .map_err(|error| error.to_string())? + .unwrap_or(false); + let track = value(if has_track { + "trackItemBytes" + } else { + "track_item_bytes" + })? + .as_bool() + .unwrap_or(true); + let persistent = value("persistentCheckpoint")? + .as_bool() + .map_or(policy.persistent_checkpoint, |enabled| enabled && resume); + let callback = value("onProgress")?; + let callback = callback.as_function(); + let mut progress = |bytes, total| { + if let Some(callback) = callback + && callback.call::<_, ()>((bytes, total)).is_err() + { + // Go ignores download progress callback errors. Clear the + // pending JS exception; cancellation is checked by the host. + ctx.catch(); + } + }; + let headers = + serde_json::from_str(&headers).map_err(|error| error.to_string())?; + Ok( + match download::download( + &files, + &path, + network, + &url, + DownloadOptions { + item: download::progress::ItemProgressTarget::new( + Arc::clone(&downloads.progress), + control.item_id(), + track, + ), + headers, + // Go snapshots the UA for a chunked transfer; + // ordinary retries read the current app version. + app_version: if chunk_size.is_some() { + app_version.get().into() + } else { + app_version.clone() + }, + resume, + persistent, + policy, + budget: control.resolution(), + chunk_size, + }, + &check, + &|duration| { + let _charge = control.resolution().map(|budget| budget.enter(true)); + control.sleep(duration); + check() + }, + &mut progress, + ) { + Ok(result) => result, + Err(error) => serde_json::to_value(error).expect("transfer error JSON"), + }, + ) + })(); + match result { + Ok(result) => result, + Err(error) => serde_json::json!({"success":false,"error":error}), + } + .to_string() + }, + )?, + )?; + Ok(()) +} + +fn register_segments<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + let Some(files) = &services.files else { + return Ok(()); + }; + let files = Arc::clone(files); + let downloads = Arc::clone(&services.downloads); + let network = services.network.clone(); + let policy = services.transfer_policy.clone(); + let app_version = services.app_version.clone(); + host.set( + "downloadSegmentsCall", + Function::new( + ctx.clone(), + move |ctx: Ctx<'js>, segments: String, path: String, options: Value<'js>| { + let result = (|| { + let check = || control.check().map_err(|error| error.to_string()); + check().map_err(|error| download::Failure::new("cancelled", error, 0))?; + let network = network.as_ref().ok_or_else(|| { + download::Failure::new("permission", "network access unavailable", 0) + })?; + let mut segments: Vec<download::segments::Segment> = + serde_json::from_str(&segments) + .map_err(|error| download::Failure::new("invalid_request", error, 0))?; + for (index, segment) in segments.iter_mut().enumerate() { + segment.url = segment.url.trim().to_owned(); + if segment.url.is_empty() { + return Err(download::Failure::new( + "invalid_request", + format!("segment {index} URL is empty"), + 0, + )); + } + } + for segment in &segments { + network + .validate_url(&segment.url) + .map_err(|error| download::Failure::new("permission", error, 0))?; + } + let path = files + .resolve(&path) + .map_err(|error| download::Failure::new("permission", error, 0))?; + let options = options.as_object(); + let value = |key: &str| -> Result<Value<'js>, download::Failure> { + options.map_or_else( + || Ok(Value::new_undefined(ctx.clone())), + |object| { + object.get(key).map_err(|error| { + download::Failure::new("invalid_request", error, 0) + }) + }, + ) + }; + let mut policy = policy.clone(); + for (name, target) in [ + ("maxAttempts", &mut policy.max_attempts), + ("maxParallel", &mut policy.max_parallel_segments), + ] { + if let Some(number) = value(name)?.as_number() { + let rounded = crate::transfer_policy::rounded_int(number); + if rounded > 0 { + *target = rounded.clamp(1, 8); + } + } + } + let persistent = value("persistentCheckpoint")? + .as_bool() + .unwrap_or(policy.persistent_checkpoint); + let callback = value("onProgress")?; + let callback = callback.as_function(); + let mut progress = |bytes, completed, count| { + if let Some(callback) = callback + && callback + .call::<_, ()>((bytes, 0, completed, count)) + .is_err() + { + ctx.catch(); + } + }; + download::segments::download( + &files, + &path, + network, + &segments, + DownloadOptions { + item: download::progress::ItemProgressTarget::new( + Arc::clone(&downloads.progress), + control.item_id(), + true, + ), + headers: Default::default(), + app_version: app_version.clone(), + resume: false, + persistent, + policy, + budget: control.resolution(), + chunk_size: None, + }, + &check, + &mut progress, + ) + })(); + match result { + Ok(result) => result, + Err(error) => serde_json::to_value(error).expect("transfer error JSON"), + } + .to_string() + }, + )?, + ) +} diff --git a/rust_backend/crates/extensions/src/environment.rs b/rust_backend/crates/extensions/src/environment.rs new file mode 100644 index 00000000..6f0e483f --- /dev/null +++ b/rust_backend/crates/extensions/src/environment.rs @@ -0,0 +1,770 @@ +//! Platform-owned storage and lifecycle context for managed extension runtimes. + +mod index; + +use crate::auth::{AuthRegistry, ExtensionAuth, PendingAuthRequest}; +use crate::manifest::{ExtensionManifest, ManifestError}; +use crate::runtime::LoadMode; +use crate::signed_session::SignedRegistry; +use crate::storage::{ + ExtensionStore, StorageError, StorageMasterKey, StoreKind, valid_extension_id, +}; +use crate::{ExtensionError, ExtensionRuntime, ExtensionServices, RuntimeLimits}; +use serde_json::{Map, Value}; +use spotiflac_core::app_version::AppVersion; +use spotiflac_core::cancellation::{CancellationDomain, CancellationRegistry}; +use spotiflac_core::downloads::DownloadState; +use spotiflac_core::progress::ProgressRegistry; +use spotiflac_network::{NetworkService, policy::NetworkPermissions}; +use spotiflac_providers::lyrics::{CallGraph, CallNode, LyricsService}; +use std::cmp::Ordering as Comparison; +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, LazyLock, Mutex, OnceLock, Weak}; +use std::time::Duration; + +// Keep these contract versions aligned with Go's supportedRuntimeFeatures. +pub(crate) static SUPPORTED_RUNTIME_FEATURES: LazyLock<BTreeMap<String, isize>> = + LazyLock::new(|| { + [ + ("signedSession", 3), + ("sessionRefresh", 1), + ("sessionGrant", 1), + ("globalAction", 1), + ("webviewAuth", 1), + ("downloadSegments", 1), + ("patternedFileTransform", 1), + ("preparedContext", 1), + ] + .into_iter() + .map(|(name, version)| (name.to_owned(), version)) + .collect() + }); + +#[derive(Debug, thiserror::Error)] +pub enum EnvironmentError { + #[error("extension environment closed")] + Closed, + #[error("{0}")] + Storage(#[from] StorageError), + #[error("{0}")] + Manifest(#[from] ManifestError), + #[error("{0}")] + Runtime(#[from] ExtensionError), + #[error("{0}")] + Gate(String), + #[error("network initialization failed: {0}")] + Network(#[from] std::io::Error), + #[error("{0}")] + Auth(String), + #[error("{0}")] + Index(String), +} + +#[derive(Default)] +struct State { + pending: usize, + runtimes: Vec<Weak<ExtensionRuntime>>, + stores: BTreeMap<String, Arc<ExtensionStore>>, + lyrics_nodes: BTreeMap<String, CallNode>, +} + +/// The master key comes from the native Keychain/Keystore adapter. The directory +/// must belong to this backend; Go and Rust must not concurrently own live data. +pub struct ExtensionEnvironment { + logs: Arc<crate::logging::LogBuffer>, + ffmpeg: Arc<crate::ffmpeg::CommandRegistry>, + downloads: Arc<spotiflac_core::downloads::DownloadState>, + data_directory: PathBuf, + data_directory_alias: PathBuf, + master_key: Arc<StorageMasterKey>, + app_version: AppVersion, + network: Arc<NetworkService>, + auth: Arc<AuthRegistry>, + sessions: Arc<SignedRegistry>, + files: Arc<crate::files::FileRegistry>, + isrc: Arc<spotiflac_core::isrc::IndexCache>, + closed: Arc<AtomicBool>, + state: Mutex<State>, + idle: Condvar, + shutdown_lock: Mutex<()>, + manager_owned: AtomicBool, + calls: CallGraph, + lyrics: OnceLock<Weak<LyricsService>>, +} + +impl ExtensionEnvironment { + pub fn new( + data_directory: &Path, + master_key: &str, + app_version: &str, + ) -> Result<Self, EnvironmentError> { + // Validate the key before initializing any platform services or files. + StorageMasterKey::from_base64(master_key)?; + Self::with_network( + data_directory, + master_key, + app_version, + NetworkService::new()?, + ) + } + + pub fn with_network( + data_directory: &Path, + master_key: &str, + app_version: &str, + network: Arc<NetworkService>, + ) -> Result<Self, EnvironmentError> { + let master_key = Arc::new(StorageMasterKey::from_base64(master_key)?); + if data_directory.as_os_str().is_empty() { + return Err(StorageError::InvalidPath.into()); + } + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(data_directory).map_err(StorageError::from)?; + let data_directory_alias = + crate::files::clean(&std::path::absolute(data_directory).map_err(StorageError::from)?); + let data_directory = fs::canonicalize(data_directory).map_err(StorageError::from)?; + let auth = Arc::new(AuthRegistry::default()); + let sessions = SignedRegistry::new(&data_directory, Arc::clone(&auth)); + Ok(Self { + logs: Arc::new(crate::logging::LogBuffer::default()), + ffmpeg: Arc::new(crate::ffmpeg::CommandRegistry::default()), + downloads: Arc::new(DownloadState { + progress: Arc::new(ProgressRegistry::new()), + cancellation: Arc::new(CancellationRegistry::with_waker( + CancellationDomain::Download, + network.cancellation_waker(), + )), + }), + data_directory, + data_directory_alias, + master_key, + app_version: app_version.into(), + network, + auth, + sessions, + files: Arc::new(crate::files::FileRegistry::default()), + isrc: Arc::new(spotiflac_core::isrc::IndexCache::default()), + closed: Arc::new(AtomicBool::new(false)), + state: Mutex::default(), + idle: Condvar::new(), + shutdown_lock: Mutex::new(()), + manager_owned: AtomicBool::new(false), + calls: CallGraph::default(), + lyrics: OnceLock::new(), + }) + } + + fn enter(&self) -> Result<Operation<'_>, EnvironmentError> { + let mut state = self.state.lock().expect("extension environment lock"); + if self.closed.load(Ordering::Acquire) { + return Err(EnvironmentError::Closed); + } + state.pending += 1; + Ok(Operation(self)) + } + + fn store(&self, id: &str) -> Result<Arc<ExtensionStore>, EnvironmentError> { + if !valid_extension_id(id) { + return Err(StorageError::InvalidExtensionId.into()); + } + let path = self.data_directory.join(id); + let mut state = self.state.lock().expect("extension environment lock"); + match fs::symlink_metadata(&path) { + Ok(metadata) if !metadata.is_dir() => return Err(StorageError::InvalidPath.into()), + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(StorageError::from(error).into()); + } + _ => {} + } + if let Some(store) = state.stores.get(id) { + return Ok(Arc::clone(store)); + } + let store = Arc::new(ExtensionStore::open( + &path, + id, + Some(Arc::clone(&self.master_key)), + )?); + state.stores.insert(id.to_owned(), Arc::clone(&store)); + Ok(store) + } + + pub fn settings(&self, id: &str) -> Result<Map<String, Value>, EnvironmentError> { + let _operation = self.enter()?; + Ok(self.store(id)?.all(StoreKind::Settings)?) + } + + pub fn set_setting(&self, id: &str, key: &str, value: Value) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + Ok(self.store(id)?.set(StoreKind::Settings, key, value)?) + } + + pub fn remove_setting(&self, id: &str, key: &str) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + Ok(self.store(id)?.remove(StoreKind::Settings, key)?) + } + + pub fn set_allow_private_network(&self, allow: bool) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.network.set_allow_private_network(allow); + Ok(()) + } + + pub fn set_network_compatibility_options( + &self, + allow_http: bool, + insecure_tls: bool, + ) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.network + .set_network_compatibility_options(allow_http, insecure_tls); + Ok(()) + } + + /// Recycle connections without cancelling active requests or changing policy. + pub fn cleanup_connections(&self) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.network.reset_connections(); + Ok(()) + } + + /// Trusted native composition reuses the same transport and network policy. + pub fn network_service(&self) -> Result<Arc<NetworkService>, EnvironmentError> { + let _operation = self.enter()?; + Ok(Arc::clone(&self.network)) + } + + pub fn get_app_version(&self) -> Result<String, EnvironmentError> { + let _operation = self.enter()?; + Ok(self.app_version.get()) + } + + pub fn set_app_version(&self, version: &str) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.app_version + .set(version) + .map_err(|error| EnvironmentError::Gate(error.to_string())) + } + + pub(crate) fn shared_app_version(&self) -> AppVersion { + self.app_version.clone() + } + + pub fn set_allowed_download_directories( + &self, + directories: &[PathBuf], + ) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.files + .set_allowed_directories(directories) + .map_err(StorageError::from)?; + Ok(()) + } + + pub fn replace_settings( + &self, + id: &str, + values: Map<String, Value>, + ) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + Ok(self.store(id)?.replace(StoreKind::Settings, values)?) + } + + pub fn load( + &self, + manifest_json: &str, + source: &str, + limits: RuntimeLimits, + ) -> Result<Arc<ExtensionRuntime>, EnvironmentError> { + if self.closed.load(Ordering::Acquire) { + return Err(EnvironmentError::Closed); + } + if self.manager_owned.load(Ordering::Acquire) { + return Err(EnvironmentError::Gate( + "installed runtimes must be loaded through their extension manager".into(), + )); + } + self.load_internal( + manifest_json, + source, + limits, + LoadMode::Initialize, + None, + None, + ) + } + + pub(crate) fn load_registered_with_lease( + &self, + manifest_json: &str, + source: &str, + limits: RuntimeLimits, + mode: LoadMode, + lease: Option<Arc<spotiflac_core::cancellation::RequestLease>>, + ) -> Result<Arc<ExtensionRuntime>, EnvironmentError> { + self.load_internal(manifest_json, source, limits, mode, None, lease) + } + + pub(crate) fn load_isolated( + &self, + manifest_json: &str, + source: &str, + limits: RuntimeLimits, + primary: &ExtensionRuntime, + ) -> Result<Arc<ExtensionRuntime>, EnvironmentError> { + self.load_internal( + manifest_json, + source, + limits, + LoadMode::Register, + Some(primary), + None, + ) + } + + fn load_internal( + &self, + manifest_json: &str, + source: &str, + limits: RuntimeLimits, + load_mode: LoadMode, + primary: Option<&ExtensionRuntime>, + startup_lease: Option<Arc<spotiflac_core::cancellation::RequestLease>>, + ) -> Result<Arc<ExtensionRuntime>, EnvironmentError> { + let _operation = self.enter()?; + if manifest_json.len() > 1024 * 1024 { + return Err(EnvironmentError::Gate( + "invalid extension package: manifest.json is too large".to_owned(), + )); + } + let manifest = ExtensionManifest::parse(manifest_json)?; + validate_gates( + &manifest, + &self.app_version.get(), + &SUPPORTED_RUNTIME_FEATURES, + ) + .map_err(EnvironmentError::Gate)?; + let store = self.store(&manifest.name)?; + let mut settings = store.all(StoreKind::Settings)?; + settings.retain(|key, _| !key.starts_with('_')); + let isolated = primary.is_some(); + let compiled_source = primary.map(|runtime| Arc::clone(&runtime.compiled_source)); + let network = primary + .and_then(ExtensionRuntime::network_session) + .unwrap_or_else(|| { + self.network.session( + NetworkPermissions { + domains: manifest.permissions.network.clone().unwrap_or_default(), + allow_http: manifest.permissions.allow_http, + }, + network_timeout(manifest.capabilities.get("networkTimeoutSeconds")), + ) + }); + // Legacy metadata helpers do not require the general file permission. + // They receive read access to this sandbox and native-granted roots. + let legacy_files = self + .files + .extension_with_alias( + &self.data_directory.join(&manifest.name), + &self.data_directory_alias.join(&manifest.name), + ) + .map_err(StorageError::from)?; + let services = ExtensionServices { + legacy_backend: true, + lyrics: self.lyrics.get().cloned().unwrap_or_default(), + lyrics_node: self.lyrics_node(&manifest.name, isolated), + lyrics_parent: isolated.then(|| self.lyrics_node(&manifest.name, false)), + legacy_files: Some(Arc::clone(&legacy_files)), + isrc: Arc::clone(&self.isrc), + logs: Arc::clone(&self.logs), + extension_id: manifest.name.clone(), + ffmpeg: Arc::clone(&self.ffmpeg), + raw_ffmpeg_stub: manifest + .capabilities + .get("rawFfmpeg") + .and_then(Value::as_bool) + .unwrap_or(false), + downloads: Arc::clone(&self.downloads), + transfer_policy: crate::transfer_policy::DownloadTransferPolicy::from_capabilities( + &manifest.capabilities, + ), + storage: manifest.permissions.storage.then_some(store), + files: manifest.permissions.file.then_some(legacy_files), + auth: manifest.permissions.storage.then(|| { + Arc::new(ExtensionAuth::new( + &manifest.name, + Arc::clone(&self.auth), + Arc::clone(&network), + self.app_version.clone(), + )) + }), + auth_registry: Some(Arc::clone(&self.auth)), + session: manifest + .signed_session + .clone() + .map(|config| { + self.sessions + .session(&manifest.name, config, Arc::clone(&network)) + }) + .transpose() + .map_err(EnvironmentError::Auth)?, + network: Some(network), + app_version: self.app_version.clone(), + parent_closed: Some(Arc::clone(&self.closed)), + startup_lease, + initialize_empty_settings: false, + load_mode, + compiled_source, + }; + let runtime = Arc::new(ExtensionRuntime::load_with_services( + source, + &Value::Object(settings).to_string(), + limits, + services, + )?); + let mut state = self.state.lock().expect("extension environment lock"); + if self.closed.load(Ordering::Acquire) { + drop(state); + runtime.shutdown(); + return Err(EnvironmentError::Closed); + } + state.runtimes.retain(|runtime| runtime.strong_count() > 0); + state.runtimes.push(Arc::downgrade(&runtime)); + Ok(runtime) + } + + /// Interrupt every child and in-progress initialization, then join workers. + /// Manager-driven normal unloading should run cleanup before this operation. + pub fn shutdown(&self) { + self.closed.store(true, Ordering::Release); + self.app_version.close(); + self.logs.shutdown(); + self.ffmpeg.shutdown(); + self.downloads.shutdown(); + self.auth.shutdown(); + self.sessions.shutdown(); + let _shutdown = self.shutdown_lock.lock().expect("extension shutdown lock"); + let runtimes = { + let mut state = self.state.lock().expect("extension environment lock"); + std::mem::take(&mut state.runtimes) + }; + for runtime in runtimes.into_iter().filter_map(|runtime| runtime.upgrade()) { + runtime.shutdown(); + } + let mut state = self.state.lock().expect("extension environment lock"); + while state.pending > 0 { + state = self.idle.wait(state).expect("extension environment idle"); + } + state.lyrics_nodes.clear(); + self.isrc.clear(); + } +} + +impl ExtensionEnvironment { + pub(crate) fn attach_lyrics(&self, lyrics: &Arc<LyricsService>) { + self.lyrics + .set(Arc::downgrade(lyrics)) + .expect("lyrics owner already configured"); + } + + pub(crate) fn lyrics_calls(&self) -> CallGraph { + self.calls.clone() + } + + pub(crate) fn lyrics_node(&self, id: &str, isolated: bool) -> CallNode { + if isolated { + return self.calls.node(); + } + self.state + .lock() + .expect("extension environment lock") + .lyrics_nodes + .entry(id.to_owned()) + .or_insert_with(|| self.calls.node()) + .clone() + } + + pub(crate) fn attach_manager(&self) { + self.manager_owned.store(true, Ordering::Release); + } + + pub(crate) fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + } + + pub(crate) fn closed_flag(&self) -> Arc<AtomicBool> { + Arc::clone(&self.closed) + } + pub(crate) fn data_directory(&self) -> &Path { + &self.data_directory + } + + /// Trusted root operations share download grants without widening any VM's + /// sandbox. Relative native paths resolve inside the application's data root. + pub(crate) fn native_files(&self) -> Result<Arc<crate::files::ExtensionFiles>, String> { + if self.is_closed() { + return Err("extension environment closed".into()); + } + self.files + .extension_with_alias(&self.data_directory, &self.data_directory_alias) + .map_err(|error| error.to_string()) + } + + /// Trusted native callers retain this grant until finalization completes. + pub fn grant_temporary_download_directory( + &self, + path: &Path, + ) -> Result<crate::files::TemporaryGrant, String> { + let _operation = self.enter().map_err(|error| error.to_string())?; + self.files + .grant_temporary_directory(path) + .map_err(|error| error.to_string()) + } + + pub(crate) fn validate_post_process_path( + &self, + id: &str, + input: &str, + path: &str, + ) -> Result<(), String> { + let mut bases = vec![self.data_directory.join(id)]; + // Native platforms may supply a trusted alias such as /var/... while + // the store owns /private/var/.... Accept that spelling only while it + // still resolves to this extension's actual data directory. + let alias = self.data_directory_alias.join(id); + if alias != bases[0] && fs::canonicalize(&alias).is_ok_and(|path| path == bases[0]) { + bases.push(alias); + } + if !input.is_empty() { + let input = crate::files::clean(Path::new(input)); + bases.insert( + 0, + input + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")) + .into(), + ); + } + self.files.validate_native_replacement(path, &bases) + } + + pub(crate) fn remove_data(&self, id: &str) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + if !valid_extension_id(id) { + return Err(StorageError::InvalidExtensionId.into()); + } + let mut state = self.state.lock().expect("extension environment lock"); + if let Some(store) = state.stores.remove(id) { + store.close(); + } + self.auth.clear(id); + self.sessions.forget_extension(id); + let path = self.data_directory.join(id); + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_dir() => { + fs::remove_dir_all(path).map_err(StorageError::from)? + } + Ok(_) => fs::remove_file(path).map_err(StorageError::from)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(StorageError::from(error).into()), + } + Ok(()) + } + + pub fn log_buffer(&self) -> Arc<crate::logging::LogBuffer> { + Arc::clone(&self.logs) + } + + pub fn repository( + &self, + directory: &Path, + ) -> Result<crate::repository::ExtensionRepository, crate::repository::RepositoryError> { + let _operation = self + .enter() + .map_err(|e| crate::repository::RepositoryError(e.to_string()))?; + crate::repository::ExtensionRepository::with_network(directory, Arc::clone(&self.network)) + .map(|repository| repository.attach_environment(Arc::clone(&self.closed))) + } + + pub fn ffmpeg_commands(&self) -> Arc<crate::ffmpeg::CommandRegistry> { + Arc::clone(&self.ffmpeg) + } + + pub fn download_state(&self) -> Arc<spotiflac_core::downloads::DownloadState> { + Arc::clone(&self.downloads) + } + + pub fn set_runtime_state(&self, raw: &str) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.sessions + .set_runtime_state(raw) + .map_err(EnvironmentError::Auth) + } + + pub fn set_session_grant(&self, id: &str, grant: &str) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.sessions + .set_grant(id, grant) + .map_err(EnvironmentError::Auth) + } + + pub fn pending_auth(&self, id: &str) -> Result<Option<PendingAuthRequest>, EnvironmentError> { + let _operation = self.enter()?; + Ok(self.auth.pending(id.trim())) + } + + pub fn all_pending_auth(&self) -> Result<Vec<PendingAuthRequest>, EnvironmentError> { + let _operation = self.enter()?; + Ok(self.auth.all_pending()) + } + + pub fn clear_pending_auth(&self, id: &str) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.auth.clear_pending(id); + Ok(()) + } + + pub fn resolve_callback_state( + &self, + state: &str, + consume: bool, + ) -> Result<String, EnvironmentError> { + let _operation = self.enter()?; + self.auth + .resolve_callback(state, consume) + .map_err(EnvironmentError::Auth) + } + + pub fn set_auth_code(&self, id: &str, code: &str) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.auth.set_code(id, code).map_err(EnvironmentError::Auth) + } + + pub fn set_auth_tokens( + &self, + id: &str, + access: &str, + refresh: &str, + expires_in: i64, + ) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.auth + .set_tokens(id, access, refresh, expires_in) + .map_err(EnvironmentError::Auth) + } + + pub fn is_authenticated(&self, id: &str) -> Result<bool, EnvironmentError> { + let _operation = self.enter()?; + Ok(self.auth.authenticated(id)) + } +} + +fn network_timeout(value: Option<&Value>) -> Duration { + let seconds = match value { + Some(Value::String(value)) => value.trim().parse::<i64>().unwrap_or(0), + Some(Value::Number(value)) => value.as_f64().unwrap_or(0.0) as i64, + _ => 0, + }; + Duration::from_secs(if seconds <= 0 { + 30 + } else { + seconds.clamp(5, 300) as u64 + }) +} + +impl Drop for ExtensionEnvironment { + fn drop(&mut self) { + self.shutdown(); + } +} + +struct Operation<'a>(&'a ExtensionEnvironment); +impl Drop for Operation<'_> { + fn drop(&mut self) { + let mut state = self.0.state.lock().expect("extension environment lock"); + state.pending -= 1; + self.0.idle.notify_all(); + } +} + +pub fn validate_gates( + manifest: &ExtensionManifest, + app_version: &str, + features: &BTreeMap<String, isize>, +) -> Result<(), String> { + let minimum = manifest.min_app_version.trim(); + let installed = app_version.trim(); + if !minimum.is_empty() && !installed.is_empty() && compare_versions(installed, minimum).is_lt() + { + return Err(format!( + "requires app {minimum} or later (installed: {installed})" + )); + } + for raw in &manifest.required_runtime_features { + let mut name = raw.trim(); + if name.is_empty() { + continue; + } + let mut wanted = 1; + if let Some((feature, version)) = name.rsplit_once('@') + && !feature.is_empty() + { + wanted = version + .parse::<isize>() + .ok() + .filter(|version| *version > 0) + .unwrap_or(1); + name = feature; + } + let Some(provided) = features.get(name) else { + return Err(format!( + "requires runtime feature {} this app build does not provide", + serde_json::to_string(name).expect("JSON string") + )); + }; + if *provided < wanted { + return Err(format!( + "requires runtime feature {name}@{wanted} (app provides @{provided})" + )); + } + } + Ok(()) +} + +pub fn compare_versions(first: &str, second: &str) -> Comparison { + fn parts(version: &str) -> impl Iterator<Item = isize> + '_ { + version + .strip_prefix('v') + .unwrap_or(version) + .split('.') + .map(|part| { + part.parse() + .unwrap_or_else(|error: std::num::ParseIntError| match error.kind() { + std::num::IntErrorKind::PosOverflow => isize::MAX, + std::num::IntErrorKind::NegOverflow => isize::MIN, + _ => 0, + }) + }) + } + let mut first = parts(first); + let mut second = parts(second); + loop { + let (left, right) = (first.next(), second.next()); + if left.is_none() && right.is_none() { + return Comparison::Equal; + } + let result = left.unwrap_or(0).cmp(&right.unwrap_or(0)); + if !result.is_eq() { + return result; + } + } +} diff --git a/rust_backend/crates/extensions/src/environment/index.rs b/rust_backend/crates/extensions/src/environment/index.rs new file mode 100644 index 00000000..2b4b07ba --- /dev/null +++ b/rust_backend/crates/extensions/src/environment/index.rs @@ -0,0 +1,92 @@ +use super::{EnvironmentError, ExtensionEnvironment}; +use spotiflac_core::isrc::{IndexCache, IndexFiles, NativeFiles, parse_tracks}; +use std::sync::atomic::Ordering; + +type Check<'a> = &'a (dyn Fn() -> Result<(), String> + Sync); + +impl ExtensionEnvironment { + fn with_index<T>( + &self, + check: Check<'_>, + operation: impl FnOnce(&IndexCache, &NativeFiles, Check<'_>) -> Result<T, String>, + ) -> Result<T, EnvironmentError> { + let _operation = self.enter()?; + let guarded = || { + if self.closed.load(Ordering::Acquire) { + Err("extension environment closed".into()) + } else { + check() + } + }; + guarded().map_err(EnvironmentError::Index)?; + let result = + operation(&self.isrc, &NativeFiles, &guarded).map_err(EnvironmentError::Index)?; + guarded().map_err(EnvironmentError::Index)?; + Ok(result) + } + + /// Trusted native entry points share the SDK's index, while JavaScript uses + /// a scoped IndexFiles adapter and revalidates native directory grants. + pub fn prebuild_isrc_index( + &self, + directory: &str, + check: Check<'_>, + ) -> Result<(), EnvironmentError> { + self.with_index(check, |cache, files, check| { + cache.prebuild(directory, files, check) + }) + } + + pub fn check_isrc_exists( + &self, + directory: &str, + isrc: &str, + check: Check<'_>, + ) -> Result<String, EnvironmentError> { + self.with_index(check, |cache, files, check| { + cache.check(directory, isrc, files, check) + }) + } + + pub fn add_to_isrc_index( + &self, + directory: &str, + isrc: &str, + path: &str, + check: Check<'_>, + ) -> Result<(), EnvironmentError> { + self.with_index(check, |cache, files, check| { + cache.add(directory, isrc, path, files, check) + }) + } + + pub fn check_files_exist_parallel( + &self, + directory: &str, + tracks_json: &str, + check: Check<'_>, + ) -> Result<String, EnvironmentError> { + self.with_index(check, |cache, files, check| { + if tracks_json.len() > 8 * 1024 * 1024 { + return Err("tracks JSON exceeds 8 MiB limit".into()); + } + let tracks = parse_tracks(tracks_json)?; + let results = cache.check_batch(directory, &tracks, files, check)?; + serde_json::to_string(&results).map_err(|error| error.to_string()) + }) + } + + pub fn invalidate_isrc_cache(&self, directory: &str) -> Result<(), EnvironmentError> { + let _operation = self.enter()?; + self.isrc.invalidate(directory); + Ok(()) + } + + pub fn check_file_exists(&self, path: &str) -> Result<bool, EnvironmentError> { + let _operation = self.enter()?; + NativeFiles + .stat(path) + .map(|stamp| stamp.is_some_and(|stamp| !stamp.directory && stamp.size > 0)) + .map_err(EnvironmentError::Index) + } +} diff --git a/rust_backend/crates/extensions/src/ffmpeg.rs b/rust_backend/crates/extensions/src/ffmpeg.rs new file mode 100644 index 00000000..c26fece6 --- /dev/null +++ b/rust_backend/crates/extensions/src/ffmpeg.rs @@ -0,0 +1,237 @@ +//! Commands executed by the native FFmpeg pump, scoped to one environment. + +use serde::Serialize; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Condvar, Mutex}; +use std::time::{Duration, Instant}; + +static NEXT_COMMAND: AtomicU64 = AtomicU64::new(0); +const COMMAND_TIMEOUT: Duration = Duration::from_secs(300); + +#[derive(Clone, Debug, Serialize)] +pub struct Command { + pub command_id: String, + pub extension_id: String, + pub arguments: Vec<String>, + pub input_path: String, + pub output_path: String, +} + +#[derive(Clone, Debug, Serialize)] +pub struct CommandResult { + pub success: bool, + pub output: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub error: String, +} + +struct Entry { + command: Command, + claimed: bool, + result: Option<CommandResult>, + waiter: std::thread::Thread, +} + +#[derive(Default)] +struct State { + commands: BTreeMap<String, Entry>, + closed: bool, +} + +#[derive(Default)] +pub struct CommandRegistry { + state: Mutex<State>, + changed: Condvar, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("FFmpeg command registry closed")] +pub struct RegistryClosed; + +impl CommandRegistry { + pub fn get(&self, id: &str) -> Result<Option<Command>, RegistryClosed> { + let state = self.state.lock().expect("FFmpeg state lock"); + state.check()?; + Ok(state.commands.get(id).map(|entry| entry.command.clone())) + } + + pub fn pending_json(&self) -> Result<String, RegistryClosed> { + let mut state = self.state.lock().expect("FFmpeg state lock"); + state.check()?; + Ok(state.pending(false)) + } + + /// Claims each command at most once, even with multiple native pumps. + /// A zero/negative timeout performs a nonblocking claim. Shutdown wakes waits. + pub fn wait_pending_json(&self, timeout_ms: i64) -> Result<String, RegistryClosed> { + let started = Instant::now(); + let timeout = Duration::from_millis(timeout_ms.clamp(0, 60_000) as u64); + let mut state = self.state.lock().expect("FFmpeg state lock"); + loop { + state.check()?; + let commands = state.pending(true); + let remaining = timeout.saturating_sub(started.elapsed()); + if commands != "[]" || remaining.is_zero() { + return Ok(commands); + } + state = self + .changed + .wait_timeout(state, remaining) + .expect("FFmpeg pending wait") + .0; + } + } + + /// Late or duplicate results cannot complete another command or revive one + /// removed by cancellation. The native pump can use get() to stop such work. + pub fn complete(&self, id: &str, result: CommandResult) -> Result<bool, RegistryClosed> { + let mut state = self.state.lock().expect("FFmpeg state lock"); + state.check()?; + let Some(entry) = state.commands.get_mut(id) else { + return Ok(false); + }; + if entry.result.is_some() { + return Ok(false); + } + entry.result = Some(result); + entry.waiter.unpark(); + self.changed.notify_all(); + Ok(true) + } + + pub fn shutdown(&self) { + let mut state = self.state.lock().expect("FFmpeg state lock"); + state.closed = true; + for entry in state.commands.values() { + entry.waiter.unpark(); + } + state.commands.clear(); + self.changed.notify_all(); + } + + pub(crate) fn execute( + &self, + extension_id: &str, + arguments: Vec<String>, + input_path: String, + output_path: String, + control: &crate::runtime::Control, + ) -> Result<CommandResult, String> { + let _wait = control.watch_thread(); + let check = || control.check().map_err(|error| error.to_string()); + check()?; + let sequence = NEXT_COMMAND.fetch_add(1, Ordering::Relaxed).wrapping_add(1); + let id = format!("{extension_id}_{sequence}"); + let command = Command { + command_id: id.clone(), + extension_id: extension_id.into(), + arguments, + input_path, + output_path, + }; + { + let mut state = self.state.lock().expect("FFmpeg state lock"); + state.check().map_err(|e| e.to_string())?; + state.commands.insert( + id.clone(), + Entry { + command, + claimed: false, + result: None, + waiter: std::thread::current(), + }, + ); + self.changed.notify_all(); + } + let _pending = Pending { + registry: self, + id: &id, + }; + let started = Instant::now(); + let mut state = self.state.lock().expect("FFmpeg state lock"); + loop { + check().map_err(|e| format!("FFmpeg command cancelled: {e}"))?; + state.check().map_err(|e| e.to_string())?; + if state + .commands + .get(&id) + .is_some_and(|entry| entry.result.is_some()) + { + return Ok(state + .commands + .remove(&id) + .expect("completed FFmpeg command") + .result + .expect("FFmpeg result")); + } + let remaining = COMMAND_TIMEOUT.saturating_sub(started.elapsed()); + if remaining.is_zero() { + return Err("FFmpeg command timed out".into()); + } + drop(state); + std::thread::park_timeout(remaining.min(control.deadline_remaining())); + state = self.state.lock().expect("FFmpeg state lock"); + } + } +} + +impl State { + fn check(&self) -> Result<(), RegistryClosed> { + if self.closed { + Err(RegistryClosed) + } else { + Ok(()) + } + } + + fn pending(&mut self, claim: bool) -> String { + #[derive(Serialize)] + struct PendingCommand<'a> { + command_id: &'a str, + extension_id: &'a str, + arguments: &'a [String], + output_path: &'a str, + } + let pending: Vec<_> = self + .commands + .values_mut() + .filter(|entry| !entry.claimed && entry.result.is_none()) + .map(|entry| { + if claim { + entry.claimed = true; + } + PendingCommand { + command_id: &entry.command.command_id, + extension_id: &entry.command.extension_id, + arguments: &entry.command.arguments, + output_path: &entry.command.output_path, + } + }) + .collect(); + serde_json::to_string(&pending).expect("FFmpeg pending JSON") + } +} + +struct Pending<'a> { + registry: &'a CommandRegistry, + id: &'a str, +} + +impl Drop for Pending<'_> { + fn drop(&mut self) { + self.registry + .state + .lock() + .expect("FFmpeg state lock") + .commands + .remove(self.id); + self.registry.changed.notify_all(); + } +} + +impl Drop for CommandRegistry { + fn drop(&mut self) { + self.shutdown(); + } +} diff --git a/rust_backend/crates/extensions/src/ffmpeg_host.rs b/rust_backend/crates/extensions/src/ffmpeg_host.rs new file mode 100644 index 00000000..3153f97b --- /dev/null +++ b/rust_backend/crates/extensions/src/ffmpeg_host.rs @@ -0,0 +1,155 @@ +use crate::files::ExtensionFiles; +use crate::runtime::{Control, ExtensionServices}; +use cap_std::fs::OpenOptions; +use rquickjs::{Ctx, Function, Object, Value}; +use std::sync::Arc; + +pub(crate) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + host.set("rawFfmpegStub", services.raw_ffmpeg_stub)?; + let Some(files) = &services.files else { + return Ok(()); + }; + let info_files = Arc::clone(files); + let info_control = Arc::clone(&control); + host.set( + "mediaInfo", + Function::new(ctx.clone(), move |ctx: Ctx<'js>, path: String| { + let check = || info_control.check().map_err(|e| e.to_string()); + let result = media_info(&info_files, &path, &check); + let object = Object::new(ctx.clone())?; + match result { + Ok(quality) => { + object.set("success", true)?; + object.set("bit_depth", quality.bit_depth)?; + object.set("sample_rate", quality.sample_rate)?; + object.set("total_samples", quality.total_samples)?; + object.set( + "duration", + quality.total_samples as f64 / quality.sample_rate as f64, + )?; + object.set("codec", quality.codec)?; + } + Err(error) => { + object.set("success", false)?; + object.set("error", error)?; + } + } + Ok::<_, rquickjs::Error>(object) + })?, + )?; + let files = Arc::clone(files); + let registry = Arc::clone(&services.ffmpeg); + let id = services.extension_id.clone(); + host.set( + "mediaConvert", + Function::new( + ctx.clone(), + move |input: String, output: String, options: Value<'js>| { + let result = (|| { + let input = files + .resolve(&input) + .and_then(|path| path.native_display()) + .map_err(|e| format!("invalid input path: {e}"))?; + let output = files + .resolve(&output) + .and_then(|path| path.native_display()) + .map_err(|e| format!("invalid output path: {e}"))?; + let arguments = conversion_arguments(&input, &output, options.as_object())?; + let budget = control.resolution(); + let _pause = budget.as_ref().map(|budget| budget.enter(false)); + registry.execute(&id, arguments, input, output, &control) + })(); + match result { + Ok(result) => serde_json::to_string(&result).expect("FFmpeg result JSON"), + Err(error) => serde_json::json!({"success":false,"error":error}).to_string(), + } + }, + )?, + )?; + Ok(()) +} + +fn media_info( + files: &ExtensionFiles, + path: &str, + check: &dyn Fn() -> Result<(), String>, +) -> Result<spotiflac_core::media::AudioQuality, String> { + let path = files.resolve(path)?; + let mut file = path + .open(OpenOptions::new().read(true)) + .map_err(|e| format!("failed to open file: {e}"))?; + spotiflac_core::media::probe_quality(&mut file, check) +} + +fn conversion_arguments( + input: &str, + output: &str, + options: Option<&Object<'_>>, +) -> Result<Vec<String>, String> { + let mut arguments = vec![ + "-hide_banner".into(), + "-nostdin".into(), + "-i".into(), + input.into(), + ]; + if let Some(options) = options { + for (key, flag) in [("codec", "-c:a"), ("bitrate", "-b:a")] { + let value: Value = options.get(key).map_err(|e| e.to_string())?; + if let Some(value) = value.as_string() { + let value = value.to_string().map_err(|e| e.to_string())?; + if key == "codec" { + if !matches!( + value.as_str(), + "aac" + | "alac" + | "copy" + | "flac" + | "libmp3lame" + | "libopus" + | "opus" + | "pcm_s16le" + | "pcm_s24le" + ) { + return Err("unsupported audio codec".into()); + } + } else { + let digits = value.strip_suffix(['k', 'K', 'm', 'M']).unwrap_or(&value); + if digits.is_empty() + || digits.len() > 8 + || digits.starts_with('0') + || !digits.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err("invalid audio bitrate".into()); + } + } + arguments.extend([flag.into(), value]); + } + } + for (key, message) in [ + ("sample_rate", "invalid sample rate"), + ("channels", "invalid channel count"), + ] { + let value: Value = options.get(key).map_err(|e| e.to_string())?; + if let Some(value) = value.as_number() { + // Goja exports integral JS Numbers as int64; the old host only + // accepts float64 here. Preserve its ignored integer options. + // Every remaining float is fractional, non-finite, or outside + // Goja's integer range, and fails the host's range/integer check. + if !value.is_finite() + || value.fract() != 0.0 + || (value == 0.0 && value.is_sign_negative()) + || !(-9_007_199_254_740_992.0..=9_007_199_254_740_992.0).contains(&value) + { + return Err(message.into()); + } + } + } + } + arguments.extend(["-y".into(), output.into()]); + Ok(arguments) +} diff --git a/rust_backend/crates/extensions/src/file_host.rs b/rust_backend/crates/extensions/src/file_host.rs new file mode 100644 index 00000000..5d345132 --- /dev/null +++ b/rust_backend/crates/extensions/src/file_host.rs @@ -0,0 +1,311 @@ +use crate::binary; +use crate::files::{ExtensionFiles, MAX_READ, READ_LIMIT_ERROR}; +use crate::runtime::{Control, ExtensionServices}; +use cap_std::fs::OpenOptions; +use rquickjs::{ArrayBuffer, Ctx, Function, Object, Value}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::sync::Arc; + +pub(crate) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + host.set("filesEnabled", services.files.is_some())?; + let Some(files) = &services.files else { + return Ok(()); + }; + let files = Arc::clone(files); + host.set( + "fileCall", + Function::new( + ctx.clone(), + move |ctx: Ctx<'js>, + method: String, + path: String, + data: Value<'js>, + options: Value<'js>, + callback: Value<'js>| { + let check = || control.check().map_err(|error| error.to_string()); + let result = call( + &ctx, &files, &method, &path, data, options, callback, &check, + ); + if method == "exists" { + return Ok::<_, rquickjs::Error>(Value::new_bool(ctx, result.is_ok())); + } + match result { + Ok(object) => Ok(object.into_value()), + Err(error) => { + let object = Object::new(ctx)?; + object.set("success", false)?; + object.set("error", error)?; + Ok(object.into_value()) + } + } + }, + )?, + )?; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn call<'js>( + ctx: &Ctx<'js>, + files: &ExtensionFiles, + method: &str, + path: &str, + data: Value<'js>, + options: Value<'js>, + callback: Value<'js>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Object<'js>, String> { + check()?; + let path = files.resolve(path)?; + let options = Options(options.as_object()); + let result = Object::new(ctx.clone()).map_err(js_error)?; + result.set("success", true).map_err(js_error)?; + match method { + "exists" | "getSize" => { + let info = path.metadata().map_err(io_error)?; + if method == "getSize" { + result.set("size", info.len()).map_err(js_error)?; + } + } + "delete" => { + let _guard = files.lock(&path, check)?; + path.remove().map_err(io_error)?; + } + "read" | "readBytes" => { + let offset = if method == "read" { + 0 + } else { + options.int("offset", 0)? + }; + let requested = options.int("length", -1)?; + let encoding = options.text("encoding", "base64")?; + if offset < 0 { + return Err("offset must be >= 0".into()); + } + let mut input = path.open(OpenOptions::new().read(true)).map_err(io_error)?; + let size = input.metadata().map_err(io_error)?.len(); + let offset = (offset as u64).min(size); + let mut bytes; + if method == "read" { + bytes = Vec::new(); + let mut buffer = [0; 64 << 10]; + while bytes.len() <= MAX_READ { + check()?; + let capacity = buffer.len().min(MAX_READ + 1 - bytes.len()); + let count = input.read(&mut buffer[..capacity]).map_err(io_error)?; + if count == 0 { + break; + } + bytes.extend_from_slice(&buffer[..count]); + } + if bytes.len() > MAX_READ { + return Err(READ_LIMIT_ERROR.into()); + } + result + .set("data", crate::host::decode_go_utf8(&bytes)) + .map_err(js_error)?; + } else { + input + .seek(SeekFrom::Start(offset)) + .map_err(|error| format!("failed to seek file: {error}"))?; + let length = if requested < 0 { + size - offset + } else { + requested as u64 + }; + if length > MAX_READ as u64 { + return Err(READ_LIMIT_ERROR.into()); + } + bytes = vec![0; length.min(size - offset) as usize]; + let count = read_chunk(&mut input, &mut bytes, check) + .map_err(|error| format!("failed to read file: {error}"))?; + bytes.truncate(count); + if matches!(encoding.trim().to_lowercase().as_str(), "bytes" | "raw") { + result + .set( + "data", + ArrayBuffer::new_copy(ctx.clone(), &bytes).map_err(js_error)?, + ) + .map_err(js_error)?; + } else { + result + .set("data", binary::encode(&bytes, &encoding)?) + .map_err(js_error)?; + } + result.set("bytes_read", bytes.len()).map_err(js_error)?; + result.set("offset", offset).map_err(js_error)?; + result.set("size", size).map_err(js_error)?; + result + .set("eof", offset + bytes.len() as u64 >= size) + .map_err(js_error)?; + } + } + "write" | "writeBytes" => { + let append = options.boolean("append", false)?; + let truncate = options.boolean("truncate", false)?; + let has_offset = options + .0 + .map(|object| object.contains_key("offset")) + .transpose() + .map_err(js_error)? + .unwrap_or(false); + let offset = options.int("offset", 0)?; + if append && has_offset { + return Err("append and offset cannot be used together".into()); + } + if offset < 0 { + return Err("offset must be >= 0".into()); + } + let encoding = if method == "write" { + "utf8".into() + } else { + options.text("encoding", "base64")? + }; + let bytes = binary::decode_value(data, &encoding)?; + let _guard = files.lock(&path, check)?; + path.mkdir_parent() + .map_err(|error| format!("failed to create directory: {error}"))?; + if method == "write" { + let mut output = path.stage().map_err(io_error)?; + write_chunks(&mut output.file, &bytes, check)?; + output.publish(check)?; + } else { + let mut output = path + .open( + OpenOptions::new() + .create(true) + .write(true) + .append(append) + .truncate(truncate), + ) + .map_err(io_error)?; + if has_offset { + output + .seek(SeekFrom::Start(offset as u64)) + .map_err(|error| format!("failed to seek file: {error}"))?; + } + write_chunks(&mut output, &bytes, check)?; + result.set("bytes_written", bytes.len()).map_err(js_error)?; + result + .set("size", output.metadata().map_err(io_error)?.len()) + .map_err(js_error)?; + } + result.set("path", path.display()).map_err(js_error)?; + } + "copy" | "move" | "transformPatternedBlocks" => { + let destination = data + .as_string() + .ok_or("destination path is required")? + .to_string() + .map_err(js_error)?; + let destination = files.resolve(&destination)?; + if method == "transformPatternedBlocks" { + return crate::file_transform::transform( + ctx, + files, + &path, + &destination, + options, + callback, + check, + ); + } + let _guard = files.lock(&destination, check)?; + if method == "move" { + destination + .mkdir_parent() + .map_err(|error| format!("failed to create directory: {error}"))?; + check()?; + path.rename_to(&destination) + .map_err(|error| format!("failed to move file: {error}"))?; + } else { + let mut input = path + .open(OpenOptions::new().read(true)) + .map_err(|error| format!("failed to read source: {error}"))?; + destination + .mkdir_parent() + .map_err(|error| format!("failed to create directory: {error}"))?; + let mut output = destination + .open(OpenOptions::new().write(true).create(true).truncate(true)) + .map_err(|error| format!("failed to open destination: {error}"))?; + let mut buffer = [0; 64 << 10]; + loop { + let count = read_chunk(&mut input, &mut buffer, check) + .map_err(|error| format!("failed to copy file: {error}"))?; + if count == 0 { + break; + } + write_chunks(&mut output, &buffer[..count], check) + .map_err(|error| format!("failed to copy file: {error}"))?; + } + } + result + .set("path", destination.display()) + .map_err(js_error)?; + } + _ => return Err("unknown file operation".into()), + } + check()?; + Ok(result) +} + +pub(crate) struct Options<'a, 'js>(pub Option<&'a Object<'js>>); + +impl Options<'_, '_> { + pub fn text(&self, key: &str, fallback: &str) -> Result<String, String> { + binary::option_string(self.0, key, fallback) + } + pub fn int(&self, key: &str, fallback: i64) -> Result<i64, String> { + self.0.map_or(Ok(fallback), |object| { + binary::option_i64(object, key, fallback) + }) + } + pub fn boolean(&self, key: &str, fallback: bool) -> Result<bool, String> { + self.0.map_or(Ok(fallback), |object| { + binary::option_bool(object, key, fallback) + }) + } +} + +pub(crate) fn read_chunk( + input: &mut impl Read, + buffer: &mut [u8], + check: &dyn Fn() -> Result<(), String>, +) -> Result<usize, String> { + let mut total = 0; + while total < buffer.len() { + check()?; + let end = buffer.len().min(total + (64 << 10)); + match input.read(&mut buffer[total..end]) { + Ok(0) => break, + Ok(count) => total += count, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error.to_string()), + } + } + Ok(total) +} + +pub(crate) fn write_chunks( + output: &mut impl Write, + bytes: &[u8], + check: &dyn Fn() -> Result<(), String>, +) -> Result<(), String> { + for chunk in bytes.chunks(64 << 10) { + check()?; + output.write_all(chunk).map_err(io_error)?; + } + check() +} + +pub(crate) fn js_error(error: rquickjs::Error) -> String { + error.to_string() +} +fn io_error(error: std::io::Error) -> String { + error.to_string() +} diff --git a/rust_backend/crates/extensions/src/file_transform.rs b/rust_backend/crates/extensions/src/file_transform.rs new file mode 100644 index 00000000..72748dd5 --- /dev/null +++ b/rust_backend/crates/extensions/src/file_transform.rs @@ -0,0 +1,159 @@ +use crate::binary; +use crate::crypto::BlockCipher; +use crate::file_host::{Options, js_error, read_chunk, write_chunks}; +use crate::files::{ExtensionFiles, FilePath, MAX_READ}; +use cap_std::fs::OpenOptions; +use rquickjs::{Coerced, Ctx, FromJs, Object, Value}; +use zeroize::Zeroizing; + +#[allow(clippy::too_many_arguments)] +pub(crate) fn transform<'js>( + ctx: &Ctx<'js>, + files: &ExtensionFiles, + input: &FilePath, + output: &FilePath, + options: Options<'_, 'js>, + callback: Value<'js>, + check: &dyn Fn() -> Result<(), String>, +) -> Result<Object<'js>, String> { + let algorithm = options.text("algorithm", "")?.to_lowercase(); + let mode = options.text("mode", "cbc")?.to_lowercase(); + let padding = options.text("padding", "none")?.to_lowercase(); + if algorithm.is_empty() { + return Err("algorithm is required".into()); + } + let key = Zeroizing::new( + binary::decode_option_bytes(options.0, "key", &options.text("keyEncoding", "utf8")?) + .map_err(|error| format!("invalid key: {error}"))?, + ); + if key.is_empty() { + return Err("key is required".into()); + } + let iv = binary::decode_option_bytes(options.0, "iv", &options.text("ivEncoding", "utf8")?) + .map_err(|error| format!("invalid iv: {error}"))?; + if mode != "cbc" && mode != "ctr" { + return Err(format!("unsupported block cipher mode: {mode}")); + } + if padding != "none" { + return Err("patterned file transforms only support padding: none".into()); + } + let operation = options.text("operation", "decrypt")?.to_lowercase(); + if operation != "decrypt" && operation != "encrypt" { + return Err("operation must be decrypt or encrypt".into()); + } + let segment_size = options.int("segmentSize", 0)?; + let every = options.int("transformEvery", 1)?; + let offset = options.int("transformOffset", 0)?; + let buffer_size = options.int("bufferSize", 1 << 20)?; + let partial = options.boolean("transformPartial", false)?; + if !(1..=MAX_READ as i64).contains(&segment_size) { + return Err(format!( + "segmentSize must be between 1 and {MAX_READ} bytes" + )); + } + if every <= 0 { + return Err("transformEvery must be greater than zero".into()); + } + if offset < 0 || offset >= every { + return Err("transformOffset must be between 0 and transformEvery - 1".into()); + } + let buffer_size = buffer_size.clamp(segment_size, MAX_READ as i64); + let buffer_size = (buffer_size - buffer_size % segment_size) as usize; + let cipher = BlockCipher::new(&algorithm, &key)?; + let block_size = cipher.block_size(); + if iv.len() != block_size { + return Err(format!("iv must be {block_size} bytes for {algorithm}")); + } + if mode == "cbc" && segment_size % block_size as i64 != 0 { + return Err(format!( + "segmentSize must be a multiple of {block_size} bytes for CBC" + )); + } + let callback = if callback.is_null() || callback.is_undefined() { + None + } else { + Some( + callback + .as_function() + .ok_or("progress callback must be a function")?, + ) + }; + let _guard = files.lock(output, check)?; + let mut source = input + .open(OpenOptions::new().read(true)) + .map_err(|error| format!("failed to open input file: {error}"))?; + let total = source + .metadata() + .map_err(|error| format!("failed to stat input file: {error}"))? + .len(); + // Keep the legacy collision check although randomized staging no longer + // risks clobbering a caller's existing .transform.partial file. + if input.display() == format!("{}.transform.partial", output.display()) { + return Err("input path conflicts with transform staging path".into()); + } + output + .mkdir_parent() + .map_err(|error| format!("failed to create output directory: {error}"))?; + let mut staged = output + .stage() + .map_err(|error| format!("failed to create staged output: {error}"))?; + let mut buffer = Zeroizing::new(vec![0; buffer_size]); + let (mut processed, mut index, mut transformed) = (0_u64, 0_u64, 0_u64); + loop { + let count = read_chunk(&mut source, &mut buffer, check) + .map_err(|error| format!("failed to read input file: {error}"))?; + if count == 0 { + break; + } + for segment in buffer[..count].chunks_mut(segment_size as usize) { + check()?; + if index % every as u64 == offset as u64 + && (segment.len() == segment_size as usize || partial) + { + if mode == "cbc" && segment.len() % block_size != 0 { + return Err(format!( + "selected segment {index} is not a multiple of {block_size} bytes" + )); + } + // Each selected segment restarts its IV/counter, matching Go. + cipher.transform(segment, &iv, &mode, operation == "decrypt", check)?; + transformed += 1; + } + index += 1; + } + write_chunks(&mut staged.file, &buffer[..count], check) + .map_err(|error| format!("failed to write transformed file: {error}"))?; + processed += count as u64; + if let Some(callback) = callback + && let Err(error) = callback.call::<_, ()>((processed, total)) + { + let message = if error.is_exception() { + let exception = ctx.catch(); + match Coerced::<String>::from_js(ctx, exception) { + Ok(message) => message.0, + Err(_) => { + ctx.catch(); + "JavaScript exception".into() + } + } + } else { + error.to_string() + }; + return Err(format!("progress callback failed: {message}")); + } + check()?; + } + drop(source); + staged + .publish(check) + .map_err(|error| format!("failed to publish transformed file: {error}"))?; + let result = Object::new(ctx.clone()).map_err(js_error)?; + result.set("success", true).map_err(js_error)?; + result.set("path", output.display()).map_err(js_error)?; + result.set("bytes_processed", processed).map_err(js_error)?; + result.set("segments_processed", index).map_err(js_error)?; + result + .set("segments_transformed", transformed) + .map_err(js_error)?; + Ok(result) +} diff --git a/rust_backend/crates/extensions/src/files.rs b/rust_backend/crates/extensions/src/files.rs new file mode 100644 index 00000000..3ca5d56f --- /dev/null +++ b/rust_backend/crates/extensions/src/files.rs @@ -0,0 +1,936 @@ +//! File capabilities granted by the native manager, never by JavaScript. + +mod index; + +use cap_std::ambient_authority; +use cap_std::fs::{Dir, File, OpenOptions}; +use std::collections::BTreeMap; +use std::io; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, Condvar, Mutex, RwLock, Weak}; +use std::thread::{self, ThreadId}; +use std::time::Duration; + +pub(crate) const MAX_READ: usize = 16 << 20; +pub(crate) const READ_LIMIT_ERROR: &str = "file read exceeds 16 MiB limit; use file.readBytes with offset and length to read it in chunks"; + +struct Root { + path: PathBuf, + directory: Dir, +} + +impl Root { + fn open(path: &Path) -> io::Result<Arc<Self>> { + let path = clean(&std::path::absolute(path)?); + std::fs::create_dir_all(&path)?; + Ok(Arc::new(Self { + directory: Dir::open_ambient_dir(&path, ambient_authority())?, + path, + })) + } +} + +#[derive(Default)] +pub struct FileRegistry { + allowed: RwLock<Vec<Arc<Root>>>, + temporary: RwLock<Vec<Weak<Root>>>, + outputs: Mutex<BTreeMap<PathBuf, Weak<OutputLock>>>, +} + +/// Retain one host-created download directory while its provider is running. +/// Open file capabilities may outlive the grant; future opens may not. +pub struct TemporaryGrant { + root: Arc<Root>, + registry: Arc<FileRegistry>, +} + +impl Drop for TemporaryGrant { + fn drop(&mut self) { + let key = Arc::downgrade(&self.root); + self.registry + .temporary + .write() + .expect("temporary file grants lock") + .retain(|root| !Weak::ptr_eq(root, &key)); + } +} + +impl FileRegistry { + /// Replace the native download-directory grants for future operations. + /// Open descriptors keep an in-progress operation bound to its original root. + pub fn set_allowed_directories(&self, paths: &[PathBuf]) -> io::Result<()> { + let roots = paths + .iter() + .map(|path| Root::open(path)) + .collect::<io::Result<Vec<_>>>()?; + *self.allowed.write().expect("file grants lock") = roots; + Ok(()) + } + + pub(crate) fn grant_temporary_directory( + self: &Arc<Self>, + path: &Path, + ) -> io::Result<TemporaryGrant> { + let root = Root::open(path)?; + let mut temporary = self.temporary.write().expect("temporary file grants lock"); + temporary.retain(|root| root.strong_count() != 0); + temporary.push(Arc::downgrade(&root)); + Ok(TemporaryGrant { + root, + registry: Arc::clone(self), + }) + } + + pub fn extension(self: &Arc<Self>, directory: &Path) -> io::Result<Arc<ExtensionFiles>> { + self.extension_with_alias(directory, directory) + } + + pub(crate) fn extension_with_alias( + self: &Arc<Self>, + directory: &Path, + alias: &Path, + ) -> io::Result<Arc<ExtensionFiles>> { + let root = Root::open(directory)?; + let alias = clean(&std::path::absolute(alias)?); + let alias = if alias == root.path { + None + } else if std::fs::canonicalize(&alias)? == std::fs::canonicalize(&root.path)? { + Some(alias) + } else { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "sandbox alias refers to a different directory", + )); + }; + Ok(Arc::new(ExtensionFiles { + root, + legacy_alias: alias, + registry: Arc::clone(self), + })) + } + + pub(crate) fn validate_native_replacement( + &self, + path: &str, + bases: &[PathBuf], + ) -> Result<(), String> { + let target = clean(Path::new(path)); + let granted = self.allowed.read().expect("file grants lock"); + let temporary: Vec<_> = self + .temporary + .read() + .expect("temporary file grants lock") + .iter() + .filter_map(Weak::upgrade) + .collect(); + for base in bases + .iter() + .chain(granted.iter().chain(&temporary).map(|root| &root.path)) + { + let base = clean(&std::path::absolute(base).map_err(|error| error.to_string())?); + if let Ok(relative) = target.strip_prefix(&base) { + let directory = match Dir::open_ambient_dir(&base, ambient_authority()) { + Ok(directory) => directory, + // Go accepts a future output path; validation must not create it. + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.to_string()), + }; + return FilePath { + root: Arc::new(Root { + path: base.clone(), + directory, + }), + absolute: target.clone(), + relative: relative.into(), + } + .native_display() + .map(|_| ()); + } + } + Err("replacement file path is outside allowed directories".into()) + } + + fn lock( + &self, + path: &Path, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<OutputGuard, String> { + let lock = { + let mut outputs = self.outputs.lock().expect("file outputs lock"); + outputs.retain(|_, value| value.strong_count() > 0); + let lock = outputs + .get(path) + .and_then(Weak::upgrade) + .unwrap_or_default(); + outputs.insert(path.to_owned(), Arc::downgrade(&lock)); + lock + }; + let mut owner = lock.owner.lock().expect("file owner lock"); + while let Some(id) = *owner { + check()?; + if id == thread::current().id() { + return Err("file output is already in use by this callback".into()); + } + owner = lock + .idle + .wait_timeout(owner, Duration::from_millis(10)) + .expect("file output wait") + .0; + } + check()?; + *owner = Some(thread::current().id()); + drop(owner); + Ok(OutputGuard(lock)) + } +} + +pub struct ExtensionFiles { + root: Arc<Root>, + legacy_alias: Option<PathBuf>, + registry: Arc<FileRegistry>, +} + +impl ExtensionFiles { + pub(crate) fn resolve_legacy(&self, path: &str) -> Result<FilePath, String> { + let original = Path::new(path); + let path = clean(original); + if path.is_absolute() { + for root in std::iter::once(&self.root.path).chain(self.legacy_alias.as_ref()) { + if let Ok(relative) = path.strip_prefix(root) { + if root != &self.root.path + && !std::fs::canonicalize(root).is_ok_and(|alias| { + std::fs::canonicalize(&self.root.path) + .is_ok_and(|canonical| alias == canonical) + }) + { + return Err( + "file access denied: sandbox alias refers to a different directory" + .into(), + ); + } + let mut resolved = + self.resolve(relative.to_str().ok_or("invalid UTF-8 path")?)?; + // Keep native path spelling in cache keys and results while + // opening through the original sandbox capability. + resolved.absolute = original.to_owned(); + return Ok(resolved); + } + } + } + let mut resolved = self.resolve(path.to_str().ok_or("invalid UTF-8 path")?)?; + if original.is_absolute() { + resolved.absolute = original.to_owned(); + } + Ok(resolved) + } + + pub(crate) fn resolve(&self, path: &str) -> Result<FilePath, String> { + let input = clean(Path::new(path)); + let (root, absolute) = if input.is_absolute() { + let root = self + .registry + .allowed + .read() + .expect("file grants lock") + .iter() + .find(|root| input.starts_with(&root.path)) + .cloned(); + let root = root.or_else(|| self.registry.temporary.read().expect("temporary file grants lock") + .iter().filter_map(Weak::upgrade).find(|root| input.starts_with(&root.path))) + .ok_or("file access denied: absolute paths are not allowed. Use relative paths within extension sandbox")?; + (root, input) + } else { + let absolute = clean(&self.root.path.join(input)); + if !absolute.starts_with(&self.root.path) { + return Err(format!( + "file access denied: path '{path}' is outside sandbox" + )); + } + (Arc::clone(&self.root), absolute) + }; + let relative = absolute + .strip_prefix(&root.path) + .expect("validated file root") + .to_owned(); + Ok(FilePath { + root, + absolute, + relative: if relative.as_os_str().is_empty() { + ".".into() + } else { + relative + }, + }) + } + + pub(crate) fn lock( + &self, + path: &FilePath, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<OutputGuard, String> { + // Go coordinates case-folded paths, including aliases on the default + // case-insensitive mobile/macOS filesystems. + self.registry + .lock(Path::new(&path.display().to_lowercase()), check) + } +} + +pub(crate) struct FilePath { + root: Arc<Root>, + pub absolute: PathBuf, + relative: PathBuf, +} + +impl FilePath { + /// Native FFmpeg opens a pathname rather than this capability descriptor. + /// Reject existing symlinks and special files before handing it to native. + /// The adapter must retain exclusive ownership of these paths through use. + pub(crate) fn native_display(&self) -> Result<String, String> { + let mut path = PathBuf::new(); + for component in self.relative.components() { + path.push(component); + match self.root.directory.symlink_metadata(&path) { + Ok(metadata) => { + if metadata.is_symlink() || (!metadata.is_file() && !metadata.is_dir()) { + return Err( + "native media path must not contain symlinks or special files".into(), + ); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => break, + Err(error) => return Err(error.to_string()), + } + } + Ok(self.display()) + } + + pub(crate) fn open_parent(&self) -> io::Result<(Dir, std::ffi::OsString)> { + self.mkdir_parent()?; + let name = self.relative.file_name().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "invalid output filename") + })?; + Ok(( + self.root.directory.open_dir(self.parent())?, + name.to_owned(), + )) + } + + pub fn display(&self) -> String { + self.absolute.to_string_lossy().into_owned() + } + + pub fn metadata(&self) -> io::Result<cap_std::fs::Metadata> { + self.root.directory.metadata(&self.relative) + } + + pub(crate) fn entries(&self) -> io::Result<Vec<(String, bool)>> { + let mut entries = self + .root + .directory + .read_dir(&self.relative)? + .map(|entry| { + let entry = entry?; + Ok(( + entry.file_name().to_string_lossy().into_owned(), + entry.file_type()?.is_dir(), + )) + }) + .collect::<io::Result<Vec<_>>>()?; + entries.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(entries) + } + + pub fn mkdir_parent(&self) -> io::Result<()> { + self.root.directory.create_dir_all(self.parent()) + } + + pub(crate) fn require_parent(&self) -> Result<(), String> { + self.root + .directory + .open_dir(self.parent()) + .map(|_| ()) + .map_err(|error| error.to_string()) + } + + fn parent(&self) -> &Path { + self.relative + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")) + } + + pub fn open(&self, options: &mut OpenOptions) -> io::Result<File> { + #[cfg(unix)] + { + use cap_std::fs::OpenOptionsExt; + // FIFOs/devices must not block a VM worker before cancellation can + // run. Only regular files are accepted after opening the descriptor. + options + .custom_flags(rustix::fs::OFlags::NONBLOCK.bits() as i32) + .mode(0o644); + } + let file = self.root.directory.open_with(&self.relative, options)?; + if !file.metadata()?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "not a regular file", + )); + } + Ok(file) + } + + /// Open a native read through its granted directory, rejecting symlinks at + /// every component without separately walking the same path for validation. + #[cfg(unix)] + pub(crate) fn open_native_read(&self) -> io::Result<File> { + use rustix::fd::{AsFd, OwnedFd}; + use rustix::fs::{AtFlags, FileType, Mode, OFlags, openat, statat}; + + let mut directory: Option<OwnedFd> = None; + let mut components = self.relative.components().peekable(); + while let Some(component) = components.next() { + if !matches!(component, Component::Normal(_) | Component::CurDir) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid native path", + )); + } + let name = component.as_os_str(); + let parent = directory + .as_ref() + .map_or_else(|| self.root.directory.as_fd(), AsFd::as_fd); + let last = components.peek().is_none(); + let mut flags = OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW | OFlags::NONBLOCK; + if !last { + flags |= OFlags::DIRECTORY; + // Match capability traversal: a known child only needs search + // permission on its parents, not permission to list them. + #[cfg(any(target_os = "linux", target_os = "android"))] + { + flags |= OFlags::PATH; + } + } + let fd = match openat(parent, name, flags, Mode::empty()) { + Ok(fd) => fd, + Err(error) => { + // Preserve the native-path error for links and special + // files even when a platform reports ENOTDIR or ENXIO. + if error != rustix::io::Errno::NOENT + && let Ok(info) = statat(parent, name, AtFlags::SYMLINK_NOFOLLOW) + { + let kind = FileType::from_raw_mode(info.st_mode); + if !matches!(kind, FileType::RegularFile | FileType::Directory) { + return Err(io::Error::other( + "native media path must not contain symlinks or special files", + )); + } + } + return Err(error.into()); + } + }; + if last { + let file = std::fs::File::from(fd); + let metadata = file.metadata()?; + if metadata.is_file() { + return Ok(File::from_std(file)); + } + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + if metadata.is_dir() { + "not a regular file" + } else { + "native media path must not contain symlinks or special files" + }, + )); + } + directory = Some(fd); + } + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "empty native path", + )) + } + + #[cfg(not(unix))] + pub(crate) fn open_native_read(&self) -> io::Result<File> { + self.native_display().map_err(io::Error::other)?; + self.open(OpenOptions::new().read(true)) + } + + pub fn remove(&self) -> io::Result<()> { + let info = self.root.directory.symlink_metadata(&self.relative)?; + if info.is_dir() { + self.root.directory.remove_dir(&self.relative) + } else { + self.root.directory.remove_file(&self.relative) + } + } + + pub fn rename_to(&self, target: &Self) -> io::Result<()> { + self.root + .directory + .rename(&self.relative, &target.root.directory, &target.relative) + } + + pub fn stage(&self) -> io::Result<StagedFile> { + self.stage_with_parent(true, None).map(|(stage, _)| stage) + } + + pub(crate) fn stage_existing_parent(&self) -> io::Result<StagedFile> { + self.stage_with_parent(false, None).map(|(stage, _)| stage) + } + + /// Link a completed private provider file into staging without copying its + /// bytes. Verify the link against the caller's open descriptor before use. + pub(crate) fn stage_from(&self, input: &Self, source: &File) -> io::Result<(StagedFile, bool)> { + self.stage_with_parent(true, Some((input, source))) + } + + fn stage_with_parent( + &self, + create_parent: bool, + source: Option<(&Self, &File)>, + ) -> io::Result<(StagedFile, bool)> { + use aes_gcm::aead::{OsRng, rand_core::RngCore}; + if create_parent { + self.mkdir_parent()?; + } + let parent = self.root.directory.open_dir(self.parent())?; + let target = self + .relative + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid output filename"))? + .to_owned(); + for _ in 0..8 { + let mut nonce = [0; 16]; + OsRng + .try_fill_bytes(&mut nonce) + .map_err(|error| io::Error::other(error.to_string()))?; + let name = format!( + ".spotiflac-{}.partial", + crate::binary::encode(&nonce, "hex").expect("hex encoding") + ); + #[cfg(unix)] + if let Some((input, source)) = source { + use cap_std::fs::MetadataExt; + match input + .root + .directory + .hard_link(&input.relative, &parent, &name) + { + Ok(()) => { + let linked = rustix::fs::openat( + &parent, + &name, + rustix::fs::OFlags::RDONLY + | rustix::fs::OFlags::NOFOLLOW + | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + ) + .map(|fd| File::from_std(std::fs::File::from(fd))) + .map_err(io::Error::from); + let linked = linked.and_then(|file| { + let actual = file.metadata()?; + let expected = source.metadata()?; + if actual.is_file() + && actual.dev() == expected.dev() + && actual.ino() == expected.ino() + { + Ok(file) + } else { + Err(io::Error::other( + "provider source changed before publication", + )) + } + }); + if let Ok(file) = linked { + return Ok(( + StagedFile { + parent, + name, + target, + file, + published: false, + }, + true, + )); + } + parent.remove_file(&name)?; + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + // Cross-device and filesystems without hard links retain + // the descriptor-based copy path below. + Err(_) => {} + } + } + #[cfg(not(unix))] + let _ = source; + let mut options = OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(unix)] + { + use cap_std::fs::OpenOptionsExt; + options.mode(0o644); + } + match parent.open_with(&name, &options) { + Ok(file) => { + return Ok(( + StagedFile { + parent, + name, + target, + file, + published: false, + }, + false, + )); + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + } + Err(io::Error::other("unable to allocate staged output")) + } +} + +pub(crate) struct StagedFile { + parent: Dir, + name: String, + target: std::ffi::OsString, + pub file: File, + published: bool, +} + +impl StagedFile { + /// Album resolution must not replace a file created by another publisher + /// after the planner checked the destination. Both names are in this parent. + pub fn publish_new(mut self, check: &dyn Fn() -> Result<(), String>) -> io::Result<()> { + self.file.sync_all()?; + check().map_err(io::Error::other)?; + #[cfg(any(target_vendor = "apple", target_os = "linux", target_os = "android"))] + rustix::fs::renameat_with( + &self.parent, + &self.name, + &self.parent, + &self.target, + rustix::fs::RenameFlags::NOREPLACE, + )?; + #[cfg(not(any(target_vendor = "apple", target_os = "linux", target_os = "android")))] + { + self.parent + .hard_link(&self.name, &self.parent, &self.target)?; + let _ = self.parent.remove_file(&self.name); + } + self.published = true; + let _ = self + .parent + .try_clone() + .and_then(|parent| parent.into_std_file().sync_all()); + Ok(()) + } + + pub fn publish(mut self, check: &dyn Fn() -> Result<(), String>) -> Result<(), String> { + self.file.sync_all().map_err(|error| error.to_string())?; + check()?; + self.parent + .rename(&self.name, &self.parent, &self.target) + .map_err(|error| error.to_string())?; + self.published = true; + // Match native tag/download publication durability where the filesystem + // supports directory fsync. Unsupported directory sync is best-effort. + let _ = self + .parent + .try_clone() + .and_then(|parent| parent.into_std_file().sync_all()); + Ok(()) + } +} + +impl Drop for StagedFile { + fn drop(&mut self) { + if !self.published { + let _ = self.parent.remove_file(&self.name); + } + } +} + +#[derive(Default)] +struct OutputLock { + owner: Mutex<Option<ThreadId>>, + idle: Condvar, +} + +pub(crate) struct OutputGuard(Arc<OutputLock>); + +impl Drop for OutputGuard { + fn drop(&mut self) { + *self.0.owner.lock().expect("file owner lock") = None; + self.0.idle.notify_all(); + } +} + +pub(crate) fn clean(path: &Path) -> PathBuf { + let mut result = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + if result.file_name().is_some_and(|name| name != "..") { + result.pop(); + } else if !result.has_root() { + result.push(".."); + } + } + _ => result.push(component), + } + } + if result.as_os_str().is_empty() { + ".".into() + } else { + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Seek, SeekFrom, Write}; + + #[cfg(unix)] + #[test] + fn native_reads_reject_links_at_every_component_and_never_write_or_wait_on_special_files() { + use std::io::Read; + use std::os::unix::{fs::symlink, net::UnixListener}; + + let root = tempfile::tempdir().unwrap(); + let files = Arc::new(FileRegistry::default()) + .extension(root.path()) + .unwrap(); + std::fs::create_dir_all(root.path().join("album/disc")).unwrap(); + std::fs::write(root.path().join("album/disc/song.wav"), b"original").unwrap(); + #[cfg(any(target_os = "linux", target_os = "android"))] + { + use std::os::unix::fs::PermissionsExt; + let parent = root.path().join("album"); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o111)).unwrap(); + let result = files + .resolve("album/disc/song.wav") + .unwrap() + .open_native_read(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); + assert!(result.is_ok(), "execute-only parent: {result:?}"); + } + let mut file = files + .resolve("album/disc/song.wav") + .unwrap() + .open_native_read() + .unwrap(); + let mut data = Vec::new(); + file.read_to_end(&mut data).unwrap(); + assert_eq!(data, b"original"); + assert!(file.write_all(b"changed").is_err()); + for (target, link) in [ + ("album", "album-link"), + ("album/disc/song.wav", "song-link.wav"), + ("missing.wav", "dangling.wav"), + ] { + symlink(target, root.path().join(link)).unwrap(); + } + for path in ["album-link/disc/song.wav", "song-link.wav", "dangling.wav"] { + assert!( + files + .resolve(path) + .unwrap() + .open_native_read() + .unwrap_err() + .to_string() + .contains("symlinks") + ); + } + assert_eq!( + files + .resolve("missing.wav") + .unwrap() + .open_native_read() + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); + assert!( + std::process::Command::new("mkfifo") + .arg(root.path().join("pipe.wav")) + .status() + .unwrap() + .success() + ); + let _socket = UnixListener::bind(root.path().join("socket.wav")).unwrap(); + for path in ["pipe.wav", "socket.wav"] { + assert!( + files + .resolve(path) + .unwrap() + .open_native_read() + .unwrap_err() + .to_string() + .contains("special files") + ); + } + assert_eq!( + std::fs::read(root.path().join("album/disc/song.wav")).unwrap(), + b"original" + ); + } + + #[test] + fn exclusive_publication_preserves_racing_output_and_cleans_cancelled_stage() { + let root = tempfile::tempdir().unwrap(); + let files = Arc::new(FileRegistry::default()) + .extension(root.path()) + .unwrap(); + let path = files.resolve("track.flac").unwrap(); + let mut stage = path.stage().unwrap(); + stage.file.write_all(b"new audio").unwrap(); + assert!(!path.absolute.exists()); + let error = stage + .publish_new(&|| { + // A different publisher creates the destination after the + // planner's existence check but before the publication syscall. + std::fs::write(&path.absolute, b"other publisher").unwrap(); + Ok(()) + }) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(std::fs::read(&path.absolute).unwrap(), b"other publisher"); + + let cancelled = files.resolve("cancelled.flac").unwrap(); + let mut stage = cancelled.stage().unwrap(); + stage.file.write_all(b"cancelled audio").unwrap(); + assert_eq!( + stage + .publish_new(&|| Err("cancelled".into())) + .unwrap_err() + .to_string(), + "cancelled" + ); + assert!(!cancelled.absolute.exists()); + + path.remove().unwrap(); + let mut stage = path.stage().unwrap(); + stage.file.write_all(b"complete audio").unwrap(); + stage.publish_new(&|| Ok(())).unwrap(); + assert_eq!(std::fs::read(&path.absolute).unwrap(), b"complete audio"); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 1); + } + + #[cfg(unix)] + #[test] + fn stage_from_links_same_filesystem_source_and_survives_source_cleanup() { + use cap_std::fs::MetadataExt; + + let root = tempfile::tempdir().unwrap(); + let files = Arc::new(FileRegistry::default()) + .extension(root.path()) + .unwrap(); + let source_path = files.resolve("source.bin").unwrap(); + std::fs::write(&source_path.absolute, b"source audio").unwrap(); + let source = source_path.open(OpenOptions::new().read(true)).unwrap(); + let destination = files.resolve("destination.bin").unwrap(); + let (stage, promoted) = destination.stage_from(&source_path, &source).unwrap(); + assert!(promoted); + let source_metadata = source.metadata().unwrap(); + let stage_metadata = stage.file.metadata().unwrap(); + assert_eq!(source_metadata.dev(), stage_metadata.dev()); + assert_eq!(source_metadata.ino(), stage_metadata.ino()); + + stage.publish_new(&|| Ok(())).unwrap(); + drop(source); + source_path.remove().unwrap(); + assert_eq!( + std::fs::read(&destination.absolute).unwrap(), + b"source audio" + ); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 1); + } + + #[cfg(unix)] + #[test] + fn stage_from_copies_open_source_after_path_replacement() { + let root = tempfile::tempdir().unwrap(); + let files = Arc::new(FileRegistry::default()) + .extension(root.path()) + .unwrap(); + let source_path = files.resolve("source.bin").unwrap(); + std::fs::write(&source_path.absolute, b"original audio").unwrap(); + let mut source = source_path.open(OpenOptions::new().read(true)).unwrap(); + std::fs::rename(&source_path.absolute, root.path().join("source.old")).unwrap(); + std::fs::write(&source_path.absolute, b"replacement audio").unwrap(); + let destination = files.resolve("destination.bin").unwrap(); + let (mut stage, promoted) = destination.stage_from(&source_path, &source).unwrap(); + assert!(!promoted); + source.seek(SeekFrom::Start(0)).unwrap(); + std::io::copy(&mut source, &mut stage.file).unwrap(); + stage.publish_new(&|| Ok(())).unwrap(); + assert_eq!( + std::fs::read(&destination.absolute).unwrap(), + b"original audio" + ); + } + + #[cfg(unix)] + #[test] + fn stage_from_copies_open_source_after_path_becomes_symlink() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let files = Arc::new(FileRegistry::default()) + .extension(root.path()) + .unwrap(); + let source_path = files.resolve("source.bin").unwrap(); + std::fs::write(&source_path.absolute, b"original audio").unwrap(); + let mut source = source_path.open(OpenOptions::new().read(true)).unwrap(); + std::fs::rename(&source_path.absolute, root.path().join("source.old")).unwrap(); + symlink("source.old", &source_path.absolute).unwrap(); + let destination = files.resolve("destination.bin").unwrap(); + let (mut stage, promoted) = destination.stage_from(&source_path, &source).unwrap(); + assert!(!promoted); + source.seek(SeekFrom::Start(0)).unwrap(); + std::io::copy(&mut source, &mut stage.file).unwrap(); + stage.publish_new(&|| Ok(())).unwrap(); + assert_eq!( + std::fs::read(&destination.absolute).unwrap(), + b"original audio" + ); + } + + #[cfg(unix)] + #[test] + fn stage_from_preserves_existing_destination_and_cleans_cancelled_stage() { + let root = tempfile::tempdir().unwrap(); + let files = Arc::new(FileRegistry::default()) + .extension(root.path()) + .unwrap(); + let source_path = files.resolve("source.bin").unwrap(); + std::fs::write(&source_path.absolute, b"source audio").unwrap(); + let source = source_path.open(OpenOptions::new().read(true)).unwrap(); + let destination = files.resolve("destination.bin").unwrap(); + std::fs::write(&destination.absolute, b"existing audio").unwrap(); + + let (stage, promoted) = destination.stage_from(&source_path, &source).unwrap(); + assert!(promoted); + let error = stage.publish_new(&|| Ok(())).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!( + std::fs::read(&destination.absolute).unwrap(), + b"existing audio" + ); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 2); + + std::fs::remove_file(&destination.absolute).unwrap(); + let (stage, promoted) = destination.stage_from(&source_path, &source).unwrap(); + assert!(promoted); + let error = stage.publish_new(&|| Err("cancelled".into())).unwrap_err(); + assert_eq!(error.to_string(), "cancelled"); + assert!(!destination.absolute.exists()); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 1); + } +} diff --git a/rust_backend/crates/extensions/src/files/index.rs b/rust_backend/crates/extensions/src/files/index.rs new file mode 100644 index 00000000..c8721a1d --- /dev/null +++ b/rust_backend/crates/extensions/src/files/index.rs @@ -0,0 +1,91 @@ +use super::{ExtensionFiles, clean}; +use cap_std::fs::{Metadata, OpenOptions}; +use spotiflac_core::isrc::{FileStamp, IndexFiles}; +use std::time::UNIX_EPOCH; + +impl IndexFiles for ExtensionFiles { + fn list( + &self, + directory: &str, + check: &(dyn Fn() -> Result<(), String> + Sync), + ) -> Result<Vec<FileStamp>, String> { + let directory = self.resolve_legacy(directory)?; + let mut pending = vec![(directory.relative.clone(), directory.absolute.clone())]; + let mut result = Vec::new(); + while let Some((relative, absolute)) = pending.pop() { + check()?; + let Ok(metadata) = directory.root.directory.symlink_metadata(&relative) else { + continue; + }; + if !metadata.is_dir() { + let path = absolute.to_str().ok_or("invalid UTF-8 path")?; + result.push(stamp(path, &metadata)); + continue; + } + let Ok(opened) = directory.root.directory.open_dir(&relative) else { + continue; + }; + let Ok(entries) = opened.entries() else { + continue; + }; + let mut names = Vec::new(); + for entry in entries { + check()?; + if let Ok(entry) = entry { + names.push(entry.file_name()); + } + } + names.sort(); + pending.extend( + names + .into_iter() + .rev() + .map(|name| (relative.join(&name), clean(&absolute.join(name)))), + ); + } + check()?; + Ok(result) + } + + fn read( + &self, + path: &str, + check: &(dyn Fn() -> Result<(), String> + Sync), + ) -> Result<String, String> { + check()?; + let resolved = self.resolve_legacy(path)?; + let Ok(mut file) = resolved.open(OpenOptions::new().read(true)) else { + return Ok(String::new()); + }; + let format = std::path::Path::new(path) + .extension() + .and_then(|extension| extension.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + spotiflac_core::isrc::read_isrc(&mut file, &format, check) + } + + fn stat(&self, path: &str) -> Result<Option<FileStamp>, String> { + let path = self.resolve_legacy(path)?; + Ok(path + .metadata() + .ok() + .map(|metadata| stamp(&path.display(), &metadata))) + } +} + +fn stamp(path: &str, metadata: &Metadata) -> FileStamp { + let modified_ns = metadata + .modified() + .map(|time| match time.into_std().duration_since(UNIX_EPOCH) { + Ok(duration) => duration.as_nanos() as i128, + Err(error) => -(error.duration().as_nanos() as i128), + }) + .unwrap_or_default(); + FileStamp { + path: path.into(), + size: metadata.len(), + modified_ns, + directory: metadata.is_dir(), + } +} diff --git a/rust_backend/crates/extensions/src/host.rs b/rust_backend/crates/extensions/src/host.rs new file mode 100644 index 00000000..710593b3 --- /dev/null +++ b/rust_backend/crates/extensions/src/host.rs @@ -0,0 +1,270 @@ +use crate::runtime::{Control, ExtensionServices}; +use crate::storage::{StorageError, StoreKind}; +use base64::Engine; +use base64::alphabet; +use base64::engine::general_purpose::{GeneralPurpose, GeneralPurposeConfig, STANDARD}; +use hmac::{Hmac, Mac}; +use md5::Md5; +use rquickjs::{Ctx, Function, Object}; +use sha1::Sha1; +use sha2::{Digest, Sha256}; +use std::sync::Arc; +use std::time::Duration; + +pub(crate) fn register<'js>( + ctx: &Ctx<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<Object<'js>> { + let host = Object::new(ctx.clone())?; + crate::utility_host::register(ctx, &host, Arc::clone(&control), services)?; + crate::legacy_host::register(ctx, &host, Arc::clone(&control), services)?; + let item_control = Arc::clone(&control); + host.set( + "downloadItemActive", + Function::new(ctx.clone(), move || !item_control.item_id().is_empty())?, + )?; + let status_control = Arc::clone(&control); + let downloads = Arc::clone(&services.downloads); + host.set( + "downloadStatus", + Function::new(ctx.clone(), move |status: String| { + let id = status_control.item_id(); + if id.is_empty() { + return; + } + let _ = match status.trim().to_lowercase().as_str() { + "preparing" => downloads.progress.preparing(&id, ""), + "downloading" => downloads.progress.downloading(&id), + "finalizing" => downloads.progress.finalizing(&id), + _ => Ok(()), + }; + })?, + )?; + let resolution_control = Arc::clone(&control); + host.set( + "resolutionRemaining", + Function::new(ctx.clone(), move || { + resolution_control + .resolution() + .map_or(60_000, |budget| budget.remaining().as_millis() as u64) + })?, + )?; + crate::crypto_host::register(ctx, &host, Arc::clone(&control))?; + crate::file_host::register(ctx, &host, Arc::clone(&control), services)?; + crate::ffmpeg_host::register(ctx, &host, Arc::clone(&control), services)?; + crate::download_host::register(ctx, &host, Arc::clone(&control), services)?; + crate::network_host::register(ctx, &host, Arc::clone(&control), services)?; + crate::session_host::register(ctx, &host, Arc::clone(&control), services)?; + host.set("authEnabled", services.auth.is_some())?; + if let Some(auth) = &services.auth { + let auth = Arc::clone(auth); + let control = Arc::clone(&control); + host.set( + "authCall", + Function::new( + ctx.clone(), + move |method: String, arguments: String, expires_is_float: bool| { + let arguments = zeroize::Zeroizing::new(arguments); + let result = serde_json::from_str::<Vec<serde_json::Value>>(&arguments) + .map_err(|error| error.to_string()) + .and_then(|arguments| { + auth.call(&method, &arguments, expires_is_float, || { + control.check().map_err(|error| error.to_string()) + }) + }); + match result { + Ok(value) => value, + Err(error) if method == "generatePKCE" => { + serde_json::json!({"error":error}) + } + Err(error) => serde_json::json!({"success":false,"error":error}), + } + .to_string() + }, + )?, + )?; + } + host.set("storageEnabled", services.storage.is_some())?; + if let Some(store) = &services.storage { + let reader = Arc::clone(store); + host.set( + "storageRead", + Function::new(ctx.clone(), move |credentials: bool, key: String| { + let kind = if credentials { + StoreKind::Credentials + } else { + StoreKind::Storage + }; + match reader.get(kind, &key) { + Ok(Some(value)) => serde_json::json!({"found":true,"value":value}), + Ok(None) => serde_json::json!({"found":false}), + Err(error) => serde_json::json!({"error":error.to_string()}), + } + .to_string() + })?, + )?; + let writer = Arc::clone(store); + host.set( + "storageWrite", + Function::new( + ctx.clone(), + move |credentials: bool, key: String, value: String| { + let kind = if credentials { + StoreKind::Credentials + } else { + StoreKind::Storage + }; + let result = serde_json::from_str(&value) + .map_err(StorageError::from) + .and_then(|value| writer.set(kind, &key, value)); + storage_result(result) + }, + )?, + )?; + let remover = Arc::clone(store); + host.set( + "storageRemove", + Function::new(ctx.clone(), move |credentials: bool, key: String| { + let kind = if credentials { + StoreKind::Credentials + } else { + StoreKind::Storage + }; + storage_result(remover.remove(kind, &key)) + })?, + )?; + } + host.set( + "encodeBytes", + Function::new(ctx.clone(), |value: Vec<u8>| STANDARD.encode(value))?, + )?; + host.set( + "base64Encode", + Function::new(ctx.clone(), |value: String| STANDARD.encode(value))?, + )?; + host.set( + "base64Decode", + Function::new(ctx.clone(), |value: String, url_safe: bool| { + // Go accepts nonzero padding bits and ignores CR/LF, but requires padding. + let value = value.replace(['\r', '\n'], ""); + let config = GeneralPurposeConfig::new().with_decode_allow_trailing_bits(true); + let decoded = GeneralPurpose::new(&alphabet::STANDARD, config) + .decode(&value) + .or_else(|error| { + if url_safe { + GeneralPurpose::new(&alphabet::URL_SAFE, config).decode(&value) + } else { + Err(error) + } + }) + .unwrap_or_default(); + decode_go_utf8(&decoded) + })?, + )?; + host.set( + "encode", + Function::new(ctx.clone(), |value: String| value.into_bytes())?, + )?; + host.set( + "decode", + Function::new(ctx.clone(), |value: Vec<u8>| decode_go_utf8(&value))?, + )?; + host.set( + "md5", + Function::new(ctx.clone(), |value: String| { + format!("{:x}", Md5::digest(value)) + })?, + )?; + host.set( + "sha256", + Function::new(ctx.clone(), |value: String| { + format!("{:x}", Sha256::digest(value)) + })?, + )?; + host.set( + "hmacSHA256", + Function::new(ctx.clone(), |message: String, key: String| { + let mut mac = Hmac::<Sha256>::new_from_slice(key.as_bytes()) + .expect("HMAC accepts any key length"); + mac.update(message.as_bytes()); + format!("{:x}", mac.finalize().into_bytes()) + })?, + )?; + host.set( + "hmacSHA256Base64", + Function::new(ctx.clone(), |message: String, key: String| { + let mut mac = Hmac::<Sha256>::new_from_slice(key.as_bytes()) + .expect("HMAC accepts any key length"); + mac.update(message.as_bytes()); + STANDARD.encode(mac.finalize().into_bytes()) + })?, + )?; + host.set( + "hmacSHA1", + Function::new(ctx.clone(), |key: Vec<u8>, message: Vec<u8>| { + let mut mac = Hmac::<Sha1>::new_from_slice(&key).expect("HMAC accepts any key length"); + mac.update(&message); + mac.finalize().into_bytes().to_vec() + })?, + )?; + host.set( + "quoteJSON", + Function::new(ctx.clone(), |value: String| { + // Go encoding/json escapes HTML and JS separators in strings. + serde_json::to_string(&value) + .expect("JSON string encoding") + .replace('&', "\\u0026") + .replace('<', "\\u003c") + .replace('>', "\\u003e") + .replace('\u{2028}', "\\u2028") + .replace('\u{2029}', "\\u2029") + })?, + )?; + host.set( + "sortKeys", + Function::new(ctx.clone(), |mut keys: Vec<String>| { + // UTF-8/code-point ordering, rather than JS sort's UTF-16 ordering. + keys.sort(); + keys + })?, + )?; + let sleep_control = Arc::clone(&control); + host.set( + "sleep", + Function::new(ctx.clone(), move |milliseconds: u64| { + sleep_control.sleep(Duration::from_millis(milliseconds.min(300_000))) + })?, + )?; + Ok(host) +} + +fn storage_result(result: Result<(), StorageError>) -> String { + match result { + Ok(()) => serde_json::json!({"success":true}), + Err(error) => serde_json::json!({"success":false,"error":error.to_string()}), + } + .to_string() +} + +// Go's UTF-8 decoder consumes one byte for every invalid rune. Rust's lossy +// conversion groups some invalid prefixes, which would change extension text. +pub(crate) fn decode_go_utf8(mut bytes: &[u8]) -> String { + let mut decoded = String::with_capacity(bytes.len()); + while !bytes.is_empty() { + match std::str::from_utf8(bytes) { + Ok(valid) => { + decoded.push_str(valid); + break; + } + Err(error) => { + let prefix = error.valid_up_to(); + decoded + .push_str(std::str::from_utf8(&bytes[..prefix]).expect("valid UTF-8 prefix")); + decoded.push('\u{fffd}'); + bytes = &bytes[prefix + 1..]; + } + } + } + decoded +} diff --git a/rust_backend/crates/extensions/src/legacy_host.rs b/rust_backend/crates/extensions/src/legacy_host.rs new file mode 100644 index 00000000..b1646e42 --- /dev/null +++ b/rust_backend/crates/extensions/src/legacy_host.rs @@ -0,0 +1,52 @@ +use crate::runtime::{Control, ExtensionServices}; +use rquickjs::{Ctx, Exception, Function, Object}; +use std::sync::Arc; + +pub(crate) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + host.set("legacyBackend", services.legacy_backend)?; + if !services.legacy_backend { + return Ok(()); + } + files::register(ctx, host, Arc::clone(&control), services)?; + lyrics::register(ctx, host, Arc::clone(&control), services)?; + host.set( + "legacySanitize", + Function::new(ctx.clone(), |value: String| { + spotiflac_core::filename::sanitize_filename(&value) + })?, + )?; + host.set( + "legacyFilename", + Function::new( + ctx.clone(), + move |ctx: Ctx<'js>, template: String, metadata: String| { + let check = || control.check().map_err(|error| error.to_string()); + let result = serde_json::from_str(&metadata) + .map_err(|error| error.to_string()) + .and_then(|metadata| { + spotiflac_core::filename::build_filename_checked( + &template, + &metadata, + 8 * 1024 * 1024, + &check, + ) + }); + result.map_err(|error| Exception::throw_message(&ctx, &error)) + }, + )?, + )?; + host.set( + "legacyLocalTime", + Function::new(ctx.clone(), || { + serde_json::to_string(&spotiflac_core::clock::local_time()).expect("local time JSON") + })?, + )?; + Ok(()) +} +mod files; +mod lyrics; diff --git a/rust_backend/crates/extensions/src/legacy_host/files.rs b/rust_backend/crates/extensions/src/legacy_host/files.rs new file mode 100644 index 00000000..16bbc52b --- /dev/null +++ b/rust_backend/crates/extensions/src/legacy_host/files.rs @@ -0,0 +1,79 @@ +use crate::runtime::{Control, ExtensionServices}; +use cap_std::fs::OpenOptions; +use rquickjs::{Ctx, Function, Object}; +use serde_json::json; +use std::sync::Arc; + +pub(super) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + let files = services + .legacy_files + .clone() + .or_else(|| services.files.clone()); + let quality_files = files.clone(); + let quality_control = Arc::clone(&control); + host.set( + "legacyQuality", + Function::new(ctx.clone(), move |path: String| { + let check = || quality_control.check().map_err(|error| error.to_string()); + let result = (|| { + check()?; + let files = quality_files.as_ref().ok_or("file access unavailable")?; + let path = files.resolve_legacy(&path)?; + let mut file = path + .open(OpenOptions::new().read(true)) + .map_err(|error| format!("failed to open file: {error}"))?; + spotiflac_core::media::probe_quality(&mut file, &check) + })(); + match result { + Ok(quality) => json!({ + "bitDepth":quality.bit_depth,"sampleRate":quality.sample_rate, + "totalSamples":quality.total_samples,"duration":quality.duration, + "codec":quality.codec + }), + Err(error) => json!({"error":error}), + } + .to_string() + })?, + )?; + let cache = Arc::clone(&services.isrc); + host.set( + "legacyIsrc", + Function::new( + ctx.clone(), + move |add: bool, directory: String, isrc: String, path: String| { + let check = || control.check().map_err(|error| error.to_string()); + let (directory, isrc, path) = (directory.trim(), isrc.trim(), path.trim()); + let result = (|| { + if directory.is_empty() || isrc.is_empty() || (add && path.is_empty()) { + return Err(if add { + "outputDir, isrc, and filePath are required" + } else { + "outputDir and isrc are required" + } + .to_owned()); + } + check()?; + let files = files.as_ref().ok_or("file access unavailable")?; + let directory = files.resolve_legacy(directory)?.display(); + if add { + let path = files.resolve_legacy(path)?.display(); + cache.add(&directory, isrc, &path, files.as_ref(), &check)?; + Ok(json!({"success":true})) + } else { + let path = cache.check(&directory, isrc, files.as_ref(), &check)?; + Ok(json!({"exists":!path.is_empty(),"filePath":path})) + } + })(); + result + .unwrap_or_else(|error| json!({"error":error})) + .to_string() + }, + )?, + )?; + Ok(()) +} diff --git a/rust_backend/crates/extensions/src/legacy_host/lyrics.rs b/rust_backend/crates/extensions/src/legacy_host/lyrics.rs new file mode 100644 index 00000000..96c93b15 --- /dev/null +++ b/rust_backend/crates/extensions/src/legacy_host/lyrics.rs @@ -0,0 +1,131 @@ +use crate::runtime::{Control, ExtensionServices}; +use cap_std::fs::OpenOptions; +use rquickjs::{Ctx, Function, Object}; +use serde_json::json; +use spotiflac_core::lyrics::{file, lrc}; +use spotiflac_providers::lyrics::SearchRequest; +use std::sync::Arc; + +pub(super) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + let lyrics = services.lyrics.clone(); + let node = services.lyrics_node.clone(); + let files = services + .legacy_files + .clone() + .or_else(|| services.files.clone()); + host.set( + "legacyLyrics", + Function::new( + ctx.clone(), + move |spotify_id: String, + track: String, + artist: String, + path: String, + duration: f64, + duration_text: Option<String>, + number_conversion: bool| { + let check = || control.check().map_err(|error| error.to_string()); + let result: Result<String, String> = (|| { + check()?; + let (spotify_id, track, artist, path) = + (spotify_id.trim(), track.trim(), artist.trim(), path.trim()); + if !path.is_empty() { + let result = file::extract( + path, + &|path| { + files + .as_ref() + .ok_or("file access unavailable")? + .resolve_legacy(path)? + .open(OpenOptions::new().read(true)) + .map_err(|error| error.to_string()) + }, + &check, + ); + check()?; + return Ok(result + .ok() + .filter(|text| lrc::has_usable_content(text)) + .unwrap_or_default()); + } + let lyrics = lyrics.upgrade().ok_or("lyrics service unavailable")?; + let response = lyrics + .fetch( + SearchRequest { + spotify_id: spotify_id.into(), + track: track.into(), + artist: artist.into(), + duration: duration_ms( + duration, + duration_text.as_deref(), + number_conversion, + ) as f64 + / 1000.0, + caller: Some(node.clone()), + ..SearchRequest::default() + }, + &check, + ) + .map_err(|error| error.to_string())?; + check()?; + Ok(if response.instrumental { + "[instrumental:true]".into() + } else { + lrc::with_metadata(&response, track, artist) + }) + })(); + match result { + Ok(text) => json!({"lyrics":text}), + Err(error) => json!({"error":error}), + } + .to_string() + }, + )?, + )?; + Ok(()) +} + +fn duration_ms(number: f64, text: Option<&str>, number_conversion: bool) -> i64 { + let Some(text) = text else { + return number as i64; + }; + // Goja's primitive unicodeString.ToInteger returns zero. Object coercion + // first calls ToNumber, which trims ECMAScript whitespace before Go space. + if !number_conversion && !text.is_ascii() { + return 0; + } + let text = text.trim(); + let (digits, radix) = match text.get(..2) { + Some("0x" | "0X") => (&text[2..], 16), + Some("0b" | "0B") => (&text[2..], 2), + Some("0o" | "0O") => (&text[2..], 8), + _ => (text, 10), + }; + match i64::from_str_radix(digits, radix) { + Ok(value) => value, + Err(_) if radix != 10 && number_conversion => 0, + Err(error) if radix != 10 => match error.kind() { + std::num::IntErrorKind::PosOverflow => i64::MAX, + std::num::IntErrorKind::NegOverflow => i64::MIN, + _ => 0, + }, + _ => { + // Preserve Go's decimal parsing and reject non-ECMAScript inf names. + let parsed = if text.contains('_') { + None + } else { + text.parse::<f64>().ok().filter(|value| { + !value.is_infinite() + || matches!(text, "Infinity" | "+Infinity" | "-Infinity") + || text.contains(['e', 'E']) + }) + }; + parsed.unwrap_or(0.0) as i64 + } + } +} diff --git a/rust_backend/crates/extensions/src/lib.rs b/rust_backend/crates/extensions/src/lib.rs new file mode 100644 index 00000000..fc691d8c --- /dev/null +++ b/rust_backend/crates/extensions/src/lib.rs @@ -0,0 +1,35 @@ +//! Extension JavaScript execution and host services, isolated from mobile UI code. + +pub mod auth; +pub mod backend; +mod binary; +pub mod crypto; +mod crypto_host; +mod download; +mod download_host; +pub mod environment; +pub mod ffmpeg; +mod ffmpeg_host; +mod file_host; +mod file_transform; +pub mod files; +mod host; +mod legacy_host; +pub mod logging; +pub mod lyrics; +pub mod manager; +pub mod manifest; +mod network_host; +pub mod package; +mod provider; +mod redact; +pub mod repository; +mod resolution; +mod runtime; +mod session_host; +pub mod signed_session; +pub mod storage; +pub mod transfer_policy; +mod utility_host; + +pub use runtime::{ExtensionError, ExtensionRuntime, ExtensionServices, RuntimeLimits}; diff --git a/rust_backend/crates/extensions/src/logging.rs b/rust_backend/crates/extensions/src/logging.rs new file mode 100644 index 00000000..f1f21a9f --- /dev/null +++ b/rust_backend/crates/extensions/src/logging.rs @@ -0,0 +1,220 @@ +//! Process-style diagnostic buffer scoped to the native backend environment. + +use crate::host::decode_go_utf8; +use crate::redact::sanitize_bytes; +use serde::Serialize; +use std::collections::VecDeque; +use std::sync::Mutex; + +const CAPACITY: usize = 500; +const MAX_MESSAGE: usize = 4000; + +#[derive(Clone, Debug, Serialize)] +pub struct LogEntry { + pub timestamp: String, + pub level: String, + pub tag: String, + pub message: String, +} + +#[derive(Default)] +struct State { + entries: VecDeque<LogEntry>, + next_index: i64, + enabled: bool, + closed: bool, +} + +#[derive(Default)] +pub struct LogBuffer(Mutex<State>); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("log buffer closed")] +pub struct LogClosed; + +impl State { + fn check(&self) -> Result<(), LogClosed> { + if self.closed { Err(LogClosed) } else { Ok(()) } + } +} + +impl LogBuffer { + pub fn set_enabled(&self, enabled: bool) -> Result<(), LogClosed> { + let mut state = self.0.lock().expect("log buffer lock"); + state.check()?; + state.enabled = enabled; + Ok(()) + } + + pub fn is_enabled(&self) -> Result<bool, LogClosed> { + let state = self.0.lock().expect("log buffer lock"); + state.check()?; + Ok(state.enabled) + } + + pub fn add(&self, level: &str, tag: &str, message: &str) -> Result<(), LogClosed> { + self.add_bytes(level, tag, message.as_bytes().to_vec()) + } + + fn add_bytes(&self, level: &str, tag: &str, message: Vec<u8>) -> Result<(), LogClosed> { + let mut state = self.0.lock().expect("log buffer lock"); + state.check()?; + if !state.enabled && level != "ERROR" && level != "FATAL" { + return Ok(()); + } + let message = decode_go_utf8(&truncate(sanitize_bytes(message))); + let entry = LogEntry { + timestamp: chrono::Local::now().format("%H:%M:%S%.3f").to_string(), + level: level.into(), + tag: tag.into(), + message, + }; + if state.entries.len() == CAPACITY { + state.entries.pop_front(); + } + use std::io::Write; + let _ = writeln!( + std::io::stderr().lock(), + "[{}] {}", + entry.tag, + entry.message + ); + state.entries.push_back(entry); + state.next_index = state.next_index.wrapping_add(1); + Ok(()) + } + + pub fn backend(&self, message: &str) -> Result<(), LogClosed> { + self.backend_bytes(message.as_bytes().to_vec()) + } + + fn backend_bytes(&self, mut message: Vec<u8>) -> Result<(), LogClosed> { + if message.last() == Some(&b'\n') { + message.pop(); + } + let mut tag = "Rust".to_owned(); + if message.first() == Some(&b'[') + && let Some(end) = message + .iter() + .position(|byte| *byte == b']') + .filter(|end| *end > 1) + { + tag = decode_go_utf8(&message[1..end]); + message = trim_bytes(&message[end + 1..]); + } + let lowercase = spotiflac_core::matching::lowercase(&decode_go_utf8(&message)); + let level = if lowercase.contains("error") || lowercase.contains("failed") { + "ERROR" + } else if lowercase.contains("warning") || lowercase.contains("warn") { + "WARN" + } else if lowercase.contains("success") || lowercase.contains("match found") { + "INFO" + } else if lowercase.contains("searching") + || lowercase.contains("trying") + || lowercase.contains("found") + { + "DEBUG" + } else { + "INFO" + }; + self.add_bytes(level, &tag, message) + } + + pub(crate) fn extension(&self, id: &str, level: &str, arguments: Vec<String>, total: usize) { + let mut message = Vec::new(); + for (index, argument) in arguments.into_iter().take(8).enumerate() { + if index > 0 { + message.push(b' '); + } + let bytes = argument.as_bytes(); + message.extend_from_slice(&bytes[..bytes.len().min(512)]); + if bytes.len() > 512 { + message.extend_from_slice(b"...[truncated]"); + } + } + if total > 8 { + if !message.is_empty() { + message.push(b' '); + } + message.extend_from_slice(format!("...[{} more args]", total - 8).as_bytes()); + } + let formatted = truncate(sanitize_bytes(message)); + let mut line = if level.is_empty() { + format!("[Extension:{id}] ") + } else { + format!("[Extension:{id}:{level}] ") + } + .into_bytes(); + line.extend(formatted); + line.push(b'\n'); + let _ = self.backend_bytes(line); + } + + pub fn all(&self) -> Result<String, LogClosed> { + let state = self.0.lock().expect("log buffer lock"); + state.check()?; + Ok(serde_json::to_string(&state.entries).expect("log entries JSON")) + } + + pub fn since(&self, index: i64) -> Result<String, LogClosed> { + let state = self.0.lock().expect("log buffer lock"); + state.check()?; + let earliest = state.next_index - state.entries.len() as i64; + let skip = if index >= state.next_index { + state.entries.len() + } else { + (index.max(earliest) - earliest) as usize + }; + let logs: Vec<_> = state.entries.iter().skip(skip).collect(); + Ok(serde_json::json!({"logs":logs,"next_index":state.next_index}).to_string()) + } + + pub fn clear(&self) -> Result<(), LogClosed> { + let mut state = self.0.lock().expect("log buffer lock"); + state.check()?; + state.entries.clear(); + Ok(()) + } + + pub fn count(&self) -> Result<u64, LogClosed> { + let state = self.0.lock().expect("log buffer lock"); + state.check()?; + Ok(state.entries.len() as u64) + } + + pub fn shutdown(&self) { + let mut state = self.0.lock().expect("log buffer lock"); + state.closed = true; + state.entries.clear(); + } +} + +impl Drop for LogBuffer { + fn drop(&mut self) { + self.shutdown(); + } +} + +fn truncate(mut message: Vec<u8>) -> Vec<u8> { + if message.len() > MAX_MESSAGE { + message.truncate(MAX_MESSAGE); + if let Err(error) = std::str::from_utf8(&message) { + message.truncate(error.valid_up_to()); + } + message.extend_from_slice(b"...[truncated]"); + } + message +} + +fn trim_bytes(bytes: &[u8]) -> Vec<u8> { + // Invalid bytes are non-whitespace in Go and are retained until JSON export. + let decoded = decode_go_utf8(bytes); + let left = decoded.len() - decoded.trim_start().len(); + let right = decoded.len() - decoded.trim_end().len(); + let end = bytes.len().saturating_sub(right); + if left >= end { + Vec::new() + } else { + bytes[left..end].to_vec() + } +} diff --git a/rust_backend/crates/extensions/src/lyrics.rs b/rust_backend/crates/extensions/src/lyrics.rs new file mode 100644 index 00000000..ffdd6de8 --- /dev/null +++ b/rust_backend/crates/extensions/src/lyrics.rs @@ -0,0 +1,146 @@ +//! Lyrics service adapter for installed providers and the shared built-in clients. + +use crate::manager::ExtensionManager; +use serde_json::json; +use spotiflac_core::cancellation::{CancellationDomain, CancellationRegistry}; +use spotiflac_core::lyrics::LyricsResponse; +use spotiflac_core::matching::lowercase; +use spotiflac_providers::lyrics::{ + CallGraph, Check, LyricsError, LyricsFetcher, SearchRequest, builtin::BuiltinLyricsClient, +}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Weak, mpsc}; +use std::time::Duration; + +/// The native owner retains the manager and service separately. A weak manager +/// reference allows that owner to keep this adapter without a reference cycle. +/// Lyrics flights register their wait on the manager VM before queueing work. +/// A host call back into the service can then reject recursive dependencies. +pub struct InstalledLyricsFetcher { + manager: Weak<ExtensionManager>, + closed: Arc<AtomicBool>, + builtin: BuiltinLyricsClient, + calls: CallGraph, +} + +impl InstalledLyricsFetcher { + pub fn new(manager: &Arc<ExtensionManager>, builtin: BuiltinLyricsClient) -> Self { + Self { + manager: Arc::downgrade(manager), + closed: manager.environment().closed_flag(), + builtin, + calls: manager.environment().lyrics_calls(), + } + } +} + +impl LyricsFetcher for InstalledLyricsFetcher { + fn call_graph(&self) -> Option<CallGraph> { + Some(self.calls.clone()) + } + + fn check(&self) -> Result<(), LyricsError> { + if self.closed.load(Ordering::Acquire) || self.manager.strong_count() == 0 { + Err(LyricsError::Cancelled("extension manager closed".into())) + } else { + Ok(()) + } + } + + fn extensions(&self) -> Vec<String> { + self.manager + .upgrade() + .and_then(|manager| manager.provider_ids("lyrics_provider").ok()) + .unwrap_or_default() + .into_iter() + .map(|id| lowercase(id.trim())) + .collect() + } + + fn fetch( + &self, + provider: &str, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + check().map_err(LyricsError::Cancelled)?; + let manager = self + .manager + .upgrade() + .ok_or_else(|| LyricsError::Cancelled("extension manager closed".into()))?; + let environment = manager.environment(); + let check = || { + check()?; + if environment.is_closed() { + Err("extension manager closed".into()) + } else { + Ok(()) + } + }; + check().map_err(LyricsError::Cancelled)?; + let Some(id) = provider.strip_prefix("extension:") else { + return self.builtin.fetch(provider, request, &check); + }; + let id = manager + .provider_ids("lyrics_provider") + .map_err(|error| LyricsError::Other(error.to_string()))? + .into_iter() + .find(|candidate| lowercase(candidate.trim()) == id) + .ok_or_else(|| LyricsError::Other(format!("lyrics provider unavailable: {id}")))?; + let arguments = json!([request.track, request.artist, "", request.duration]).to_string(); + let _dependency = request + .caller + .as_ref() + .map(|caller| caller.wait_for(&environment.lyrics_node(&id, false))) + .transpose() + .map_err(LyricsError::Recursive)?; + let cancellation = CancellationRegistry::new(CancellationDomain::ExtensionRequest); + let lease = Arc::new( + cancellation + .acquire("") + .expect("new lyrics cancellation lease"), + ); + let result = std::thread::scope(|scope| { + let (done, completion) = mpsc::channel::<()>(); + let worker_lease = &lease; + let worker = std::thread::Builder::new() + .name("installed-lyrics".into()) + .spawn_scoped(scope, move || { + let _done = done; + manager.provider_call( + &id, + "fetchLyrics", + &arguments, + Some(worker_lease.clone()), + 30_000, + ) + }) + .map_err(|error| LyricsError::Other(error.to_string()))?; + let mut cancelled = None; + loop { + if let Err(message) = check() { + cancelled = Some(message); + lease.release(); + break; + } + if !matches!( + completion.recv_timeout(Duration::from_millis(5)), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + break; + } + } + let result = worker + .join() + .map_err(|_| LyricsError::Other("installed lyrics provider panicked".into()))?; + if let Some(message) = cancelled { + return Err(LyricsError::Cancelled(message)); + } + check().map_err(LyricsError::Cancelled)?; + let response = result.map_err(|error| LyricsError::Other(error.to_string()))?; + serde_json::from_str(&response).map_err(|error| LyricsError::Other(error.to_string())) + }); + lease.release(); + result + } +} diff --git a/rust_backend/crates/extensions/src/manager.rs b/rust_backend/crates/extensions/src/manager.rs new file mode 100644 index 00000000..0f8d00bd --- /dev/null +++ b/rust_backend/crates/extensions/src/manager.rs @@ -0,0 +1,1111 @@ +//! Installed packages and runtime lifecycle belong to one native owner. + +mod downloads; +mod metadata; +mod postprocess; +mod providers; + +pub use downloads::ProviderDownloadRequest; +pub use metadata::ProviderAvailabilityRequest; +pub use postprocess::PostProcessInput; + +use crate::environment::{ + ExtensionEnvironment, SUPPORTED_RUNTIME_FEATURES, compare_versions, validate_gates, +}; +use crate::manifest::ExtensionManifest; +use crate::package::{ExtensionPackage, is_package_path}; +use crate::runtime::{LoadMode, lifecycle_result}; +use crate::{ExtensionError, ExtensionRuntime, RuntimeLimits}; +use serde_json::{Map, Value, json}; +use spotiflac_core::cancellation::RequestLease; +use std::cmp::Ordering as Comparison; +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::Read; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; + +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +pub struct ManagerError(pub String); + +fn error(message: impl Into<String>) -> ManagerError { + ManagerError(message.into()) +} +fn cause(context: &str, error: impl std::fmt::Display) -> ManagerError { + ManagerError(format!("{context}: {error}")) +} + +#[derive(Default)] +struct Status { + enabled: bool, + error: String, +} +#[derive(Default)] +struct Engine { + runtime: Option<Arc<ExtensionRuntime>>, + initialized: bool, + source: Option<Arc<str>>, +} +struct Installed { + manifest: ExtensionManifest, + manifest_json: String, + source: PathBuf, + status: RwLock<Status>, + engine: Mutex<Engine>, + // Pool maintenance must not wait for a primary VM running post-processing. + // Code needing both locks takes engine before downloads. + downloads: Mutex<downloads::Pool>, +} + +struct MetadataChange<'a>(&'a AtomicU64); +impl Drop for MetadataChange<'_> { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::AcqRel); + } +} + +pub struct ExtensionManager { + environment: Arc<ExtensionEnvironment>, + sources: PathBuf, + limits: RuntimeLimits, + entries: Mutex<BTreeMap<String, Arc<Installed>>>, + mutation: Mutex<()>, + closed: AtomicBool, + directory_locks: Mutex<Option<Vec<File>>>, + priorities: Mutex<providers::Priorities>, + metadata_revision: AtomicU64, +} + +impl ExtensionManager { + pub fn new( + sources: &Path, + data: &Path, + master_key: &str, + app_version: &str, + limits: RuntimeLimits, + ) -> Result<Self, ManagerError> { + let environment = ExtensionEnvironment::new(data, master_key, app_version) + .map_err(|e| error(e.to_string()))?; + Self::with_environment(sources, environment, limits) + } + + pub(crate) fn with_environment( + sources: &Path, + environment: ExtensionEnvironment, + limits: RuntimeLimits, + ) -> Result<Self, ManagerError> { + let environment = Arc::new(environment); + environment.attach_manager(); + if sources.as_os_str().is_empty() { + return Err(error("extension directory is not configured")); + } + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(sources) + .map_err(|e| cause("failed to create extensions directory", e))?; + let sources = fs::canonicalize(sources) + .map_err(|e| cause("failed to inspect extensions directory", e))?; + let data = environment.data_directory(); + if sources.starts_with(data) || data.starts_with(&sources) { + return Err(error( + "extension source and data directories must be separate", + )); + } + let mut locks = Vec::new(); + for directory in [&sources, data] { + let lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags( + (rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32, + ) + .open(directory.join(".backend-manager.lock")) + .map_err(|e| cause("failed to lock extension directory", e))?; + if !lock + .metadata() + .map_err(|e| cause("failed to inspect directory lock", e))? + .is_file() + { + return Err(error("extension directory lock must be a regular file")); + } + lock.try_lock() + .map_err(|_| error("extension directories are already managed"))?; + locks.push(lock); + } + Ok(Self { + environment, + sources, + limits, + entries: Mutex::default(), + mutation: Mutex::new(()), + closed: AtomicBool::new(false), + directory_locks: Mutex::new(Some(locks)), + priorities: Mutex::default(), + metadata_revision: AtomicU64::new(0), + }) + } + + pub fn environment(&self) -> Arc<ExtensionEnvironment> { + Arc::clone(&self.environment) + } + + fn check(&self) -> Result<(), ManagerError> { + if self.closed.load(Ordering::Acquire) || self.environment.is_closed() { + Err(error("extension manager closed")) + } else { + Ok(()) + } + } + + fn get(&self, id: &str) -> Result<Arc<Installed>, ManagerError> { + self.check()?; + self.entries + .lock() + .expect("extension manager lock") + .get(id) + .cloned() + .ok_or_else(|| error("extension not found")) + } + + fn current(&self, entry: &Arc<Installed>) -> bool { + !self.closed.load(Ordering::Acquire) + && self + .entries + .lock() + .expect("extension manager lock") + .get(&entry.manifest.name) + .is_some_and(|value| Arc::ptr_eq(value, entry)) + } + + pub(crate) fn health_manifest(&self, id: &str) -> Result<ExtensionManifest, ManagerError> { + // The explicit health API also accepts disabled installed extensions. + Ok(self.get(id)?.manifest.clone()) + } + + pub fn install(&self, path: &Path) -> Result<String, ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let _mutation = self.mutation.lock().expect("extension mutation lock"); + self.check()?; + let package = ExtensionPackage::open(path).map_err(|e| error(e.to_string()))?; + self.install_or_upgrade(package) + } + + fn install_or_upgrade(&self, package: ExtensionPackage) -> Result<String, ManagerError> { + let existing = self + .entries + .lock() + .expect("extension manager lock") + .get(&package.manifest.name) + .cloned(); + if let Some(existing) = existing { + match compare_versions(&package.manifest.version, &existing.manifest.version) { + Comparison::Greater => return self.upgrade_package(package, existing), + Comparison::Equal => { + return Err(error(format!( + "extension '{}' v{} is already installed", + existing.manifest.display_name, existing.manifest.version + ))); + } + Comparison::Less => { + return Err(error(format!( + "cannot downgrade '{}' from v{} to v{}", + existing.manifest.display_name, + existing.manifest.version, + package.manifest.version + ))); + } + } + } + self.install_package(package) + } + + fn stage( + &self, + package: &mut ExtensionPackage, + kind: &str, + ) -> Result<tempfile::TempDir, ManagerError> { + let stage = tempfile::Builder::new() + .prefix(&format!(".{}-{kind}-", package.manifest.name)) + .tempdir_in(&self.sources) + .map_err(|e| cause("failed to create extension staging directory", e))?; + package + .extract(stage.path(), &|| self.check().map_err(|e| e.to_string())) + .map_err(|e| error(e.to_string()))?; + Ok(stage) + } + + fn candidate(&self, package: &ExtensionPackage, path: &Path, enabled: bool) -> Arc<Installed> { + Arc::new(Installed { + manifest: package.manifest.clone(), + manifest_json: package.manifest_json.clone(), + source: path.to_owned(), + status: RwLock::new(Status { + enabled, + error: String::new(), + }), + engine: Mutex::default(), + downloads: Mutex::default(), + }) + } + + fn install_package(&self, mut package: ExtensionPackage) -> Result<String, ManagerError> { + let target = self.sources.join(&package.manifest.name); + if fs::symlink_metadata(&target).is_ok() { + return Err(error(format!( + "extension directory already exists for {:?}", + package.manifest.name + ))); + } + let stage = self.stage(&mut package, "install")?; + let mut candidate = self.candidate(&package, stage.path(), false); + if let Err(error) = self.validate(&candidate) { + self.failed(&candidate, &error); + } + self.check()?; + fs::rename(stage.path(), &target).map_err(|e| cause("failed to activate extension", e))?; + Arc::get_mut(&mut candidate) + .expect("unpublished extension") + .source = target; + self.entries + .lock() + .expect("extension manager lock") + .insert(candidate.manifest.name.clone(), Arc::clone(&candidate)); + Ok(summary(&candidate)) + } + + fn prepare_with_lease( + &self, + entry: &Installed, + lease: Option<Arc<RequestLease>>, + mode: LoadMode, + ) -> Result<(Arc<ExtensionRuntime>, Arc<str>), ManagerError> { + let source: Arc<str> = read_source(&entry.source.join("index.js"), 8 * 1024 * 1024) + .map_err(|e| cause("failed to read index.js", e))? + .into(); + let runtime = self + .environment + .load_registered_with_lease( + &entry.manifest_json, + &source, + self.limits.clone(), + mode, + lease, + ) + .map_err(|error| match error { + crate::environment::EnvironmentError::Runtime(ExtensionError::Script(message)) => { + cause( + if message.starts_with("SyntaxError:") { + "failed to compile extension code" + } else { + "failed to execute extension code" + }, + message, + ) + } + other => self + .check() + .err() + .unwrap_or_else(|| error_from_environment(other)), + })?; + Ok((runtime, source)) + } + + fn validate(&self, entry: &Installed) -> Result<(), ManagerError> { + let (runtime, _) = self.prepare_with_lease(entry, None, LoadMode::Validate)?; + // The worker already ran cleanup; join it before publishing the package. + runtime.shutdown(); + Ok(()) + } + + fn lifecycle( + &self, + runtime: &ExtensionRuntime, + method: &str, + arguments: &str, + ) -> Result<(), ManagerError> { + self.lifecycle_with_lease(runtime, method, arguments, None) + } + + fn lifecycle_with_lease( + &self, + runtime: &ExtensionRuntime, + method: &str, + arguments: &str, + lease: Option<Arc<RequestLease>>, + ) -> Result<(), ManagerError> { + lifecycle_result( + method, + runtime.managed_call_with_lease( + method, + arguments, + false, + self.limits.timeout_ms, + lease, + ), + ) + .map_err(|e| error(e.to_string())) + } + + fn teardown_runtime(&self, runtime: &ExtensionRuntime) { + if let Err(error) = self.lifecycle(runtime, "cleanup", "[]") { + let _ = self + .environment + .log_buffer() + .backend(&format!("[Extension] Cleanup error: {error}")); + } + runtime.shutdown(); + } + + fn teardown(&self, entry: &Installed, engine: &mut Engine) { + let mut pool = entry.downloads.lock().expect("download pool lock"); + self.invalidate_download_pool(&mut pool); + for runtime in pool.active.drain(..) { + // Join every native operation before source/data can be removed. + runtime.shutdown(); + } + drop(pool); + if let Some(runtime) = engine.runtime.take() { + self.teardown_runtime(&runtime); + } + engine.initialized = false; + engine.source = None; + } + + fn failed(&self, entry: &Installed, failure: &ManagerError) { + let mut status = entry.status.write().expect("extension status lock"); + status.enabled = false; + status.error = failure.to_string(); + } + + fn ready( + &self, + entry: &Installed, + engine: &mut Engine, + settings: bool, + ) -> Result<Arc<ExtensionRuntime>, ManagerError> { + self.ready_with_lease(entry, engine, settings, None) + } + + fn ready_with_lease( + &self, + entry: &Installed, + engine: &mut Engine, + settings: bool, + lease: Option<Arc<RequestLease>>, + ) -> Result<Arc<ExtensionRuntime>, ManagerError> { + // Go rechecks gates even when a VM already exists. A version change + // disables an incompatible provider on its next managed call. + let version = self + .environment + .get_app_version() + .map_err(error_from_environment)?; + validate_gates(&entry.manifest, &version, &SUPPORTED_RUNTIME_FEATURES) + .map_err(error) + .inspect_err(|failure| self.failed(entry, failure))?; + if engine + .runtime + .as_ref() + .is_some_and(|runtime| runtime.is_closed()) + { + self.invalidate_download_pool(&mut entry.downloads.lock().expect("download pool lock")); + engine.runtime = None; + engine.initialized = false; + engine.source = None; + } + if engine.runtime.is_none() { + let (runtime, source) = + self.prepare_with_lease(entry, lease.clone(), LoadMode::Register)?; + engine.runtime = Some(runtime); + engine.source = Some(source); + } + let runtime = Arc::clone(engine.runtime.as_ref().unwrap()); + if settings && !engine.initialized { + let mut values = self + .environment + .settings(&entry.manifest.name) + .map_err(error_from_environment)?; + values.retain(|key, _| !key.starts_with('_')); + if !values.is_empty() + && let Err(error) = self.lifecycle_with_lease( + &runtime, + "initialize", + &json!([values]).to_string(), + lease.clone(), + ) + { + if lease + .as_ref() + .is_some_and(|lease| lease.check_active().is_err()) + { + runtime.shutdown(); + engine.runtime = None; + engine.initialized = false; + engine.source = None; + } else { + self.teardown(entry, engine); + } + return Err(error); + } + engine.initialized = true; + } + entry + .status + .write() + .expect("extension status lock") + .error + .clear(); + Ok(runtime) + } + + pub fn set_enabled(&self, id: &str, enabled: bool) -> Result<(), ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let _mutation = self.mutation.lock().expect("extension mutation lock"); + let entry = self.get(id)?; + { + let mut status = entry.status.write().expect("extension status lock"); + status.enabled = enabled; + if !enabled { + status.error.clear(); + } + } + let mut engine = entry.engine.lock().expect("extension engine lock"); + if enabled { + if let Err(error) = self.ready(&entry, &mut engine, true) { + self.failed(&entry, &error); + let _ = self.environment.set_setting(id, "_enabled", false.into()); + return Err(error); + } + } else { + self.teardown(&entry, &mut engine); + } + if let Err(error) = self.environment.set_setting(id, "_enabled", enabled.into()) { + let _ = self.environment.log_buffer().backend(&format!( + "[Extension] Failed to persist enabled state for {id}: {error}" + )); + } + Ok(()) + } + + pub fn update_settings( + &self, + id: &str, + settings: Map<String, Value>, + ) -> Result<(), ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let _mutation = self.mutation.lock().expect("extension mutation lock"); + self.get(id)?; + self.environment + .replace_settings(id, settings.clone()) + .map_err(error_from_environment)?; + self.initialize(id, settings) + } + + pub fn initialize(&self, id: &str, settings: Map<String, Value>) -> Result<(), ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let entry = self.get(id)?; + let mut engine = entry.engine.lock().expect("extension engine lock"); + if !self.current(&entry) { + return Err(error("extension is no longer installed")); + } + self.invalidate_download_pool(&mut entry.downloads.lock().expect("download pool lock")); + let result = self.ready(&entry, &mut engine, false).and_then(|runtime| { + self.lifecycle(&runtime, "initialize", &json!([settings]).to_string()) + }); + if let Err(error) = &result { + self.failed(&entry, error); + } else { + engine.initialized = true; + } + result + } + + pub fn cleanup(&self, id: &str) -> Result<(), ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let entry = self.get(id)?; + let engine = entry.engine.lock().expect("extension engine lock"); + if !self.current(&entry) { + return Err(error("extension is no longer installed")); + } + engine + .runtime + .as_ref() + .map_or(Ok(()), |runtime| self.lifecycle(runtime, "cleanup", "[]")) + } + + pub fn invoke_action(&self, id: &str, action: &str) -> Result<String, ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let entry = self.get(id).map_err(|e| { + if e.0 == "extension not found" { + error(format!("extension not found: {id}")) + } else { + e + } + })?; + if !entry.status.read().expect("extension status lock").enabled { + return Err(error("extension is disabled")); + } + let mut engine = entry.engine.lock().expect("extension engine lock"); + if !self.current(&entry) || !entry.status.read().expect("extension status lock").enabled { + return Err(error("extension is disabled or no longer installed")); + } + let runtime = self + .ready(&entry, &mut engine, true) + .inspect_err(|e| self.failed(&entry, e))?; + runtime + .managed_call(action, "[]", true, self.limits.timeout_ms) + .map_err(|e| cause("action failed", e)) + } + + pub fn call( + &self, + id: &str, + method: &str, + arguments: &str, + lease: Option<Arc<spotiflac_core::cancellation::RequestLease>>, + timeout_ms: u64, + ) -> Result<String, ManagerError> { + let entry = self.get(id)?; + if !entry.status.read().expect("extension status lock").enabled { + return Err(error("extension is disabled")); + } + let mut engine = entry.engine.lock().expect("extension engine lock"); + if !self.current(&entry) || !entry.status.read().expect("extension status lock").enabled { + return Err(error("extension is disabled or no longer installed")); + } + let runtime = self + .ready(&entry, &mut engine, true) + .inspect_err(|e| self.failed(&entry, e))?; + runtime + .call(method, arguments, lease, timeout_ms) + .map_err(|e| error(e.to_string())) + } + + pub fn unload(&self, id: &str) -> Result<(), ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let _mutation = self.mutation.lock().expect("extension mutation lock"); + self.unload_locked(id) + } + + fn unload_locked(&self, id: &str) -> Result<(), ManagerError> { + self.check()?; + let entry = self + .entries + .lock() + .expect("extension manager lock") + .remove(id) + .ok_or_else(|| error("extension not found"))?; + entry.status.write().expect("extension status lock").enabled = false; + self.teardown( + &entry, + &mut entry.engine.lock().expect("extension engine lock"), + ); + Ok(()) + } + + pub fn unload_all(&self) -> Result<(), ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let _mutation = self.mutation.lock().expect("extension mutation lock"); + self.check()?; + let ids: Vec<_> = self + .entries + .lock() + .expect("extension manager lock") + .keys() + .cloned() + .collect(); + for id in ids { + self.unload_locked(&id)?; + } + Ok(()) + } + + pub fn remove(&self, id: &str) -> Result<(), ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let _mutation = self.mutation.lock().expect("extension mutation lock"); + let entry = self.get(id)?; + if entry.source != self.sources.join(id) { + return Err(error( + "refusing to remove extension outside the managed source directory", + )); + } + self.unload_locked(id)?; + // Attempt both roots even if one fails. Keep a disabled registry entry + // on partial removal so the native caller can retry the same operation. + let source = + remove_directory(&entry.source).map_err(|e| cause("failed to remove source dir", e)); + let data = self + .environment + .remove_data(id) + .map_err(error_from_environment); + let failures: Vec<_> = [source, data].into_iter().filter_map(Result::err).collect(); + if failures.is_empty() { + Ok(()) + } else { + let failure = error( + failures + .iter() + .map(ToString::to_string) + .collect::<Vec<_>>() + .join("; "), + ); + self.failed(&entry, &failure); + self.entries + .lock() + .expect("extension manager lock") + .insert(id.to_owned(), entry); + Err(failure) + } + } + + pub fn upgrade(&self, path: &Path) -> Result<String, ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let _mutation = self.mutation.lock().expect("extension mutation lock"); + self.check()?; + let package = ExtensionPackage::open(path).map_err(|e| error(e.to_string()))?; + let existing = self.get(&package.manifest.name).map_err(|_| { + error(format!( + "extension '{}' is not installed; use install instead of upgrade", + package.manifest.display_name + )) + })?; + self.upgrade_package(package, existing) + } + + fn upgrade_package( + &self, + mut package: ExtensionPackage, + existing: Arc<Installed>, + ) -> Result<String, ManagerError> { + match compare_versions(&package.manifest.version, &existing.manifest.version) { + Comparison::Less => { + return Err(error(format!( + "cannot downgrade extension: current version: {}, new version: {}", + existing.manifest.version, package.manifest.version + ))); + } + Comparison::Equal => { + return Err(error(format!( + "extension is already at version {}", + existing.manifest.version + ))); + } + Comparison::Greater => {} + } + let target = self.sources.join(&existing.manifest.name); + let stage = self.stage(&mut package, "upgrade")?; + let enabled = existing + .status + .read() + .expect("extension status lock") + .enabled; + let mut candidate = self.candidate(&package, stage.path(), enabled); + let valid = if enabled { + self.ready( + &candidate, + &mut candidate.engine.lock().expect("extension engine lock"), + true, + ) + .map(|_| ()) + } else { + self.validate(&candidate) + }; + if let Err(error) = valid { + return Err(cause("upgraded extension failed validation", error)); + } + let backup = (|| { + self.check()?; + let backup = tempfile::Builder::new() + .prefix(&format!(".{}-backup-", existing.manifest.name)) + .tempdir_in(&self.sources) + .map_err(|e| cause("failed to prepare upgrade backup", e))?; + fs::remove_dir(backup.path()) + .map_err(|e| cause("failed to prepare upgrade backup", e))?; + Ok(backup) + })() + .inspect_err(|_: &ManagerError| { + self.teardown( + &candidate, + &mut candidate.engine.lock().expect("extension engine lock"), + ); + })?; + if let Err(error) = fs::rename(&target, backup.path()) { + self.teardown( + &candidate, + &mut candidate.engine.lock().expect("extension engine lock"), + ); + return Err(cause("failed to preserve current extension", error)); + } + if let Err(error) = fs::rename(stage.path(), &target) { + let restored = fs::rename(backup.path(), &target); + self.teardown( + &candidate, + &mut candidate.engine.lock().expect("extension engine lock"), + ); + if let Err(rollback) = restored { + let path = backup.keep(); + return Err(error_from_rollback(error, rollback, &path)); + } + return Err(cause("failed to activate upgraded extension", error)); + } + Arc::get_mut(&mut candidate) + .expect("unpublished extension") + .source = target; + existing + .status + .write() + .expect("extension status lock") + .enabled = false; + self.teardown( + &existing, + &mut existing.engine.lock().expect("extension engine lock"), + ); + self.entries + .lock() + .expect("extension manager lock") + .insert(candidate.manifest.name.clone(), Arc::clone(&candidate)); + Ok(summary(&candidate)) + } + + pub fn check_upgrade(&self, path: &Path) -> Result<String, ManagerError> { + self.check()?; + let package = ExtensionPackage::open(path).map_err(|e| { + if e.0.starts_with("cannot open extension file:") { + error("cannot open extension file") + } else { + error(e.to_string()) + } + })?; + let entries = self.entries.lock().expect("extension manager lock"); + let existing = entries.get(&package.manifest.name); + Ok(json!({"extension_id":package.manifest.name,"current_version":existing.map_or("", |entry| entry.manifest.version.as_str()),"new_version":package.manifest.version,"is_installed":existing.is_some(),"can_upgrade":existing.is_some_and(|entry| compare_versions(&package.manifest.version, &entry.manifest.version)==Comparison::Greater)}).to_string()) + } + + pub fn load_all(&self) -> Result<String, ManagerError> { + let _metadata = MetadataChange(&self.metadata_revision); + let _mutation = self.mutation.lock().expect("extension mutation lock"); + self.check()?; + let mut paths: Vec<_> = fs::read_dir(&self.sources) + .map_err(|e| cause("failed to read extensions directory", e))? + .collect::<Result<_, _>>() + .map_err(|e| cause("failed to read extensions directory", e))?; + paths.sort_by_key(|entry| entry.file_name()); + let mut loaded = Vec::new(); + let mut errors = Vec::new(); + for path in paths { + self.check()?; + let result = if path.file_type().map_err(|e| error(e.to_string()))?.is_dir() { + if !path.path().join("manifest.json").exists() { + continue; + } + self.load_directory(&path.path()) + } else if is_package_path(&path.path()) { + // Reuse the mutation already held instead of entering install(). + ExtensionPackage::open(&path.path()) + .map_err(|e| error(e.to_string())) + .and_then(|package| self.install_or_upgrade(package)) + .and_then(|value| { + serde_json::from_str::<Value>(&value) + .map(|value| value["id"].as_str().unwrap().to_owned()) + .map_err(|e| error(e.to_string())) + }) + } else { + continue; + }; + match result { + Ok(id) => loaded.push(id), + Err(error) => { + errors.push(format!("{}: {error}", path.file_name().to_string_lossy())) + } + } + } + Ok(json!({"loaded":if loaded.is_empty() { Value::Null } else { json!(loaded) },"errors":errors}).to_string()) + } + + fn load_directory(&self, path: &Path) -> Result<String, ManagerError> { + let manifest_json = read_source(&path.join("manifest.json"), 1024 * 1024) + .map_err(|e| cause("failed to read manifest.json", e))?; + let manifest = ExtensionManifest::parse(&manifest_json) + .map_err(|e| cause("invalid extension manifest", e))?; + if !path.join("index.js").exists() { + return Err(error("extension is missing index.js file")); + } + if self + .entries + .lock() + .expect("extension manager lock") + .contains_key(&manifest.name) + { + return Ok(manifest.name); + } + if path != self.sources.join(&manifest.name) { + return Err(error(format!( + "extension directory name must match manifest name {:?}", + manifest.name + ))); + } + let enabled = self + .environment + .settings(&manifest.name) + .map_err(error_from_environment)? + .get("_enabled") + == Some(&Value::Bool(true)); + let entry = Arc::new(Installed { + manifest, + manifest_json, + source: path.to_owned(), + status: RwLock::new(Status { + enabled, + error: String::new(), + }), + engine: Mutex::default(), + downloads: Mutex::default(), + }); + if let Err(error) = self.validate(&entry) { + self.failed(&entry, &error); + } + self.check()?; + let id = entry.manifest.name.clone(); + self.entries + .lock() + .expect("extension manager lock") + .insert(id.clone(), entry); + Ok(id) + } + + pub fn installed(&self) -> Result<String, ManagerError> { + self.check()?; + let entries: Vec<_> = self + .entries + .lock() + .expect("extension manager lock") + .values() + .cloned() + .collect(); + Ok(Value::Array(entries.iter().map(|entry| information(entry)).collect()).to_string()) + } + + pub fn installed_versions(&self) -> Result<BTreeMap<String, String>, ManagerError> { + self.check()?; + Ok(self + .entries + .lock() + .expect("extension manager lock") + .values() + .map(|entry| (entry.manifest.name.clone(), entry.manifest.version.clone())) + .collect()) + } + + pub fn provider_ids(&self, kind: &str) -> Result<Vec<String>, ManagerError> { + self.check()?; + Ok(self + .entries + .lock() + .expect("extension manager lock") + .values() + .filter(|entry| { + let status = entry.status.read().expect("extension status lock"); + entry.manifest.has_type(kind) && status.enabled && status.error.is_empty() + }) + .map(|entry| entry.manifest.name.clone()) + .collect()) + } + + /// Emergency shutdown interrupts workers; orderly unload runs cleanup first. + pub fn shutdown(&self) { + self.closed.store(true, Ordering::Release); + self.environment.shutdown(); + let _mutation = self.mutation.lock().expect("extension mutation lock"); + self.entries.lock().expect("extension manager lock").clear(); + if let Some(locks) = self + .directory_locks + .lock() + .expect("extension directory locks") + .take() + { + // A subprocess can inherit the descriptor between fork and exec. + // Closing our copy alone may leave the old owner's lock held. + for lock in locks { + let _ = lock.unlock(); + } + } + } +} + +impl Drop for ExtensionManager { + fn drop(&mut self) { + self.shutdown(); + } +} + +fn error_from_environment(error: crate::environment::EnvironmentError) -> ManagerError { + ManagerError(error.to_string()) +} +fn error_from_rollback( + error: std::io::Error, + rollback: std::io::Error, + path: &Path, +) -> ManagerError { + cause( + "failed to activate upgraded extension", + format!( + "{error}; restore failed: {rollback}; previous source preserved at {}", + path.display() + ), + ) +} +fn remove_directory(path: &Path) -> std::io::Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => fs::remove_dir_all(path), + Ok(_) => fs::remove_file(path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} +fn read_source(path: &Path, limit: u64) -> std::io::Result<String> { + let mut file = OpenOptions::new() + .read(true) + .custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32) + .open(path)?; + if !file.metadata()?.is_file() { + return Err(std::io::Error::other( + "extension source must be a regular file", + )); + } + let mut bytes = Vec::new(); + (&mut file).take(limit + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 > limit { + return Err(std::io::Error::other( + "extension source exceeds its size limit", + )); + } + Ok(crate::host::decode_go_utf8(&bytes)) +} +fn summary(entry: &Installed) -> String { + json!({"id":entry.manifest.name,"name":entry.manifest.name,"display_name":entry.manifest.display_name,"version":entry.manifest.version,"enabled":entry.status.read().expect("extension status lock").enabled}).to_string() +} +fn asset(root: &Path, name: &str) -> Option<String> { + if name.is_empty() || name.contains('\\') { + return None; + } + let mut parts = Vec::new(); + for component in Path::new(name).components() { + match component { + Component::Normal(part) => parts.push(part), + Component::CurDir => {} + Component::ParentDir => { + parts.pop()?; + } + _ => return None, + } + } + let mut path = root.to_owned(); + for part in parts { + path.push(part); + if fs::symlink_metadata(&path).ok()?.file_type().is_symlink() { + return None; + } + } + path.is_file().then(|| path.to_string_lossy().into_owned()) +} +fn information(entry: &Installed) -> Value { + let manifest = &entry.manifest; + let status = entry.status.read().expect("extension status lock"); + let mut permissions: Vec<String> = manifest + .permissions + .network + .clone() + .unwrap_or_default() + .iter() + .map(|domain| format!("network:{domain}")) + .collect(); + if manifest.permissions.storage { + permissions.push("storage:enabled".into()); + } + if manifest.permissions.file { + permissions.push("file:enabled".into()); + } + if manifest.permissions.allow_http { + permissions.push("network:http".into()); + } + if manifest.has_capability("rawFfmpeg") { + permissions.push("ffmpeg:raw".into()); + } + let mut value = json!({"id":manifest.name,"name":manifest.name,"display_name":manifest.display_name,"version":manifest.version,"description":manifest.description,"types":manifest.types,"enabled":status.enabled,"status":if !status.error.is_empty() {"error"} else if status.enabled {"loaded"} else {"disabled"},"permissions":permissions,"has_metadata_provider":manifest.has_type("metadata_provider"),"has_download_provider":manifest.has_type("download_provider"),"has_lyrics_provider":manifest.has_type("lyrics_provider"),"skip_metadata_enrichment":manifest.skip_metadata_enrichment,"skip_lyrics":manifest.skip_lyrics,"stop_provider_fallback":manifest.stops_provider_fallback()}); + if !status.error.is_empty() { + value["error_message"] = status.error.clone().into(); + } + if let Some(icon) = + asset(&entry.source, &manifest.icon).or_else(|| asset(&entry.source, "icon.png")) + { + value["icon_path"] = icon.into(); + } + let serialized = serde_json::to_value(manifest).expect("manifest serialization"); + for (source, target) in [ + ("homepage", "homepage"), + ("settings", "settings"), + ("qualityOptions", "quality_options"), + ("searchBehavior", "search_behavior"), + ("trackMatching", "track_matching"), + ("postProcessing", "post_processing"), + ("serviceHealth", "service_health"), + ("capabilities", "capabilities"), + ] { + if let Some(field) = serialized.get(source) { + value[target] = field.clone(); + } + } + value +} + +#[cfg(test)] +mod tests { + use super::{ExtensionManager, RuntimeLimits}; + use std::fs::File; + + #[test] + fn shutdown_unlocks_inherited_descriptors_without_unlocking_reopened_owner() { + let directory = tempfile::tempdir().unwrap(); + let open = || { + ExtensionManager::new( + &directory.path().join("sources"), + &directory.path().join("data"), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "1", + RuntimeLimits::default(), + ) + }; + let manager = open().unwrap(); + // A fork before exec can retain the same open file description even + // though the manager closes its own descriptor during shutdown. + let inherited = manager + .directory_locks + .lock() + .unwrap() + .as_ref() + .unwrap() + .iter() + .map(File::try_clone) + .collect::<Result<Vec<_>, _>>() + .unwrap(); + manager.shutdown(); + let reopened = open().unwrap(); + drop(inherited); + drop(manager); + assert!( + open() + .err() + .unwrap() + .to_string() + .contains("already managed") + ); + reopened.shutdown(); + assert!(open().is_ok()); + } +} diff --git a/rust_backend/crates/extensions/src/manager/downloads.rs b/rust_backend/crates/extensions/src/manager/downloads.rs new file mode 100644 index 00000000..8bac7409 --- /dev/null +++ b/rust_backend/crates/extensions/src/manager/downloads.rs @@ -0,0 +1,456 @@ +use super::*; +use serde::Deserialize; + +#[derive(Default)] +pub(super) struct Pool { + pub active: Vec<Arc<ExtensionRuntime>>, + idle: Option<Arc<ExtensionRuntime>>, + generation: u64, + settings: Option<Map<String, Value>>, +} + +#[derive(Default, Deserialize)] +#[serde(default)] +pub struct ProviderDownloadRequest { + pub track_id: String, + pub quality: String, + pub output_path: String, + pub item_id: String, + pub prepared_context: Option<Map<String, Value>>, +} + +struct Borrowed<'a> { + manager: &'a ExtensionManager, + entry: Arc<Installed>, + runtime: Arc<ExtensionRuntime>, + generation: u64, + healthy: bool, +} + +impl Drop for Borrowed<'_> { + fn drop(&mut self) { + let mut pool = self.entry.downloads.lock().expect("download pool lock"); + // Environment settings access may create an absent store. Check manager + // ownership first so a late borrower cannot resurrect uninstalled data. + let reusable = self.healthy + && !self.runtime.is_closed() + && self.manager.current(&self.entry) + && self + .entry + .status + .read() + .expect("extension status lock") + .enabled; + let settings_match = reusable + && self + .manager + .environment + .settings(&self.entry.manifest.name) + .is_ok_and(|mut settings| { + settings.retain(|key, _| !key.starts_with('_')); + pool.settings.as_ref() == Some(&settings) + }); + pool.active + .retain(|runtime| !Arc::ptr_eq(runtime, &self.runtime)); + if reusable && pool.generation == self.generation && pool.idle.is_none() && settings_match { + pool.idle = Some(Arc::clone(&self.runtime)); + } else if !self.runtime.is_closed() { + self.manager.teardown_runtime(&self.runtime); + } else { + self.runtime.shutdown(); + } + } +} + +impl ExtensionManager { + pub(crate) fn download_manifest(&self, id: &str) -> Result<ExtensionManifest, ManagerError> { + let entry = self.get(id)?; + let status = entry.status.read().expect("extension status lock"); + if !status.enabled || !status.error.is_empty() { + return Err(error(format!("extension '{id}' is unavailable"))); + } + Ok(entry.manifest.clone()) + } + + pub(crate) fn preflight_download( + &self, + id: &str, + lease: Arc<RequestLease>, + ) -> Result<bool, ManagerError> { + self.preflight_session(id, lease, true) + } + + pub(crate) fn preflight_auth( + &self, + id: &str, + lease: Arc<RequestLease>, + ) -> Result<bool, ManagerError> { + self.preflight_session(id, lease, false) + } + + fn preflight_session( + &self, + id: &str, + lease: Arc<RequestLease>, + download_only: bool, + ) -> Result<bool, ManagerError> { + let Ok(entry) = self.get(id) else { + self.check()?; + return Ok(false); + }; + if !entry.status.read().expect("extension status lock").enabled + || (download_only && !entry.manifest.has_type("download_provider")) + || entry.manifest.signed_session.is_none() + { + return Ok(false); + } + let runtime = { + let mut engine = self.download_engine(&entry, Some(&lease))?; + if !self.current(&entry) || !entry.status.read().expect("extension status lock").enabled + { + return Err(error("extension is disabled or no longer installed")); + } + self.ready_with_lease(&entry, &mut engine, true, Some(lease.clone()))? + }; + runtime + .preflight_signed_session(Some(lease), 30_000) + .map_err(|failure| error(failure.to_string())) + } + + pub(super) fn download_engine<'a>( + &self, + entry: &'a Installed, + lease: Option<&Arc<RequestLease>>, + ) -> Result<std::sync::MutexGuard<'a, Engine>, ManagerError> { + loop { + self.check()?; + if let Some(lease) = lease { + lease + .check_active() + .map_err(|failure| error(failure.to_string()))?; + } + match entry.engine.try_lock() { + Ok(engine) => return Ok(engine), + Err(std::sync::TryLockError::WouldBlock) => { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + Err(std::sync::TryLockError::Poisoned(_)) => { + return Err(error("extension engine lock poisoned")); + } + } + } + } + + pub(super) fn invalidate_download_pool(&self, pool: &mut Pool) -> u64 { + pool.generation = pool.generation.wrapping_add(1); + pool.settings = None; + if let Some(runtime) = pool.idle.take() { + self.teardown_runtime(&runtime); + 1 + } else { + 0 + } + } + + /// Memory pressure releases idle heaps. Borrowed workers may finish but + /// cannot repopulate the pool created before this request. + pub fn release_idle_download_runtimes(&self) -> Result<u64, ManagerError> { + self.check()?; + let entries: Vec<_> = self + .entries + .lock() + .expect("extension manager lock") + .values() + .cloned() + .collect(); + let mut released = 0; + for entry in entries { + let mut pool = entry.downloads.lock().expect("download pool lock"); + released += self.invalidate_download_pool(&mut pool); + } + Ok(released) + } + + fn borrow_download( + &self, + entry: Arc<Installed>, + lease: Option<Arc<RequestLease>>, + ) -> Result<Borrowed<'_>, ManagerError> { + let mut engine = self.download_engine(&entry, lease.as_ref())?; + if !self.current(&entry) || !entry.status.read().expect("extension status lock").enabled { + return Err(error("extension is disabled or no longer installed")); + } + let primary = self.ready_with_lease(&entry, &mut engine, true, lease)?; + let mut settings = self + .environment + .settings(&entry.manifest.name) + .map_err(error_from_environment)?; + settings.retain(|key, _| !key.starts_with('_')); + let mut pool = entry.downloads.lock().expect("download pool lock"); + if pool.settings.as_ref() != Some(&settings) { + self.invalidate_download_pool(&mut pool); + pool.settings = Some(settings.clone()); + } + let idle = pool.idle.take(); + let generation = pool.generation; + // Initialization runs extension code. Pressure may invalidate this + // borrow while it initializes, without cancelling the operation. + drop(pool); + let runtime = match idle { + Some(runtime) if !runtime.is_closed() => runtime, + _ => { + let runtime = self + .environment + .load_isolated( + &entry.manifest_json, + engine.source.as_ref().expect("initialized source"), + self.limits.clone(), + &primary, + ) + .map_err(error_from_environment)?; + if !settings.is_empty() + && let Err(error) = + self.lifecycle(&runtime, "initialize", &json!([settings]).to_string()) + { + self.teardown_runtime(&runtime); + return Err(error); + } + runtime + } + }; + entry + .downloads + .lock() + .expect("download pool lock") + .active + .push(Arc::clone(&runtime)); + drop(engine); + Ok(Borrowed { + manager: self, + entry, + runtime, + generation, + healthy: false, + }) + } + + /// Execute the installed provider's download contract in an isolated worker. + /// File placement, post-processing and finalization remain caller-owned. + pub fn download( + &self, + id: &str, + request: ProviderDownloadRequest, + resolution_timeout_ms: u64, + ) -> Result<String, ManagerError> { + self.download_with_lease(id, request, resolution_timeout_ms, None) + } + + pub(crate) fn download_with_lease( + &self, + id: &str, + request: ProviderDownloadRequest, + resolution_timeout_ms: u64, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ManagerError> { + let entry = self.get(id)?; + if !entry.manifest.has_type("download_provider") { + return Err(error(format!( + "extension '{id}' is not a download provider" + ))); + } + if !entry.status.read().expect("extension status lock").enabled { + return Err(error(format!("extension '{id}' is disabled"))); + } + let mut borrowed = match self.borrow_download(entry, lease.clone()) { + Ok(borrowed) => borrowed, + Err(error) => return Ok(failure("init_error", &error.to_string())), + }; + let allowance = if resolution_timeout_ms == 0 { + 60_000 + } else { + resolution_timeout_ms.min(300_000) + }; + let mut options = json!({"resolutionTimeoutMs":allowance}); + if let Some(context) = request + .prepared_context + .filter(|context| !context.is_empty()) + { + options["preparedContext"] = context.into(); + } + // This marker belongs to one VM and one invocation, never to a previous + // download that returned its worker to the idle pool. + borrowed.runtime.take_verification_url(); + let response = borrowed.runtime.call_download_provider( + &json!([ + request.track_id, + request.quality, + request.output_path, + options + ]) + .to_string(), + &request.item_id, + allowance, + lease, + ); + let response = match response { + Ok(response) => response, + Err((ExtensionError::Cancelled(cause), _)) => return Err(error(cause.to_string())), + Err((ExtensionError::Timeout, resolution)) => { + return Ok(failure( + "timeout", + if resolution { + "stream resolution timeout: extension took too long to resolve an audio stream" + } else { + "download timeout: extension took too long to complete" + }, + )); + } + Err((error, _)) => return Ok(failure("script_error", &error.to_string())), + }; + let mut response: Value = serde_json::from_str(&response) + .map_err(|error| cause("invalid download result", error))?; + if let Some(error) = response.get("parseError").and_then(Value::as_str) { + return Ok(failure("script_error", error)); + } + borrowed.healthy = true; + let mut value = response["value"].take(); + if value.is_null() { + return Ok(failure("not_implemented", "download returned null")); + } + normalize_decryption(&mut value); + if value["success"] != true + && !borrowed.runtime.take_verification_url().is_empty() + && !value["error_type"] + .as_str() + .unwrap_or("") + .eq_ignore_ascii_case("verification_required") + { + value["error_type"] = "verification_required".into(); + if value["error_message"].as_str().unwrap_or("").is_empty() { + value["error_message"] = "Verification required".into(); + } + } + Ok(value.to_string()) + } +} + +fn failure(kind: &str, message: &str) -> String { + json!({"success":false,"error_type":kind,"error_message":message}).to_string() +} + +fn normalize_decryption(value: &mut Value) { + const MOV: &str = "ffmpeg.mov_key"; + let legacy = value["decryption_key"] + .as_str() + .unwrap_or("") + .trim() + .to_owned(); + let mut info = value + .as_object_mut() + .unwrap() + .remove("decryption") + .and_then(|info| info.as_object().cloned()); + if info.is_none() && !legacy.is_empty() { + info = Some(Map::new()); + } + if let Some(info) = &mut info { + for key in ["strategy", "key", "iv", "input_format", "output_extension"] { + if let Some(value) = info.get_mut(key) { + *value = value.as_str().unwrap_or("").trim().into(); + } + } + let strategy = info.get("strategy").and_then(Value::as_str).unwrap_or(""); + let strategy = match spotiflac_core::matching::lowercase(strategy).as_str() { + "" + | MOV + | "ffmpeg_mov_key" + | "mov_decryption_key" + | "mp4_decryption_key" + | "ffmpeg.mp4_decryption_key" => MOV.to_owned(), + _ => strategy.to_owned(), + }; + info.insert("strategy".into(), strategy.clone().into()); + if info + .get("key") + .and_then(Value::as_str) + .unwrap_or("") + .is_empty() + && !legacy.is_empty() + { + info.insert("key".into(), legacy.clone().into()); + } + if strategy == MOV + && info + .get("input_format") + .and_then(Value::as_str) + .unwrap_or("") + .is_empty() + { + info.insert("input_format".into(), "mov".into()); + } + info.retain(|_, value| value != ""); + } + if info.as_ref().is_some_and(|info| { + info.get("strategy").and_then(Value::as_str) == Some(MOV) + && info + .get("key") + .and_then(Value::as_str) + .unwrap_or("") + .is_empty() + }) { + info = None; + } + let key = info + .as_ref() + .filter(|info| info.get("strategy").and_then(Value::as_str) == Some(MOV)) + .and_then(|info| info.get("key")) + .and_then(Value::as_str) + .unwrap_or(&legacy) + .to_owned(); + let value = value.as_object_mut().unwrap(); + value.remove("decryption_key"); + if !key.is_empty() { + value.insert("decryption_key".into(), key.into()); + } + if let Some(info) = info { + value.insert("decryption".into(), info.into()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn returning_a_borrow_after_uninstall_cannot_recreate_extension_data() { + let directory = tempfile::tempdir().unwrap(); + let sources = directory.path().join("sources"); + let data = directory.path().join("data"); + let manager = ExtensionManager::new( + &sources, + &data, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "1", + RuntimeLimits::default(), + ) + .unwrap(); + let source = sources.join("example.pool"); + fs::create_dir(&source).unwrap(); + fs::write(source.join("manifest.json"), json!({"name":"example.pool","version":"1","description":"Generic pool ownership test","type":["download_provider"],"permissions":{"storage":true}}).to_string()).unwrap(); + fs::write( + source.join("index.js"), + "registerExtension({download(){return {success:true}}});", + ) + .unwrap(); + manager.load_all().unwrap(); + manager.set_enabled("example.pool", true).unwrap(); + let mut borrowed = manager + .borrow_download(manager.get("example.pool").unwrap(), None) + .unwrap(); + borrowed.healthy = true; + manager.remove("example.pool").unwrap(); + assert!(!data.join("example.pool").exists()); + drop(borrowed); + assert!(!data.join("example.pool").exists()); + } +} diff --git a/rust_backend/crates/extensions/src/manager/metadata.rs b/rust_backend/crates/extensions/src/manager/metadata.rs new file mode 100644 index 00000000..b2d7b404 --- /dev/null +++ b/rust_backend/crates/extensions/src/manager/metadata.rs @@ -0,0 +1,453 @@ +use super::*; +use serde::Deserialize; +use spotiflac_core::cancellation::RequestLease; +use spotiflac_core::matching::{lowercase, uppercase}; +use std::collections::BTreeSet; + +#[derive(Default, Deserialize)] +#[serde(default)] +pub struct ProviderAvailabilityRequest { + pub isrc: String, + pub track_name: String, + pub artist_name: String, + pub spotify_id: String, + pub deezer_id: String, + pub tidal_id: String, + pub qobuz_id: String, + pub duration_ms: isize, + pub item_id: String, + pub track: Option<Map<String, Value>>, +} + +impl ExtensionManager { + fn item_lease(&self, id: &str) -> Result<Option<Arc<RequestLease>>, ManagerError> { + if id.is_empty() { + return Ok(None); + } + self.environment + .download_state() + .cancellation + .acquire(id) + .map(|lease| Some(Arc::new(lease))) + .map_err(|failure| error(failure.to_string())) + } + + pub fn check_availability( + &self, + id: &str, + request: ProviderAvailabilityRequest, + timeout_ms: u64, + ) -> Result<String, ManagerError> { + let lease = self.item_lease(&request.item_id)?; + self.check_availability_with_lease(id, request, timeout_ms, lease) + } + + pub(crate) fn check_availability_with_lease( + &self, + id: &str, + request: ProviderAvailabilityRequest, + timeout_ms: u64, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ManagerError> { + let mut options = json!({"spotify_id":request.spotify_id,"deezer_id":request.deezer_id, + "tidal_id":request.tidal_id,"qobuz_id":request.qobuz_id,"duration_ms":request.duration_ms}); + if let Some(track) = request.track.filter(|track| !track.is_empty()) { + options["track"] = track.into(); + } + self.provider_operation( + id, + "checkAvailability", + &json!([ + request.isrc, + request.track_name, + request.artist_name, + options + ]) + .to_string(), + lease, + timeout_ms, + &request.item_id, + ) + } + + /// Best-effort enrichment preserves the original provider attribution. + /// Cancellation remains an error, allowing the download worker to stop. + pub fn enrich_track( + &self, + id: &str, + track_json: &str, + item_id: &str, + timeout_ms: u64, + ) -> Result<String, ManagerError> { + self.enrich_track_with_lease( + id, + track_json, + item_id, + timeout_ms, + self.item_lease(item_id)?, + ) + } + + pub(crate) fn enrich_track_export( + &self, + id: &str, + track_json: &str, + ) -> Result<String, ManagerError> { + self.check()?; + let Ok(entry) = self.get(id) else { + return Ok(track_json.into()); + }; + if !entry.manifest.has_type("metadata_provider") { + return Ok(track_json.into()); + } + // This export unmarshals into a value struct, unlike the nullable + // provider argument used by the internal download API. + let input = if track_json.trim() == "null" { + "{}" + } else { + track_json + }; + self.enrich_track(id, input, "", 30_000).map_err(|failure| { + if let Some(message) = failure.0.strip_prefix("invalid track: ") { + error(format!("failed to parse track: {message}")) + } else { + failure + } + }) + } + + pub(crate) fn enrich_track_with_lease( + &self, + id: &str, + track_json: &str, + item_id: &str, + timeout_ms: u64, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ManagerError> { + let (original, input) = track_input(track_json)?; + let entry = self.get(id)?; + if !entry.manifest.has_type("metadata_provider") + || !entry.status.read().expect("extension status lock").enabled + { + return Ok(original.to_string()); + } + match self.provider_operation( + id, + "enrichTrack", + &json!([input]).to_string(), + lease, + timeout_ms, + item_id, + ) { + Ok(response) => { + let mut track: Value = serde_json::from_str(&response) + .map_err(|failure| cause("invalid enriched track", failure))?; + track["provider_id"] = original.get("provider_id").cloned().unwrap_or(json!("")); + Ok(track.to_string()) + } + Err(failure) if failure.0 == "download cancelled" => Err(failure), + Err(failure) => { + let _ = self.environment.log_buffer().backend(&format!( + "[Extension] EnrichTrack error for {id}: {failure}" + )); + Ok(original.to_string()) + } + } + } + + pub fn search_metadata_provider( + &self, + id: &str, + query: &str, + limit: isize, + timeout_ms: u64, + ) -> Result<String, ManagerError> { + let id = id.trim(); + if id.is_empty() { + return Err(error("metadata provider ID is required")); + } + let entry = self.get(id)?; + if !entry.manifest.has_type("metadata_provider") { + return Err(error(format!( + "extension '{id}' is not a metadata provider" + ))); + } + { + let status = entry.status.read().expect("extension status lock"); + if !status.enabled { + return Err(error(format!("extension '{id}' is disabled"))); + } + if !status.error.is_empty() { + return Err(error(format!( + "extension '{id}' is unavailable: {}", + status.error + ))); + } + } + let limit = search_limit(limit); + let response = self.provider_call( + id, + "searchTracks", + &json!([query, limit]).to_string(), + None, + timeout_ms, + )?; + let mut response: Value = + serde_json::from_str(&response).map_err(|e| error(e.to_string()))?; + let mut tracks = response["tracks"].take(); + if let Some(tracks) = tracks.as_array_mut() { + tracks.truncate(limit); + } + Ok(tracks.to_string()) + } + + pub fn search_metadata_providers( + &self, + query: &str, + limit: isize, + include_extensions: bool, + item_id: &str, + timeout_ms: u64, + ) -> Result<String, ManagerError> { + self.search_metadata_providers_with_lease( + query, + limit, + include_extensions, + item_id, + timeout_ms, + None, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn search_metadata_providers_with_lease( + &self, + query: &str, + limit: isize, + include_extensions: bool, + item_id: &str, + timeout_ms: u64, + request_lease: Option<Arc<RequestLease>>, + ) -> Result<String, ManagerError> { + self.check()?; + let mut ordered = self + .priorities + .lock() + .expect("provider priority lock") + .metadata + .clone(); + let providers: BTreeSet<_> = if include_extensions { + self.provider_ids("metadata_provider")? + .into_iter() + .collect() + } else { + BTreeSet::new() + }; + let prioritized: BTreeSet<_> = ordered.iter().cloned().collect(); + ordered.extend(providers.difference(&prioritized).cloned()); + let limit = search_limit(limit); + let mut tracks = Vec::new(); + let mut seen = BTreeSet::new(); + let mut verification = None; + // Keep the same cancellation acquisition alive between providers. + let lease = if request_lease.is_some() { + request_lease + } else if ordered.is_empty() { + None + } else { + self.item_lease(item_id)? + }; + for id in ordered { + if let Some(lease) = &lease { + lease.check_active().map_err(|e| error(e.to_string()))?; + } + if !providers.contains(&id) { + continue; + } + let response = self.provider_operation( + &id, + "searchTracks", + &json!([query, limit]).to_string(), + lease.clone(), + timeout_ms, + item_id, + ); + if let Some(lease) = &lease { + lease.check_active().map_err(|e| error(e.to_string()))?; + } + let response = match response { + Ok(response) => response, + Err(failure) if failure.0 == "download cancelled" => return Err(failure), + Err(failure) => { + let _ = self.environment.log_buffer().backend(&format!( + "[MetadataSearch] Search error from {id}: {failure}" + )); + if verification.is_none() && requires_verification(&failure.0) { + verification = Some(error(format!( + "verification_required: extension '{id}' needs verification: {failure}" + ))); + } + continue; + } + }; + let response: Value = + serde_json::from_str(&response).map_err(|e| error(e.to_string()))?; + if let Some(found) = response["tracks"].as_array() { + for track in found { + if seen.insert(dedup_key(track)) { + tracks.push(track.clone()); + if tracks.len() >= limit { + return Ok(json!(tracks).to_string()); + } + } + } + } + } + if tracks.is_empty() + && let Some(failure) = verification + { + return Err(failure); + } + Ok(json!(tracks).to_string()) + } +} + +fn search_limit(limit: isize) -> usize { + if limit <= 0 { 20 } else { limit as usize } +} + +fn requires_verification(message: &str) -> bool { + let message = lowercase(message); + if ["isp blocking", "try using vpn", "change dns", "cancel"] + .iter() + .any(|part| message.contains(part)) + { + return false; + } + [ + "verification_required", + "session is not authenticated", + "signed session is not authenticated", + "signed session expired", + ] + .iter() + .any(|part| message.contains(part)) +} + +fn dedup_key(track: &Value) -> String { + let field = |name: &str| track[name].as_str().unwrap_or("").trim(); + if !field("isrc").is_empty() { + format!("isrc:{}", uppercase(field("isrc"))) + } else if !field("spotify_id").is_empty() { + format!("spotify:{}", field("spotify_id")) + } else if !field("provider_id").is_empty() && !field("id").is_empty() { + format!("{}:{}", field("provider_id"), field("id")) + } else { + format!("{}|{}", field("name"), field("artists")) + } +} + +// Native inputs use the canonical Go metadata JSON schema, not the aliases +// accepted when reading extension JavaScript results. Every input field is +// present, including zero values omitted from serialized metadata results. +fn track_input(text: &str) -> Result<(Value, Value), ManagerError> { + let value: Value = serde_json::from_str(text).map_err(|e| cause("invalid track", e))?; + if value.is_null() { + return Ok((Value::Null, json!({}))); + } + let Some(value) = value.as_object() else { + return Err(error("invalid track: expected object or null")); + }; + let mut input = Map::new(); + let mut original = Map::new(); + for key in [ + "id", + "name", + "artists", + "album_name", + "album_artist", + "album_id", + "album_url", + "artist_id", + "artist_url", + "external_urls", + "cover_url", + "preview_url", + "images", + "release_date", + "isrc", + "provider_id", + "item_type", + "album_type", + "tidal_id", + "qobuz_id", + "deezer_id", + "spotify_id", + "label", + "copyright", + "genre", + "composer", + "comment", + "audio_quality", + "audio_modes", + "upc", + ] { + let text = match value.get(key).filter(|v| !v.is_null()) { + None => "", + Some(value) => value + .as_str() + .ok_or_else(|| error(format!("invalid track: {key} must be a string")))?, + }; + input.insert(key.into(), json!(text)); + if !text.is_empty() || ["id", "name", "artists", "album_name", "provider_id"].contains(&key) + { + original.insert(key.into(), json!(text)); + } + } + for key in [ + "duration_ms", + "track_number", + "total_tracks", + "disc_number", + "total_discs", + ] { + let integer = match value.get(key).filter(|v| !v.is_null()) { + None => 0, + Some(value) => value + .as_i64() + .and_then(|value| isize::try_from(value).ok()) + .ok_or_else(|| error(format!("invalid track: {key} must be a native integer")))?, + }; + input.insert(key.into(), json!(integer)); + if integer != 0 || key == "duration_ms" { + original.insert(key.into(), json!(integer)); + } + } + let explicit = match value.get("explicit").filter(|v| !v.is_null()) { + None => false, + Some(value) => value + .as_bool() + .ok_or_else(|| error("invalid track: explicit must be a boolean"))?, + }; + input.insert("explicit".into(), json!(explicit)); + if explicit { + original.insert("explicit".into(), json!(true)); + } + let links = value.get("external_links").cloned().unwrap_or(Value::Null); + if !links.is_null() + && !links + .as_object() + .is_some_and(|links| links.values().all(Value::is_string)) + { + return Err(error("invalid track: external_links must be a string map")); + } + if links.as_object().is_some_and(|links| !links.is_empty()) { + original.insert("external_links".into(), links.clone()); + } + // Go's argument adapter copies a typed nil map to an empty JS object. + input.insert( + "external_links".into(), + if links.is_null() { json!({}) } else { links }, + ); + Ok((original.into(), input.into())) +} diff --git a/rust_backend/crates/extensions/src/manager/postprocess.rs b/rust_backend/crates/extensions/src/manager/postprocess.rs new file mode 100644 index 00000000..465e8d14 --- /dev/null +++ b/rust_backend/crates/extensions/src/manager/postprocess.rs @@ -0,0 +1,306 @@ +use super::*; +use crate::files::clean; +use serde::{Deserialize, Serialize}; +use spotiflac_core::matching::lowercase; + +#[derive(Clone, Default, Deserialize, Serialize)] +#[serde(default)] +pub struct PostProcessInput { + #[serde(skip_serializing_if = "String::is_empty")] + pub item_id: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub path: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub uri: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub name: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub mime_type: String, + #[serde(skip_serializing_if = "is_zero")] + pub size: i64, + #[serde(skip_serializing_if = "is_false")] + pub is_saf: bool, +} + +fn is_zero(value: &i64) -> bool { + *value == 0 +} +fn is_false(value: &bool) -> bool { + !value +} +fn failed(message: impl std::fmt::Display) -> Value { + json!({"success":false,"error":message.to_string()}) +} + +impl ExtensionManager { + pub fn post_process( + &self, + id: &str, + input: PostProcessInput, + metadata: Map<String, Value>, + hook: &str, + timeout_ms: u64, + ) -> Result<String, ManagerError> { + let lease = self.post_process_lease(&input)?; + self.post_process_hook(id, &input, &metadata, hook, timeout_ms, lease) + .map(|value| value.to_string()) + } + + fn post_process_lease( + &self, + input: &PostProcessInput, + ) -> Result<Option<Arc<RequestLease>>, ManagerError> { + if input.item_id.is_empty() { + return Ok(None); + } + let lease = self + .environment + .download_state() + .acquire(&input.item_id) + .map_err(|failure| error(failure.to_string()))?; + lease + .check_active() + .map_err(|failure| error(failure.to_string()))?; + Ok(Some(Arc::new(lease))) + } + + fn post_process_hook( + &self, + id: &str, + input: &PostProcessInput, + metadata: &Map<String, Value>, + hook: &str, + timeout_ms: u64, + lease: Option<Arc<RequestLease>>, + ) -> Result<Value, ManagerError> { + let entry = self.get(id)?; + if !entry + .manifest + .post_processing + .as_ref() + .is_some_and(|config| config.enabled) + { + return Err(error(format!( + "extension '{id}' does not support post-processing" + ))); + } + if !entry.status.read().expect("extension status lock").enabled { + return Err(error(format!("extension '{id}' is disabled"))); + } + let mut engine = self.download_engine(&entry, lease.as_ref())?; + if !self.current(&entry) || !entry.status.read().expect("extension status lock").enabled { + return Ok(failed("extension is disabled or no longer installed")); + } + let runtime = match self.ready_with_lease(&entry, &mut engine, true, lease.clone()) { + Ok(runtime) => runtime, + Err(error) => { + if lease + .as_ref() + .is_some_and(|lease| lease.check_active().is_err()) + { + return Err(error); + } + self.failed(&entry, &error); + return Ok(failed(error)); + } + }; + let result = match runtime.call_post_process( + &json!([input, metadata, hook]).to_string(), + timeout_ms, + &input.item_id, + lease, + ) { + Ok(result) => result, + Err(ExtensionError::Timeout) => { + return Ok(failed( + "postProcess timeout: extension took too long to complete", + )); + } + Err(error) => return Ok(failed(error)), + }; + let mut response: Value = + serde_json::from_str(&result).map_err(|e| error(e.to_string()))?; + Ok(response["value"].take()) + } + + pub fn run_post_processing( + &self, + mut input: PostProcessInput, + metadata: Map<String, Value>, + timeout_ms: u64, + ) -> Result<String, ManagerError> { + self.check()?; + let lease = self.post_process_lease(&input)?; + let check = || -> Result<(), ManagerError> { + self.check()?; + if let Some(lease) = &lease { + lease + .check_active() + .map_err(|failure| error(failure.to_string()))?; + } + Ok(()) + }; + // Go's map iteration has no defined provider order. Use stable IDs; + // hooks within each provider retain their manifest order. + let providers: Vec<_> = self + .entries + .lock() + .expect("extension manager lock") + .values() + .filter(|entry| { + let status = entry.status.read().expect("extension status lock"); + status.enabled + && status.error.is_empty() + && entry + .manifest + .post_processing + .as_ref() + .is_some_and(|config| config.enabled) + }) + .cloned() + .collect(); + for provider in providers { + let config = provider + .manifest + .post_processing + .as_ref() + .expect("post-processing config"); + for hook in &config.hooks { + check()?; + if !hook.default_enabled { + continue; + } + let mut extension = extension(&input.path); + if extension.is_empty() { + extension = self::extension(&input.name); + } + if !hook.supported_formats.is_empty() + && !extension.is_empty() + && !hook.supported_formats.iter().any(|format| { + format!(".{format}") == extension || format == &extension[1..] + }) + { + continue; + } + let id = &provider.manifest.name; + let log = |message: String| { + let _ = self + .environment + .log_buffer() + .backend(&format!("[PostProcessV2] {message}")); + }; + log(format!( + "Running hook {} from {id} on {}", + hook.id, input.path + )); + let result = match self.post_process_hook( + id, + &input, + &metadata, + &hook.id, + timeout_ms, + lease.clone(), + ) { + Ok(result) => result, + Err(error) => { + log(format!("Hook {} failed: {error}", hook.id)); + continue; + } + }; + check()?; + if result["success"] != true { + continue; + } + if let Err(error) = self.validate_post_process_result(&provider, &input, &result) { + log(format!( + "Hook {} returned an unsafe result: {error}", + hook.id + )); + continue; + } + if let Some(path) = result["new_file_path"] + .as_str() + .filter(|path| !path.is_empty()) + { + input.path = path.into(); + if input.name.is_empty() { + let clean = clean(Path::new(path)); + input.name = clean + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or("/".into()); + } + } + if let Some(uri) = result["new_file_uri"] + .as_str() + .filter(|uri| !uri.is_empty()) + { + input.uri = uri.into(); + } + } + } + check()?; + let mut result = json!({"success":true}); + if !input.path.is_empty() { + result["new_file_path"] = input.path.into(); + } + if !input.uri.is_empty() { + result["new_file_uri"] = input.uri.into(); + } + Ok(result.to_string()) + } + + fn validate_post_process_result( + &self, + entry: &Installed, + input: &PostProcessInput, + result: &Value, + ) -> Result<(), ManagerError> { + let path = result["new_file_path"].as_str().unwrap_or(""); + let uri = result["new_file_uri"].as_str().unwrap_or(""); + if !uri.is_empty() && uri != input.uri { + return Err(error("an extension cannot replace the destination URI")); + } + if path.is_empty() || clean(Path::new(path)) == clean(Path::new(&input.path)) { + return Ok(()); + } + if !entry.manifest.permissions.file { + return Err(error( + "file permission is required to replace the processed file", + )); + } + if !Path::new(path).is_absolute() { + return Err(error("replacement file path must be absolute")); + } + self.environment + .validate_post_process_path(&entry.manifest.name, &input.path, path) + .map_err(error) + } +} + +fn extension(path: &str) -> String { + let name = path.rsplit('/').next().unwrap_or(""); + name.rfind('.') + .map(|index| lowercase(&name[index..])) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::extension; + + #[test] + fn extension_uses_go_filename_rules_before_path_cleaning() { + for (path, expected) in [ + ("song.FLAC", ".flac"), + ("album.flac/", ""), + (".hidden", ".hidden"), + ("a/.", "."), + ("a/..", "."), + ("a\\b.FLAC", ".flac"), + ("", ""), + ] { + assert_eq!(extension(path), expected, "{path}"); + } + } +} diff --git a/rust_backend/crates/extensions/src/manager/providers.rs b/rust_backend/crates/extensions/src/manager/providers.rs new file mode 100644 index 00000000..190d4771 --- /dev/null +++ b/rust_backend/crates/extensions/src/manager/providers.rs @@ -0,0 +1,400 @@ +use super::*; +use spotiflac_core::matching::lowercase; +use std::collections::BTreeSet; +use std::sync::TryLockError; +use std::time::Duration; + +#[derive(Default)] +pub(super) struct Priorities { + download: Vec<String>, + pub(super) metadata: Vec<String>, + fallback: Option<Vec<String>>, +} + +impl ExtensionManager { + pub(crate) fn metadata_revision(&self) -> u64 { + self.metadata_revision.load(Ordering::Acquire) + } + + pub(crate) fn share_providers( + &self, + source_id: &str, + ) -> Result<Vec<spotiflac_core::metadata::share::Provider>, ManagerError> { + self.check()?; + Ok(self + .entries + .lock() + .expect("extension manager lock") + .values() + .filter(|entry| { + let status = entry.status.read().expect("extension status lock"); + entry.manifest.name != source_id + && entry.manifest.has_type("metadata_provider") + && status.enabled + && status.error.is_empty() + }) + .map(|entry| spotiflac_core::metadata::share::Provider { + id: entry.manifest.name.clone(), + display_name: entry.manifest.display_name.clone(), + source_dir: entry.source.to_string_lossy().into_owned(), + capabilities: json!(entry.manifest.capabilities), + }) + .collect()) + } + + pub fn set_provider_priority(&self, kind: &str, ids: Vec<String>) -> Result<(), ManagerError> { + self.check()?; + let (provider_type, legacy): (&str, &[&str]) = match kind { + "download" => ("download_provider", &["deezer", "qobuz", "tidal"]), + "metadata" => ( + "metadata_provider", + &["deezer", "spotify", "qobuz", "tidal"], + ), + _ => return Err(error("unknown provider priority kind")), + }; + let enabled: BTreeSet<_> = self + .provider_ids(provider_type)? + .into_iter() + .map(|id| lowercase(id.trim())) + .collect(); + let mut seen = BTreeSet::new(); + let ids = ids + .into_iter() + .filter_map(|id| { + let id = id.trim(); + let normalized = lowercase(id); + if id.is_empty() + || (legacy.contains(&normalized.as_str()) && !enabled.contains(&normalized)) + { + return None; + } + let key = if kind == "download" { + normalized + } else { + id.to_owned() + }; + seen.insert(key).then(|| id.to_owned()) + }) + .collect(); + let mut priorities = self.priorities.lock().expect("provider priority lock"); + if kind == "download" { + priorities.download = ids; + } else { + priorities.metadata = ids; + } + Ok(()) + } + + pub fn set_fallback_providers(&self, ids: Option<Vec<String>>) -> Result<(), ManagerError> { + self.check()?; + let mut seen = BTreeSet::new(); + self.priorities + .lock() + .expect("provider priority lock") + .fallback = ids.map(|ids| { + ids.into_iter() + .filter_map(|id| { + let id = id.trim(); + (!id.is_empty() && seen.insert(id.to_owned())).then(|| id.to_owned()) + }) + .collect() + }); + Ok(()) + } + + pub fn provider_priorities(&self) -> Result<String, ManagerError> { + self.check()?; + let priorities = self.priorities.lock().expect("provider priority lock"); + Ok(json!({"download":priorities.download,"metadata":priorities.metadata,"fallback":priorities.fallback}).to_string()) + } + + pub fn fallback_allowed(&self, id: &str) -> Result<bool, ManagerError> { + self.check()?; + Ok(self + .priorities + .lock() + .expect("provider priority lock") + .fallback + .as_ref() + .is_none_or(|ids| ids.iter().any(|entry| entry == id))) + } + + pub fn find_url_handler(&self, url: &str) -> Result<Option<String>, ManagerError> { + self.check()?; + let mut matches: Vec<_> = self + .entries + .lock() + .expect("extension manager lock") + .values() + .filter(|entry| { + let status = entry.status.read().expect("extension status lock"); + status.enabled && status.error.is_empty() && entry.manifest.matches_url(url) + }) + .map(|entry| entry.manifest.name.clone()) + .collect(); + let priorities = self.priorities.lock().expect("provider priority lock"); + let ranks: BTreeMap<_, _> = priorities + .metadata + .iter() + .enumerate() + .map(|(index, id)| (lowercase(id.trim()), index)) + .collect(); + matches.sort_by_key(|id| { + ( + ranks.get(&lowercase(id)).copied().unwrap_or(usize::MAX), + id.clone(), + ) + }); + Ok(matches.into_iter().next()) + } + + /// Validate the application metadata wrapper before checking its resource type. + pub(crate) fn require_metadata_provider(&self, id: &str) -> Result<(), ManagerError> { + let entry = self.get(id)?; + if !entry.manifest.has_type("metadata_provider") { + return Err(error(format!( + "extension '{id}' is not a metadata provider" + ))); + } + if !entry.status.read().expect("extension status lock").enabled { + return Err(error(format!("extension '{id}' is disabled"))); + } + Ok(()) + } + + /// Typed metadata/lyrics results from installed providers. Download and + /// post-processing orchestration use their separate runtime/file lifecycles. + pub fn provider_call( + &self, + id: &str, + method: &str, + arguments: &str, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + ) -> Result<String, ManagerError> { + self.provider_operation(id, method, arguments, lease, timeout_ms, "") + } + + pub(super) fn provider_operation( + &self, + id: &str, + method: &str, + arguments: &str, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + item_id: &str, + ) -> Result<String, ManagerError> { + let entry = self.get(id)?; + let manifest = &entry.manifest; + let requirement = + match method { + "getTrack" | "getAlbum" | "getPlaylist" | "getArtist" | "searchTracks" + | "enrichTrack" => (!manifest.has_type("metadata_provider")) + .then_some("is not a metadata provider"), + "checkAvailability" => (!manifest.has_type("download_provider")) + .then_some("is not a download provider"), + "fetchLyrics" => { + (!manifest.has_type("lyrics_provider")).then_some("is not a lyrics provider") + } + "customSearch" => (!manifest + .search_behavior + .as_ref() + .is_some_and(|config| config.enabled)) + .then_some("does not support custom search"), + "handleUrl" => (!manifest + .url_handler + .as_ref() + .is_some_and(|config| config.enabled && !config.patterns.is_empty())) + .then_some("does not support URL handling"), + "getHomeFeed" => None, + _ => return Err(error(format!("unsupported provider method: {method}"))), + }; + if let Some(requirement) = requirement { + return Err(error(format!("extension '{id}' {requirement}"))); + } + if !entry.status.read().expect("extension status lock").enabled { + return Err(error(format!("extension '{id}' is disabled"))); + } + let mut engine = loop { + self.check()?; + if let Some(lease) = &lease { + lease + .check_active() + .map_err(|failure| error(failure.to_string()))?; + } + match entry.engine.try_lock() { + Ok(engine) => break engine, + Err(TryLockError::WouldBlock) => std::thread::sleep(Duration::from_millis(5)), + Err(TryLockError::Poisoned(_)) => { + return Err(error("extension engine lock poisoned")); + } + } + }; + if !self.current(&entry) || !entry.status.read().expect("extension status lock").enabled { + return Err(error("extension is disabled or no longer installed")); + } + let runtime = self + .ready_with_lease(&entry, &mut engine, true, lease.clone()) + .inspect_err(|failure| { + if lease + .as_ref() + .is_none_or(|lease| lease.check_active().is_ok()) + { + self.failed(&entry, failure); + } + })?; + let mut arguments: Vec<Value> = serde_json::from_str(arguments) + .map_err(|failure| cause("invalid provider arguments", failure))?; + if method == "customSearch" { + if arguments.len() < 2 { + arguments.resize(2, Value::Null); + } + if arguments[1].is_null() { + arguments[1] = json!({}); + } + } + if method == "checkAvailability" { + runtime.take_verification_url(); + } + let response = runtime + .call_provider_operation( + method, + &json!(arguments).to_string(), + lease, + timeout_ms, + item_id.into(), + ) + .map_err(|failure| match failure { + ExtensionError::Timeout => error(format!( + "{method} timeout: extension took too long to respond" + )), + ExtensionError::Cancelled(cause) => error(cause.to_string()), + _ if method == "checkAvailability" + && !runtime.take_verification_url().is_empty() => + { + cause("checkAvailability failed", verification_error(id)) + } + other => cause(&format!("{method} failed"), other), + })?; + let mut response: Value = + serde_json::from_str(&response).map_err(|e| error(e.to_string()))?; + if let Some(message) = response.get("parseError").and_then(Value::as_str) { + if method == "getHomeFeed" { + return Err(error(format!("failed to marshal result: {message}"))); + } + let kind = match method { + "searchTracks" | "customSearch" => "search result", + "getAlbum" => "album", + "getPlaylist" => "playlist", + "getArtist" => "artist", + "handleUrl" => "URL handle result", + "fetchLyrics" => "lyrics result", + "checkAvailability" => "availability result", + _ => "track", + }; + return Err(error(format!("failed to parse {kind}: {message}"))); + } + let mut value = response["value"].take(); + if method == "checkAvailability" && (value.is_null() || value["available"] != true) { + if !runtime.take_verification_url().is_empty() { + return Err(verification_error(id)); + } + if value.is_null() { + return Ok(json!({"available":false,"reason":"not implemented"}).to_string()); + } + } + if value.is_null() { + return if method == "customSearch" { + Ok("[]".into()) + } else if method == "handleUrl" { + Err(error("handleUrl returned null - URL not recognized")) + } else { + Err(error(format!("{method} returned null"))) + }; + } + match method { + "getTrack" => stamp(&mut value, id), + "getAlbum" | "getPlaylist" => stamp_album(&mut value, id), + "searchTracks" => stamp_tracks(&mut value["tracks"], id), + "customSearch" => stamp_tracks(&mut value, id), + "getArtist" => { + // Go stamps releases here; albums/top tracks keep their supplied + // provider fields until URL handling performs recursive stamping. + stamp(&mut value, id); + if let Some(releases) = value.get_mut("releases").and_then(Value::as_array_mut) { + for release in releases { + stamp_album(release, id); + } + } + } + "handleUrl" => { + if let Some(track) = value.get_mut("track") { + stamp(track, id); + } + if let Some(tracks) = value.get_mut("tracks") { + stamp_tracks(tracks, id); + } + if let Some(album) = value.get_mut("album") { + stamp_album(album, id); + } + if let Some(artist) = value.get_mut("artist") { + stamp(artist, id); + for key in ["albums", "releases"] { + if let Some(albums) = artist.get_mut(key).and_then(Value::as_array_mut) { + for album in albums { + stamp_album(album, id); + } + } + } + if let Some(tracks) = artist.get_mut("top_tracks") { + stamp_tracks(tracks, id); + } + } + } + "fetchLyrics" => { + value["source"] = format!("Extension: {id}").into(); + if value["provider"] == "" { + value["provider"] = manifest.display_name.clone().into(); + } + if value["lines"].as_array().is_some_and(Vec::is_empty) { + let plain = value["plainLyrics"].as_str().unwrap_or("").to_owned(); + value["lines"] = Value::Null; + if !plain.is_empty() && value["instrumental"] != true { + value["syncType"] = "UNSYNCED".into(); + let lines: Vec<_> = plain + .split('\n') + .filter(|line| !line.trim().is_empty()) + .map(|line| json!({"startTimeMs":0,"words":line,"endTimeMs":0})) + .collect(); + if !lines.is_empty() { + value["lines"] = lines.into(); + } + } + } + } + _ => {} + } + Ok(value.to_string()) + } +} + +fn verification_error(id: &str) -> ManagerError { + error(format!( + "verification_required: extension '{id}' needs signed-session verification" + )) +} + +fn stamp(value: &mut Value, id: &str) { + value["provider_id"] = id.into(); +} +fn stamp_tracks(value: &mut Value, id: &str) { + if let Some(tracks) = value.as_array_mut() { + for track in tracks { + stamp(track, id); + } + } +} +fn stamp_album(value: &mut Value, id: &str) { + stamp(value, id); + stamp_tracks(&mut value["tracks"], id); +} diff --git a/rust_backend/crates/extensions/src/manifest.rs b/rust_backend/crates/extensions/src/manifest.rs new file mode 100644 index 00000000..bf548da3 --- /dev/null +++ b/rust_backend/crates/extensions/src/manifest.rs @@ -0,0 +1,582 @@ +//! The installed extension manifest contract, independent of JavaScript execution. + +use crate::storage::valid_extension_id; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::{Map, Value}; +use spotiflac_network::url::UrlParts; + +trait GoNull { + fn set_null(&mut self); +} + +impl GoNull for String { + fn set_null(&mut self) {} +} +impl GoNull for bool { + fn set_null(&mut self) {} +} +impl GoNull for isize { + fn set_null(&mut self) {} +} +impl<T> GoNull for Vec<T> { + fn set_null(&mut self) { + self.clear(); + } +} +impl<T> GoNull for Option<T> { + fn set_null(&mut self) { + *self = None; + } +} +impl GoNull for Map<String, Value> { + fn set_null(&mut self) { + self.clear(); + } +} +impl GoNull for Value { + fn set_null(&mut self) { + *self = Value::Null; + } +} + +fn is_false(value: &bool) -> bool { + !value +} +fn is_zero(value: &isize) -> bool { + *value == 0 +} + +// Go treats missing and null scalar fields as their zero values. Keeping wire +// names explicit also prevents Rust naming changes from changing mobile JSON. +macro_rules! manifest_struct { + ($name:ident { $($(#[$attr:meta])* $field:ident: $ty:ty => $wire:literal),* $(,)? }) => { + #[derive(Clone, Debug, Default, Serialize)] + pub struct $name { + $( + $(#[$attr])* + #[serde(rename = $wire)] + pub $field: $ty, + )* + } + impl GoNull for $name { fn set_null(&mut self) {} } + impl<'de> Deserialize<'de> for $name { + fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = $name; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a manifest object") + } + fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> { + Ok($name::default()) + } + fn visit_map<A: serde::de::MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> { + let mut result = $name::default(); + while let Some(key) = map.next_key::<String>()? { + $( + if key.eq_ignore_ascii_case($wire) { + match map.next_value::<Option<$ty>>()? { + Some(value) => result.$field = value, + None => result.$field.set_null(), + } + continue; + } + )* + map.next_value::<serde::de::IgnoredAny>()?; + } + Ok(result) + } + } + deserializer.deserialize_any(Visitor) + } + } + }; +} + +manifest_struct!(ExtensionPermissions { + network: Option<Vec<String>> => "network", + storage: bool => "storage", + file: bool => "file", + #[serde(skip_serializing_if = "is_false")] + allow_http: bool => "allowHttp", +}); + +manifest_struct!(ExtensionSetting { + key: String => "key", kind: String => "type", label: String => "label", + #[serde(skip_serializing_if = "String::is_empty")] + description: String => "description", + #[serde(skip_serializing_if = "is_false")] + required: bool => "required", + #[serde(skip_serializing_if = "is_false")] + secret: bool => "secret", + #[serde(skip_serializing_if = "Value::is_null")] + default: Value => "default", + #[serde(skip_serializing_if = "Vec::is_empty")] + options: Vec<String> => "options", + #[serde(skip_serializing_if = "String::is_empty")] + action: String => "action", +}); + +manifest_struct!(QualitySpecificSetting { + key: String => "key", kind: String => "type", label: String => "label", + #[serde(skip_serializing_if = "String::is_empty")] + description: String => "description", + #[serde(skip_serializing_if = "is_false")] + required: bool => "required", + #[serde(skip_serializing_if = "is_false")] + secret: bool => "secret", + #[serde(skip_serializing_if = "Value::is_null")] + default: Value => "default", + #[serde(skip_serializing_if = "Vec::is_empty")] + options: Vec<String> => "options", +}); + +manifest_struct!(QualityOption { + id: String => "id", + #[serde(skip_serializing_if = "String::is_empty")] + kind: String => "kind", + label: String => "label", description: String => "description", + #[serde(skip_serializing_if = "Vec::is_empty")] + settings: Vec<QualitySpecificSetting> => "settings", +}); + +manifest_struct!(SearchFilter { + id: String => "id", + #[serde(skip_serializing_if = "String::is_empty")] + label: String => "label", + #[serde(skip_serializing_if = "String::is_empty")] + icon: String => "icon", +}); + +manifest_struct!(SearchBehavior { + enabled: bool => "enabled", + #[serde(skip_serializing_if = "String::is_empty")] + placeholder: String => "placeholder", + #[serde(skip_serializing_if = "is_false")] + primary: bool => "primary", + #[serde(skip_serializing_if = "String::is_empty")] + icon: String => "icon", + #[serde(skip_serializing_if = "String::is_empty")] + thumbnail_ratio: String => "thumbnailRatio", + #[serde(skip_serializing_if = "is_zero")] + thumbnail_width: isize => "thumbnailWidth", + #[serde(skip_serializing_if = "is_zero")] + thumbnail_height: isize => "thumbnailHeight", + #[serde(skip_serializing_if = "Vec::is_empty")] + filters: Vec<SearchFilter> => "filters", +}); + +manifest_struct!(UrlHandler { + enabled: bool => "enabled", + #[serde(skip_serializing_if = "Vec::is_empty")] + patterns: Vec<String> => "patterns", +}); + +manifest_struct!(TrackMatching { + custom_matching: bool => "customMatching", + #[serde(skip_serializing_if = "String::is_empty")] + strategy: String => "strategy", + #[serde(skip_serializing_if = "is_zero")] + duration_tolerance: isize => "durationTolerance", +}); + +manifest_struct!(PostProcessingHook { + id: String => "id", name: String => "name", + #[serde(skip_serializing_if = "String::is_empty")] + description: String => "description", + #[serde(skip_serializing_if = "is_false")] + default_enabled: bool => "defaultEnabled", + #[serde(skip_serializing_if = "Vec::is_empty")] + supported_formats: Vec<String> => "supportedFormats", +}); + +manifest_struct!(PostProcessing { + enabled: bool => "enabled", + #[serde(skip_serializing_if = "Vec::is_empty")] + hooks: Vec<PostProcessingHook> => "hooks", +}); + +manifest_struct!(HealthCheck { + id: String => "id", + #[serde(skip_serializing_if = "String::is_empty")] + label: String => "label", + url: String => "url", + #[serde(skip_serializing_if = "String::is_empty")] + method: String => "method", + #[serde(skip_serializing_if = "String::is_empty")] + service_key: String => "serviceKey", + #[serde(skip_serializing_if = "is_zero")] + timeout_ms: isize => "timeoutMs", + #[serde(skip_serializing_if = "is_zero")] + cache_ttl_seconds: isize => "cacheTtlSeconds", + #[serde(skip_serializing_if = "is_false")] + required: bool => "required", +}); + +manifest_struct!(SignedSessionEndpoints { + #[serde(skip_serializing_if = "String::is_empty")] + bootstrap: String => "bootstrap", + #[serde(skip_serializing_if = "String::is_empty")] + challenge: String => "challenge", + #[serde(skip_serializing_if = "String::is_empty")] + exchange: String => "exchange", + #[serde(skip_serializing_if = "String::is_empty")] + refresh: String => "refresh", +}); + +manifest_struct!(SignedSession { + namespace: String => "namespace", base_url: String => "baseUrl", + #[serde(skip_serializing_if = "String::is_empty")] + app_version: String => "appVersion", + #[serde(skip_serializing_if = "String::is_empty")] + platform: String => "platform", + #[serde(skip_serializing_if = "String::is_empty")] + callback_url: String => "callbackUrl", + #[serde(skip_serializing_if = "String::is_empty")] + scheme_label: String => "schemeLabel", + #[serde(skip_serializing_if = "String::is_empty")] + header_prefix: String => "headerPrefix", + #[serde(skip_serializing_if = "is_zero")] + time_window_seconds: isize => "timeWindowSeconds", + endpoints: SignedSessionEndpoints => "endpoints", +}); + +manifest_struct!(ExtensionManifest { + name: String => "name", display_name: String => "displayName", + version: String => "version", description: String => "description", + #[serde(skip_serializing_if = "String::is_empty")] + homepage: String => "homepage", + #[serde(skip_serializing_if = "String::is_empty")] + icon: String => "icon", + types: Vec<String> => "type", + permissions: ExtensionPermissions => "permissions", + #[serde(skip_serializing_if = "Vec::is_empty")] + settings: Vec<ExtensionSetting> => "settings", + #[serde(skip_serializing_if = "Vec::is_empty")] + quality_options: Vec<QualityOption> => "qualityOptions", + #[serde(skip_serializing_if = "String::is_empty")] + min_app_version: String => "minAppVersion", + #[serde(skip_serializing_if = "is_false")] + skip_metadata_enrichment: bool => "skipMetadataEnrichment", + #[serde(skip_serializing_if = "is_false")] + skip_lyrics: bool => "skipLyrics", + #[serde(skip_serializing_if = "is_false")] + stop_provider_fallback: bool => "stopProviderFallback", + #[serde(skip_serializing_if = "is_false")] + skip_built_in_fallback: bool => "skipBuiltInFallback", + #[serde(skip_serializing_if = "Option::is_none")] + search_behavior: Option<SearchBehavior> => "searchBehavior", + #[serde(skip_serializing_if = "Option::is_none")] + url_handler: Option<UrlHandler> => "urlHandler", + #[serde(skip_serializing_if = "Option::is_none")] + track_matching: Option<TrackMatching> => "trackMatching", + #[serde(skip_serializing_if = "Option::is_none")] + post_processing: Option<PostProcessing> => "postProcessing", + #[serde(skip_serializing_if = "Vec::is_empty")] + service_health: Vec<HealthCheck> => "serviceHealth", + #[serde(skip_serializing_if = "Option::is_none")] + signed_session: Option<SignedSession> => "signedSession", + #[serde(skip_serializing_if = "Vec::is_empty")] + required_runtime_features: Vec<String> => "requiredRuntimeFeatures", + #[serde(skip_serializing_if = "Map::is_empty")] + capabilities: Map<String, Value> => "capabilities", +}); + +#[derive(Debug, thiserror::Error)] +pub enum ManifestError { + #[error("failed to parse manifest JSON: {0}")] + Json(#[from] serde_json::Error), + #[error("manifest validation error: {field} - {message}")] + Validation { field: String, message: String }, +} + +fn invalid(field: impl Into<String>, message: impl Into<String>) -> ManifestError { + ManifestError::Validation { + field: field.into(), + message: message.into(), + } +} + +fn extension_quality_kind(option: &QualityOption, manifest: Option<&ExtensionManifest>) -> String { + let kind = option.kind.trim().to_ascii_lowercase(); + if matches!(kind.as_str(), "lossless" | "lossy" | "spatial") { + return kind; + } + + // Installed manifests from before the kind field was introduced need the + // same ID/label inference as the Go backend. Descriptions are deliberately + // excluded because they commonly mention fallback formats. + let token = option.id.trim().to_ascii_lowercase(); + let text = format!("{token} {}", option.label.to_ascii_lowercase()); + if text.contains("atmos") + || text.contains("dolby") + || text.contains("surround") + || matches!(token.as_str(), "ac4" | "ac-4" | "eac3" | "e-ac-3" | "ec-3") + { + return "spatial".to_owned(); + } + if text.contains("lossless") + || text.contains("flac") + || text.contains("alac") + || text.contains("24-bit") + || text.contains("16-bit") + || token == "hi_res" + { + return "lossless".to_owned(); + } + if token == "high" + || token == "low" + || text.contains("mp3") + || text.contains("aac") + || text.contains("opus") + || text.contains("vorbis") + { + return "lossy".to_owned(); + } + if matches!(token.as_str(), "best" | "default" | "") + && let Some(tier) = manifest + .and_then(|value| value.capabilities.get("downloadFallbackTier")) + .and_then(Value::as_str) + { + match tier.trim().to_ascii_lowercase().as_str() { + "hi_res" | "lossless" => return "lossless".to_owned(), + "low_res" => return "lossy".to_owned(), + _ => {} + } + } + String::new() +} + +impl ExtensionManifest { + pub fn parse(json: &str) -> Result<Self, ManifestError> { + let manifest: Self = serde_json::from_str(json)?; + manifest.validate()?; + Ok(manifest) + } + + pub fn validate(&self) -> Result<(), ManifestError> { + if self.name.trim().is_empty() { + return Err(invalid("name", "name is required")); + } + if !valid_extension_id(&self.name) { + return Err(invalid( + "name", + "name must be a lowercase extension ID containing only letters, numbers, '.', '_' or '-'", + )); + } + if self.version.trim().is_empty() { + return Err(invalid("version", "version is required")); + } + if self.description.trim().is_empty() { + return Err(invalid("description", "description is required")); + } + if self.types.is_empty() { + return Err(invalid("type", "at least one type is required")); + } + for kind in &self.types { + if !matches!( + kind.as_str(), + "metadata_provider" | "download_provider" | "lyrics_provider" + ) { + return Err(invalid( + "type", + format!( + "invalid extension type: {kind} (must be 'metadata_provider', 'download_provider', or 'lyrics_provider')" + ), + )); + } + } + for (index, setting) in self.settings.iter().enumerate() { + let failure = if setting.key.trim().is_empty() { + Some(("key", "setting key is required")) + } else if setting.kind.is_empty() { + Some(("type", "setting type is required")) + } else if setting.kind == "select" && setting.options.is_empty() { + Some(("options", "select type requires options")) + } else if setting.kind == "button" && setting.action.is_empty() { + Some(("action", "button type requires action (JS function name)")) + } else { + None + }; + if let Some((field, message)) = failure { + return Err(invalid(format!("settings[{index}].{field}"), message)); + } + } + for (index, quality) in self.quality_options.iter().enumerate() { + if !matches!(quality.kind.as_str(), "" | "lossless" | "lossy" | "spatial") { + return Err(invalid( + format!("qualityOptions[{index}].kind"), + "quality kind must be lossless, lossy, or spatial", + )); + } + } + for (index, check) in self.service_health.iter().enumerate() { + let failure = if check.id.trim().is_empty() { + Some(("id", "health check id is required")) + } else if check.url.trim().is_empty() { + Some(("url", "health check url is required")) + } else if !matches!( + check.method.trim().to_uppercase().as_str(), + "" | "GET" | "HEAD" + ) { + Some(("method", "health check method must be GET or HEAD")) + } else { + None + }; + if let Some((field, message)) = failure { + return Err(invalid(format!("serviceHealth[{index}].{field}"), message)); + } + } + if let Some(session) = &self.signed_session { + if !self.permissions.storage { + return Err(invalid( + "permissions.storage", + "signedSession requires storage permission", + )); + } + if session.namespace.trim().is_empty() { + return Err(invalid("signedSession.namespace", "namespace is required")); + } + let base = session.base_url.trim(); + if base.is_empty() { + return Err(invalid("signedSession.baseUrl", "baseUrl is required")); + } + if !base.to_lowercase().starts_with("https://") { + return Err(invalid("signedSession.baseUrl", "baseUrl must use https")); + } + let parsed = UrlParts::parse(base) + .filter(|url| !url.hostname.is_empty()) + .ok_or_else(|| invalid("signedSession.baseUrl", "baseUrl is invalid"))?; + if !self.is_domain_allowed(&parsed.hostname) { + return Err(invalid( + "signedSession.baseUrl", + "baseUrl host must be listed in permissions.network", + )); + } + } + if self.has_capability("rawFfmpeg") && !self.permissions.file { + return Err(invalid( + "permissions.file", + "rawFfmpeg capability requires file permission", + )); + } + crate::transfer_policy::validate(&self.capabilities) + .map_err(|message| invalid("capabilities.downloadTransfer", message))?; + Ok(()) + } + + pub fn has_capability(&self, name: &str) -> bool { + self.capabilities.get(name) == Some(&Value::Bool(true)) + } + + pub fn find_quality(&self, requested: &str) -> Option<&QualityOption> { + let requested = requested.trim(); + if requested.is_empty() { + return None; + } + self.quality_options + .iter() + .find(|option| option.id.trim().eq_ignore_ascii_case(requested)) + } + + pub fn resolve_download_quality( + &self, + requested: &str, + source: Option<&ExtensionManifest>, + ) -> Result<String, String> { + let requested = requested.trim(); + if self.quality_options.is_empty() { + return Ok(requested.to_owned()); + } + + let source_quality = source.and_then(|manifest| manifest.find_quality(requested)); + let source_option = source_quality.cloned().unwrap_or_else(|| QualityOption { + id: requested.to_owned(), + ..QualityOption::default() + }); + let kind = extension_quality_kind(&source_option, source); + + if let Some(exact) = self.find_quality(requested) { + let target_kind = extension_quality_kind(exact, Some(self)); + if source.is_some_and(|manifest| manifest.name == self.name) + || (!kind.is_empty() && kind == target_kind) + || (kind.is_empty() && target_kind != "spatial") + { + return Ok(exact.id.trim().to_owned()); + } + } + + let kind = if kind.is_empty() { + "lossless" + } else { + kind.as_str() + }; + let allowed_kinds: Vec<&str> = if matches!(kind, "spatial" | "lossy") { + vec![kind, "lossless"] + } else { + vec![kind] + }; + for allowed in allowed_kinds { + if let Some(candidate) = self.quality_options.iter().find(|candidate| { + let id = candidate.id.trim(); + !id.is_empty() && extension_quality_kind(candidate, Some(self)) == *allowed + }) { + return Ok(candidate.id.trim().to_owned()); + } + } + Err(format!( + "provider {} has no compatible {} quality for {:?}", + self.name, kind, requested + )) + } + + pub fn has_type(&self, kind: &str) -> bool { + self.types.iter().any(|value| value == kind) + } + pub fn stops_provider_fallback(&self) -> bool { + self.stop_provider_fallback || self.skip_built_in_fallback + } + + pub fn is_domain_allowed(&self, domain: &str) -> bool { + let domain = domain.trim().to_lowercase(); + self.permissions + .network + .as_deref() + .unwrap_or_default() + .iter() + .any(|allowed| { + let allowed = allowed.trim().to_lowercase(); + allowed == domain || (allowed.starts_with("*.") && domain.ends_with(&allowed[1..])) + }) + } + + pub fn matches_url(&self, url: &str) -> bool { + let Some(handler) = &self.url_handler else { + return false; + }; + if !handler.enabled { + return false; + } + let url = url.trim().to_lowercase(); + let parsed = UrlParts::parse(&url); + handler.patterns.iter().any(|pattern| { + let pattern = pattern.trim().to_lowercase(); + if pattern.is_empty() { + return false; + } + if !pattern.contains('/') && pattern.ends_with(':') { + return url.starts_with(&pattern); + } + let Some(parsed) = parsed.as_ref().filter(|url| !url.hostname.is_empty()) else { + return false; + }; + let pattern = pattern + .split_once("://") + .map_or(pattern.as_str(), |(_, rest)| rest); + let (host, path) = pattern.split_once('/').unwrap_or((pattern, "")); + !host.is_empty() + && (parsed.hostname == host || parsed.hostname.ends_with(&format!(".{host}"))) + && (path.is_empty() || parsed.path.starts_with(format!("/{path}").as_bytes())) + }) + } +} diff --git a/rust_backend/crates/extensions/src/network_host.rs b/rust_backend/crates/extensions/src/network_host.rs new file mode 100644 index 00000000..a72d36cb --- /dev/null +++ b/rust_backend/crates/extensions/src/network_host.rs @@ -0,0 +1,194 @@ +use crate::host::decode_go_utf8; +use crate::runtime::{Control, ExtensionServices}; +use base64::Engine; +use rquickjs::{Array, Ctx, Function, Object, TypedArray, Value}; +use spotiflac_network::{HttpRequest, HttpResponse}; +use std::collections::BTreeMap; +use std::sync::Arc; + +pub(crate) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + host.set("networkEnabled", services.network.is_some())?; + let Some(session) = &services.network else { + return Ok(()); + }; + let validator = Arc::clone(session); + host.set( + "validateURL", + Function::new(ctx.clone(), move |url: String| { + validator.validate_url(&url).err() + })?, + )?; + let clearer = Arc::clone(session); + host.set( + "clearCookies", + Function::new(ctx.clone(), move || { + clearer.clear_cookies(); + true + })?, + )?; + let session = Arc::clone(session); + host.set( + "request", + Function::new( + ctx.clone(), + move |url: String, + method: String, + body: String, + headers: Object<'js>, + default_json: bool, + user_agent: String, + fetch: bool| + -> rquickjs::Result<Object<'js>> { + let ctx = headers.ctx().clone(); + let headers = headers + .props::<String, String>() + .collect::<rquickjs::Result<BTreeMap<_, _>>>()?; + let result = session.request( + HttpRequest { + url, + method, + body, + headers, + default_json, + user_agent, + }, + || control.check().map_err(|error| error.to_string()), + ); + match result { + Ok(response) => response_object(&ctx, response, fetch), + Err(error) => { + let object = Object::new(ctx)?; + object.set("error", error)?; + Ok(object) + } + } + }, + )?, + )?; + host.set( + "decodeBuffer", + Function::new(ctx.clone(), |bytes: TypedArray<'js, u8>| { + decode_buffer(&bytes) + })?, + )?; + host.set( + "parseJSONBuffer", + Function::new(ctx.clone(), |ctx: Ctx<'js>, bytes: TypedArray<'js, u8>| { + parse_json(&ctx, &decode_buffer(&bytes)) + })?, + )?; + host.set( + "encodeBuffer", + Function::new(ctx.clone(), |bytes: TypedArray<'js, u8>| { + encode_buffer(&bytes) + })?, + )?; + Ok(()) +} + +fn parse_json<'js>(ctx: &Ctx<'js>, text: &str) -> rquickjs::Result<Value<'js>> { + let text = spotiflac_core::normalize_json_surrogates(text); + let bytes = text.as_bytes(); + let (mut position, mut depth) = (0, 0usize); + // Go validates every float64 before overwriting duplicate object keys. + // The engine supplies syntax validation and allocates directly in its heap. + while position < bytes.len() { + match bytes[position] { + b'"' => { + position += 1; + while position < bytes.len() && bytes[position] != b'"' { + position += if bytes[position] == b'\\' { 2 } else { 1 }; + } + } + b'[' | b'{' => { + depth += 1; + if depth > 10_000 { + return Err(rquickjs::Exception::throw_syntax( + ctx, + "JSON exceeds maximum depth", + )); + } + } + b']' | b'}' => depth = depth.saturating_sub(1), + b'-' | b'0'..=b'9' => { + let start = position; + while position < bytes.len() + && matches!( + bytes[position], + b'0'..=b'9' | b'-' | b'+' | b'.' | b'e' | b'E' + ) + { + position += 1; + } + if !text[start..position] + .parse::<f64>() + .is_ok_and(f64::is_finite) + { + return Err(rquickjs::Exception::throw_syntax( + ctx, + "invalid JSON number", + )); + } + continue; + } + _ => {} + } + position += 1; + } + ctx.json_parse(text.as_bytes()) +} + +fn response_object<'js>( + ctx: &Ctx<'js>, + response: HttpResponse, + fetch: bool, +) -> rquickjs::Result<Object<'js>> { + let object = Object::new(ctx.clone())?; + object.set("status", response.status)?; + object.set("ok", (200..300).contains(&response.status))?; + object.set("url", response.url)?; + let headers = Object::new(ctx.clone())?; + for (key, values) in response.headers { + if values.len() == 1 { + headers.set(key, &values[0])?; + } else { + let array = Array::new(ctx.clone())?; + for (index, value) in values.into_iter().enumerate() { + array.set(index, value)?; + } + headers.set(key, array)?; + } + } + object.set("headers", headers)?; + if fetch { + object.set("statusText", response.status_text)?; + // Copy into engine-owned memory so retained responses count against the + // VM heap limit. No base64 intermediate or per-byte JS Number array. + object.set("bytes", TypedArray::new_copy(ctx.clone(), response.body)?)?; + } else { + object.set("statusCode", response.status)?; + object.set("body", decode_go_utf8(&response.body))?; + } + Ok(object) +} + +#[allow(unsafe_code)] +fn decode_buffer(bytes: &TypedArray<'_, u8>) -> String { + // SAFETY: the VM has one worker and exposes no shared-memory workers. No + // JavaScript or engine operation runs while borrowing its buffer. The owned + // Rust String is completed before returning through rquickjs into JavaScript. + let bytes = unsafe { bytes.as_bytes() }.unwrap_or_default(); + decode_go_utf8(bytes) +} + +#[allow(unsafe_code)] +fn encode_buffer(bytes: &TypedArray<'_, u8>) -> String { + // SAFETY: same single-worker, no-engine-call borrowing rule as decode_buffer. + let bytes = unsafe { bytes.as_bytes() }.unwrap_or_default(); + base64::engine::general_purpose::STANDARD.encode(bytes) +} diff --git a/rust_backend/crates/extensions/src/package.rs b/rust_backend/crates/extensions/src/package.rs new file mode 100644 index 00000000..0606eeb4 --- /dev/null +++ b/rust_backend/crates/extensions/src/package.rs @@ -0,0 +1,335 @@ +//! Inspect before extraction; only a private staging directory receives files. + +use crate::host::decode_go_utf8; +use crate::manifest::ExtensionManifest; +use spotiflac_core::matching::lowercase; +use std::collections::HashSet; +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::os::unix::ffi::OsStringExt; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use zip::ZipArchive; + +const MAX_ENTRIES: u64 = 2048; +const MAX_EXTRACTED: u64 = 256 * 1024 * 1024; +const MAX_MANIFEST: u64 = 1024 * 1024; +const INVALID_ARCHIVE: &str = + "cannot open extension file: the file may be corrupted or not a valid extension package"; + +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +pub struct PackageError(pub String); + +fn error(message: impl Into<String>) -> PackageError { + PackageError(message.into()) +} +fn corrupt(_: impl std::fmt::Display) -> PackageError { + error(INVALID_ARCHIVE) +} +fn io_error(message: &str, cause: impl std::fmt::Display) -> PackageError { + error(format!("{message}: {cause}")) +} + +pub fn is_package_path(path: &Path) -> bool { + let name = lowercase(&path.to_string_lossy()); + name.ends_with(".spotiflac-ext") || name.ends_with(".sflx") +} + +struct Entry { + path: PathBuf, + directory: bool, + size: u64, +} + +pub struct ExtensionPackage { + archive: ZipArchive<File>, + entries: Vec<Entry>, + pub manifest: ExtensionManifest, + pub manifest_json: String, +} + +impl ExtensionPackage { + pub fn open(path: &Path) -> Result<Self, PackageError> { + if !is_package_path(path) { + return Err(error( + "invalid file format: please select a .spotiflac-ext or .sflx file", + )); + } + let mut input = OpenOptions::new() + .read(true) + .custom_flags(rustix::fs::OFlags::NONBLOCK.bits() as i32) + .open(path) + .map_err(corrupt)?; + if !input.metadata().map_err(corrupt)?.is_file() { + return Err(error(INVALID_ARCHIVE)); + } + // zip-rs coalesces duplicate raw names. Inspect the original central + // directory first, and bound its entry count before it allocates metadata. + let names = central_names(&mut input)?; + let mut archive = ZipArchive::new(input).map_err(corrupt)?; + if archive.len() != names.len() { + return Err(error(INVALID_ARCHIVE)); + } + let mut entries = Vec::with_capacity(names.len()); + let mut total = 0u64; + let mut manifest_index = None; + let mut has_index = false; + for (index, name) in names.into_iter().enumerate() { + let file = archive.by_index_raw(index).map_err(corrupt)?; + let mode = file.unix_mode().unwrap_or(0) & 0o170000; + if mode == 0o120000 { + return Err(error(format!( + "unsafe path in extension archive: {}", + decode_go_utf8(&name) + ))); + } + let path = clean_path(&name)?; + let directory = name.ends_with(b"/") || mode == 0o040000; + if !directory { + if file.size() > MAX_EXTRACTED - total { + return Err(error( + "extension archive exceeds the 256 MiB extracted size limit", + )); + } + total += file.size(); + } + if path == Path::new("manifest.json") && !directory { + manifest_index = Some(index); + } + if path == Path::new("index.js") && !directory { + has_index = true; + } + entries.push(Entry { + path, + directory, + size: file.size(), + }); + } + let index = manifest_index + .ok_or_else(|| error("invalid extension package: root manifest.json not found"))?; + if !has_index { + return Err(error("invalid extension package: root index.js not found")); + } + if entries[index].size > MAX_MANIFEST { + return Err(error( + "invalid extension package: manifest.json is too large", + )); + } + let mut bytes = Vec::new(); + archive + .by_index(index) + .map_err(|e| io_error("failed to open manifest.json", e))? + .take(MAX_MANIFEST + 1) + .read_to_end(&mut bytes) + .map_err(|e| io_error("failed to read manifest.json", e))?; + if bytes.len() as u64 > MAX_MANIFEST { + return Err(error( + "invalid extension package: manifest.json is too large", + )); + } + let manifest_json = decode_go_utf8(&bytes); + let manifest = ExtensionManifest::parse(&manifest_json) + .map_err(|e| io_error("invalid extension manifest", e))?; + Ok(Self { + archive, + entries, + manifest, + manifest_json, + }) + } + + pub fn extract( + &mut self, + destination: &Path, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<(), PackageError> { + let metadata = fs::symlink_metadata(destination) + .map_err(|e| io_error("failed to inspect staging directory", e))?; + if !metadata.is_dir() || fs::read_dir(destination).map_err(corrupt)?.next().is_some() { + return Err(error( + "extension extraction requires an empty staging directory", + )); + } + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + for (index, entry) in self.entries.iter().enumerate() { + check().map_err(error)?; + if entry.directory { + continue; + } + let path = destination.join(&entry.path); + fs::create_dir_all(path.parent().expect("staging parent")) + .map_err(|e| io_error("failed to create extension directory", e))?; + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .map_err(|e| io_error("failed to create extension file", e))?; + let mut input = self + .archive + .by_index(index) + .map_err(|e| io_error("failed to open file in archive", e))?; + let mut written = 0u64; + loop { + check().map_err(error)?; + let count = input + .read(&mut buffer) + .map_err(|e| io_error("failed to extract extension file", e))?; + if count == 0 { + break; + } + if count as u64 > MAX_EXTRACTED - total + || count as u64 > entry.size.saturating_sub(written) + { + return Err(error( + "extension archive exceeds its declared extracted size", + )); + } + output + .write_all(&buffer[..count]) + .map_err(|e| io_error("failed to extract extension file", e))?; + total += count as u64; + written += count as u64; + } + if written != entry.size { + return Err(error("failed to extract extension file: unexpected EOF")); + } + output + .sync_all() + .map_err(|e| io_error("failed to close extracted extension file", e))?; + } + Ok(()) + } +} + +fn clean_path(name: &[u8]) -> Result<PathBuf, PackageError> { + let unsafe_path = || { + error(format!( + "unsafe path in extension archive: {}", + decode_go_utf8(name) + )) + }; + if name.starts_with(b"/") || name.contains(&b'\\') || name.contains(&0) { + return Err(unsafe_path()); + } + let mut parts = Vec::new(); + for part in name.split(|byte| *byte == b'/') { + match part { + b"" | b"." => {} + b".." => { + if parts.pop().is_none() { + return Err(unsafe_path()); + } + } + _ => parts.push(part), + } + } + if parts.is_empty() { + return Err(unsafe_path()); + } + Ok(PathBuf::from(OsString::from_vec(parts.join(&b'/')))) +} + +fn number16(bytes: &[u8], offset: usize) -> u64 { + u16::from_le_bytes(bytes[offset..offset + 2].try_into().unwrap()) as u64 +} +fn number32(bytes: &[u8], offset: usize) -> u64 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) as u64 +} +fn number64(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} +fn read_at<const N: usize>(input: &mut File, offset: u64) -> Result<[u8; N], PackageError> { + input.seek(SeekFrom::Start(offset)).map_err(corrupt)?; + let mut bytes = [0; N]; + input.read_exact(&mut bytes).map_err(corrupt)?; + Ok(bytes) +} + +fn central_names(input: &mut File) -> Result<Vec<Vec<u8>>, PackageError> { + let length = input.metadata().map_err(corrupt)?.len(); + let tail_size = length.min(65557) as usize; + let mut tail = vec![0; tail_size]; + input + .seek(SeekFrom::Start(length - tail_size as u64)) + .map_err(corrupt)?; + input.read_exact(&mut tail).map_err(corrupt)?; + let end = (0..tail.len().saturating_sub(21)) + .rev() + .find(|index| { + tail[*index..].starts_with(b"PK\x05\x06") + && *index + 22 + number16(&tail, *index + 20) as usize <= tail.len() + }) + .ok_or_else(|| error(INVALID_ARCHIVE))?; + let absolute_end = length - tail_size as u64 + end as u64; + let mut count = number16(&tail, end + 10); + let mut size = number32(&tail, end + 12); + let mut offset = number32(&tail, end + 16); + let mut directory_end = absolute_end; + if absolute_end >= 20 { + let locator = read_at::<20>(input, absolute_end - 20)?; + if locator.starts_with(b"PK\x06\x07") { + let record_offset = number64(&locator, 8); + let record = read_at::<56>(input, record_offset)?; + if !record.starts_with(b"PK\x06\x06") { + return Err(error(INVALID_ARCHIVE)); + } + count = number64(&record, 32); + size = number64(&record, 40); + offset = number64(&record, 48); + directory_end = record_offset; + } + } + if count > size / 46 { + return Err(error(INVALID_ARCHIVE)); + } + if count > MAX_ENTRIES { + return Err(error( + "extension archive contains too many entries (maximum 2048)", + )); + } + let inferred = directory_end + .checked_sub(size) + .ok_or_else(|| error(INVALID_ARCHIVE))?; + let mut position = inferred; + if offset < length && read_at::<4>(input, offset)? == *b"PK\x01\x02" { + position = offset; + } + let mut names = Vec::with_capacity(count as usize); + let mut seen = HashSet::new(); + for _ in 0..count { + let header = read_at::<46>(input, position)?; + if !header.starts_with(b"PK\x01\x02") { + return Err(error(INVALID_ARCHIVE)); + } + let name_length = number16(&header, 28); + let mut name = vec![0; name_length as usize]; + input.read_exact(&mut name).map_err(corrupt)?; + let normalized = clean_path(&name)?; + use std::os::unix::ffi::OsStrExt; + let key = lowercase(&decode_go_utf8(normalized.as_os_str().as_bytes())); + if !seen.insert(key) { + return Err(error(format!( + "duplicate path in extension archive: {}", + decode_go_utf8(&name) + ))); + } + position = position + .checked_add(46 + name_length + number16(&header, 30) + number16(&header, 32)) + .filter(|value| *value <= directory_end) + .ok_or_else(|| error(INVALID_ARCHIVE))?; + names.push(name); + } + if position + .checked_add(4) + .is_some_and(|end| end <= directory_end) + && read_at::<4>(input, position)? == *b"PK\x01\x02" + { + return Err(error(INVALID_ARCHIVE)); + } + input.rewind().map_err(corrupt)?; + Ok(names) +} diff --git a/rust_backend/crates/extensions/src/prelude.js b/rust_backend/crates/extensions/src/prelude.js new file mode 100644 index 00000000..ffa99428 --- /dev/null +++ b/rust_backend/crates/extensions/src/prelude.js @@ -0,0 +1,723 @@ +(function (host) { + "use strict"; + const symbolDescription = Object.getOwnPropertyDescriptor(Symbol.prototype, "description").get; + const stringApply = Reflect.apply; + const primitiveSymbol = Symbol.toPrimitive; + const boxedStringValue = String.prototype.valueOf; + const boxedNumberValue = Number.prototype.valueOf; + const boxedBooleanValue = Boolean.prototype.valueOf; + function isPrimitive(value) { + return value === null || (typeof value !== "object" && typeof value !== "function"); + } + function goString(value) { + // Goja Value.String accepts a Symbol returned by an object's coercion. + if (!isPrimitive(value)) { + const exotic = value[primitiveSymbol]; + if (exotic !== undefined && exotic !== null) { + value = stringApply(exotic, value, ["string"]); + if (!isPrimitive(value)) throw new TypeError("cannot convert object to primitive value"); + } else { + let converted = false; + for (const key of ["toString", "valueOf"]) { + const method = value[key]; + if (typeof method !== "function") continue; + const result = stringApply(method, value, []); + if (isPrimitive(result)) { value = result; converted = true; break; } + } + if (!converted) throw new TypeError("cannot convert object to primitive value"); + } + } + if (typeof value === "symbol") return (stringApply(symbolDescription, value, []) || "").toWellFormed(); + return String(value).toWellFormed(); + } + function goNumberPrimitive(value) { + if (isPrimitive(value)) return value; + const exotic = value[primitiveSymbol]; + if (exotic !== undefined && exotic !== null) { + const result = stringApply(exotic, value, ["number"]); + if (isPrimitive(result)) return result; + } else { + for (const key of ["valueOf", "toString"]) { + const method = value[key]; + if (typeof method !== "function") continue; + const result = stringApply(method, value, []); + if (isPrimitive(result)) return result; + } + } + throw new TypeError("cannot convert object to primitive value"); + } + function isMap(value) { + return value !== null && typeof value === "object" && !Array.isArray(value) + && !ArrayBuffer.isView(value) && !(value instanceof Date) + && !(value instanceof Map) && !(value instanceof Set); + } + function formatGo(value) { + if (value === null || value === undefined) return "<nil>"; + if (typeof value === "number") { + if (Number.isNaN(value)) return "NaN"; + if (!Number.isFinite(value)) return value < 0 ? "-Inf" : "+Inf"; + if (Object.is(value, -0)) return "-0"; + } + if (Array.isArray(value) || ArrayBuffer.isView(value)) { + return "[" + Array.from(value, formatGo).join(" ") + "]"; + } + if (isMap(value)) { + return "map[" + host.sortKeys(Object.keys(value)) + .map(key => goString(key) + ":" + formatGo(value[key])).join(" ") + "]"; + } + return goString(value); + } + const byteArrays = new WeakSet(); + const responseByteArrays = new WeakMap(); + function emptyBytes() { + const bytes = []; + byteArrays.add(bytes); + return bytes; + } + function exportValue(value, ancestors) { + if (value === undefined || value === null) return null; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new TypeError("unsupported JSON number"); + return value; + } + if (typeof value === "boolean") return value; + if (typeof value === "string") return value.toWellFormed(); + if (typeof value !== "object") throw new TypeError("unsupported JSON value"); + let stringObject; + try { stringObject = stringApply(boxedStringValue, value, []); } catch (_) {} + let primitive; + try { primitive = stringApply(boxedBooleanValue, value, []); } catch (_) {} + if (typeof primitive === "boolean") return primitive; + try { primitive = stringApply(boxedNumberValue, value, []); } catch (_) {} + if (typeof primitive === "number") return exportValue(primitive, ancestors); + if (ancestors.length >= 128 || ancestors.includes(value)) throw new TypeError("cyclic or deeply nested JSON value"); + if (responseByteArrays.has(value)) return host.encodeBuffer(responseByteArrays.get(value)); + if (byteArrays.has(value) || value instanceof Uint8Array) return host.encodeBytes(Array.from(value)); + if (value instanceof Date) { + if (!Number.isFinite(value.getTime())) return null; + const pad = (number, width = 2) => String(number).padStart(width, "0"); + const year = value.getFullYear(); + if (year < 0 || year > 9999) throw new TypeError("unsupported JSON date"); + const offset = -value.getTimezoneOffset(); + const zone = offset === 0 ? "Z" : (offset < 0 ? "-" : "+") + + pad(Math.trunc(Math.abs(offset) / 60)) + ":" + pad(Math.abs(offset) % 60); + const milliseconds = value.getMilliseconds(); + const fraction = milliseconds ? "." + pad(milliseconds, 3).replace(/0+$/, "") : ""; + return pad(year, 4) + "-" + pad(value.getMonth() + 1) + "-" + pad(value.getDate()) + + "T" + pad(value.getHours()) + ":" + pad(value.getMinutes()) + ":" + + pad(value.getSeconds()) + fraction + zone; + } + if (value instanceof Map) throw new TypeError("unsupported JSON map"); + ancestors.push(value); + let result; + if (Array.isArray(value) || ArrayBuffer.isView(value) || value instanceof Set) { + result = Array.from(value, item => exportValue(item, ancestors)); + } else { + result = Object.create(null); + for (const key of Object.keys(value)) { + if (stringObject !== undefined && /^(0|[1-9][0-9]*)$/.test(key) && Number(key) < stringObject.length) continue; + result[key.toWellFormed()] = exportValue(value[key], ancestors); + } + } + ancestors.pop(); + return result; + } + function serialize(value) { + // Goja exports values before encoding/json. That retains undefined object + // properties as null, encodes byte slices as base64, and ignores toJSON. + function encode(value) { + if (value === null) return "null"; + if (typeof value === "string") return host.quoteJSON(value); + if (typeof value === "number") return Object.is(value, -0) ? "-0" : String(value); + if (typeof value === "boolean") return String(value); + if (Array.isArray(value)) return "[" + value.map(encode).join(",") + "]"; + const fields = host.sortKeys(Object.keys(value)) + .map(key => host.quoteJSON(key) + ":" + encode(value[key])); + return "{" + fields.join(",") + "}"; + } + return encode(exportValue(value, [])); + } + let registered = false; + globalThis.registerExtension = function (value) { + if (arguments.length) { + registered = value !== undefined; + globalThis.extension = value; + } + }; + + if (host.storageEnabled) { + function read(credentials, key, fallback) { + const result = JSON.parse(host.storageRead(credentials, goString(key))); + if (result.error !== undefined) return undefined; + return result.found ? result.value : fallback; + } + function write(credentials, key, value) { + try { + return JSON.parse(host.storageWrite(credentials, goString(key), serialize(value))); + } catch (error) { + return {success: false, error: goString(error)}; + } + } + globalThis.storage = { + get(key, fallback) { return arguments.length ? read(false, key, fallback) : undefined; }, + set(key, value) { return arguments.length >= 2 && write(false, key, value).success; }, + remove(key) { + return arguments.length > 0 && JSON.parse(host.storageRemove(false, goString(key))).success; + } + }; + globalThis.credentials = { + get(key, fallback) { return arguments.length ? read(true, key, fallback) : undefined; }, + store(key, value) { + return arguments.length < 2 ? {success: false, error: "key and value are required"} : write(true, key, value); + }, + remove(key) { + return arguments.length > 0 && JSON.parse(host.storageRemove(true, goString(key))).success; + }, + has(key) { + return arguments.length > 0 && JSON.parse(host.storageRead(true, goString(key))).found === true; + } + }; + } + + if (host.networkEnabled) { + function responseBytes(bytes) { + // Go exposes []byte as a fresh, array-like object sharing the body. + // A sparse array proxy preserves that shape without a Number slot + // per byte; the backing Uint8Array remains charged to the VM heap. + const array = []; + array.length = bytes.length; + function index(key) { + if (typeof key !== "string" || !/^(0|[1-9][0-9]*)$/.test(key)) return -1; + const value = Number(key); + return value < bytes.length ? value : -1; + } + const proxy = new Proxy(array, { + get(target, key, receiver) { + const offset = index(key); + return offset >= 0 && offset < target.length ? bytes[offset] : Reflect.get(target, key, receiver); + }, + set(target, key, value) { + const offset = index(key); + if (offset >= 0) { bytes[offset] = value; return true; } + return Reflect.set(target, key, value); + }, + has(target, key) { return (index(key) >= 0 && index(key) < target.length) || Reflect.has(target, key); }, + ownKeys(target) { + return Array.from({length: Math.min(target.length, bytes.length)}, (_, i) => String(i)) + .concat(Reflect.ownKeys(target).filter(key => index(key) < 0)); + }, + getOwnPropertyDescriptor(target, key) { + const offset = index(key); + return offset >= 0 && offset < target.length + ? {value: bytes[offset], writable: true, enumerable: true, configurable: true} + : Reflect.getOwnPropertyDescriptor(target, key); + } + }); + byteArrays.add(proxy); + responseByteArrays.set(proxy, bytes); + return proxy; + } + function bodyString(value, exported) { + if (value === undefined || value === null) return ""; + if (typeof value === "string") return goString(value); + if (isMap(value) || (Array.isArray(value) && !byteArrays.has(value))) return serialize(value); + return exported ? formatGo(value) : goString(value); + } + function requestHeaders(value) { + const headers = Object.create(null); + if (isMap(value)) { + for (const key of Object.keys(value)) headers[goString(key)] = formatGo(value[key]); + } + return headers; + } + function fetchError(error) { + return {ok: false, status: 0, statusText: "Network Error", error, + text() { return ""; }, json() { return undefined; }}; + } + function dispatch(args, method, fetch, options) { + const fail = error => fetch ? fetchError(error) : {error}; + if (!args.length) return fail("URL is required"); + const url = goString(args[0]); + const denied = host.validateURL(url); + if (denied !== undefined && denied !== null) return fail(denied); + let body = "", headers = {}; + try { + if (options && isMap(args[1])) { + const opts = args[1]; + if (typeof opts.method === "string") method = goString(opts.method).toUpperCase(); + body = bodyString(opts.body, true); + headers = requestHeaders(opts.headers); + } else if (!options) { + if (method === "GET" || method === "DELETE") headers = requestHeaders(args[1]); + else { + body = bodyString(args[1], false); + headers = requestHeaders(args[2]); + } + } + } catch (error) { return fail("failed to stringify body: " + goString(error)); } + const result = host.request(url, method, body, headers, + (!options && method === "POST") || body !== "", + fetch ? host.appUserAgent() : "Spotiflac-Extension/1.0", fetch); + if (result.error !== undefined) return fail(result.error); + if (!fetch) return result; + const bytes = result.bytes; + delete result.bytes; + let text; + result.text = function () { + if (text === undefined) text = host.decodeBuffer(bytes); + return text; + }; + result.json = function () { + try { return host.parseJSONBuffer(bytes); } + catch (_) { return undefined; } + }; + result.arrayBuffer = function () { return responseBytes(bytes); }; + return result; + } + globalThis.http = { + get() { return dispatch(arguments, "GET", false, false); }, + post() { return dispatch(arguments, "POST", false, false); }, + put() { return dispatch(arguments, "PUT", false, false); }, + delete() { return dispatch(arguments, "DELETE", false, false); }, + patch() { return dispatch(arguments, "PATCH", false, false); }, + request() { return dispatch(arguments, "GET", false, true); }, + clearCookies() { return host.clearCookies(); } + }; + globalThis.fetch = function () { return dispatch(arguments, "GET", true, true); }; + if (host.sessionEnabled) { + function invokeSession(method, args) { return JSON.parse(host.sessionCall(method, JSON.stringify(args))); } + globalThis.session = { + status() { return invokeSession("status", []); }, + clear() { return invokeSession("clear", []); }, + completeGrant(grant) { return invokeSession("completeGrant", arguments.length ? [goString(grant)] : []); }, + signedFetch(method, path, body, headers) { + if (arguments.length < 2) return invokeSession("signedFetch", []); + try { return invokeSession("signedFetch", [goString(method), goString(path), bodyString(body, false), requestHeaders(headers)]); } + catch (error) { return {ok: false, error: goString(error)}; } + } + }; + } + if (host.authEnabled) { + function invokeAuth(method, args, expiresIsFloat = false) { + const result = JSON.parse(host.authCall(method, JSON.stringify(args), expiresIsFloat)); + return method === "getAuthCode" && result === null ? undefined : result; + } + function goFloat(value) { + return typeof value === "number" && (!Number.isInteger(value) || Object.is(value, -0) + || value < -9223372036854775808 || value >= 9223372036854775808); + } + function authConfig(args) { + if (!args.length) return []; + if (!isMap(args[0])) return [null]; + const config = Object.create(null); + for (const key of ["authUrl", "clientId", "redirectUri", "scope", "tokenUrl", "code"]) { + if (typeof args[0][key] === "string") config[key] = goString(args[0][key]); + } + if (isMap(args[0].extraParams)) config.extraParams = requestHeaders(args[0].extraParams); + return [config]; + } + globalThis.auth = { + openAuthUrl(url, callback) { + const args = []; + if (arguments.length) args.push(goString(url)); + if (arguments.length > 1 && callback !== undefined) args.push(goString(callback)); + return invokeAuth("openAuthUrl", args); + }, + getAuthCode() { return invokeAuth("getAuthCode", []); }, + setAuthCode(value) { + if (!arguments.length) return invokeAuth("setAuthCode", []); + if (typeof value === "string") return invokeAuth("setAuthCode", [goString(value)]); + const config = Object.create(null); + if (isMap(value)) { + for (const key of ["code", "access_token", "refresh_token"]) { + if (typeof value[key] === "string") config[key] = goString(value[key]); + } + if (typeof value.expires_in === "number") config.expires_in = value.expires_in; + } + const expiresIsFloat = goFloat(config.expires_in); + // JSON cannot carry NaN/Infinity. Go's duration conversion + // makes these already expired; retain that outcome. + if (expiresIsFloat && !Number.isFinite(config.expires_in)) config.expires_in = 0; + return invokeAuth("setAuthCode", [config], expiresIsFloat); + }, + clearAuth() { return invokeAuth("clearAuth", []); }, + isAuthenticated() { return invokeAuth("isAuthenticated", []); }, + getTokens() { return invokeAuth("getTokens", []); }, + generatePKCE(length) { + // Goja exports integral JS values as int64; this legacy host + // only honors float64 lengths, otherwise it defaults to 64. + const size = goFloat(length) && length >= 43 && length <= 128 ? Math.trunc(length) : 64; + return invokeAuth("generatePKCE", [size]); + }, + getPKCE() { return invokeAuth("getPKCE", []); }, + startOAuthWithPKCE() { return invokeAuth("startOAuthWithPKCE", authConfig(arguments)); }, + exchangeCodeWithPKCE() { return invokeAuth("exchangeCodeWithPKCE", authConfig(arguments)); } + }; + } + } + + function binaryOptions(value) { + if (value === null || typeof value !== "object" || Array.isArray(value) || ArrayBuffer.isView(value) + || value instanceof ArrayBuffer || value instanceof Date || value instanceof Map || value instanceof Set) return null; + const result = Object.create(null); + for (const key of Object.keys(value)) { + let item = value[key]; + if (typeof item === "string") item = goString(item); + else if (responseByteArrays.has(item)) item = responseByteArrays.get(item); + else if (item instanceof Uint8ClampedArray) item = new Uint8Array(item.buffer, item.byteOffset, item.byteLength); + if (key === "segments" && Array.isArray(item)) item = Array.from(item, binaryOptions); + result[key] = item; + } + return result; + } + function binaryPayload(value) { + if (typeof value === "string") return goString(value); + if (responseByteArrays.has(value)) return responseByteArrays.get(value); + if (value instanceof ArrayBuffer) return new Uint8Array(value); + if (value instanceof Uint8ClampedArray) return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); + return value; + } + function blockTransform(operation, args) { + if (args.length < 2) return {success: false, error: "data and options are required"}; + return host.blockTransform(operation, binaryPayload(args[0]), binaryOptions(args[1])); + } + if (host.filesEnabled) { + globalThis.ffmpeg = { + getInfo(path) { + if (!arguments.length) return {success: false, error: "file path is required"}; + return host.mediaInfo(goString(path)); + }, + convert(input, output, options) { + if (arguments.length < 2) return {success: false, error: "input and output paths are required"}; + return JSON.parse(host.mediaConvert(goString(input), goString(output), binaryOptions(options))); + } + }; + if (host.rawFfmpegStub) { + globalThis.ffmpeg.execute = function() { + return {success: false, error: "raw FFmpeg execution is disabled; use ffmpeg.convert"}; + }; + } + function fileCall(method, args) { + const paired = ["copy", "move", "transformPatternedBlocks"].includes(method); + const write = method === "write" || method === "writeBytes"; + const required = method === "transformPatternedBlocks" ? 3 : paired || write ? 2 : 1; + if (args.length < required) { + if (method === "exists") return false; + return {success: false, error: method === "transformPatternedBlocks" + ? "input path, output path, and options are required" + : paired ? "source and destination paths are required" + : write ? "path and data are required" : "path is required"}; + } + const value = paired || method === "write" ? goString(args[1]) + : method === "writeBytes" ? binaryPayload(args[1]) : undefined; + const options = method === "readBytes" ? binaryOptions(args[1]) + : method === "writeBytes" || method === "transformPatternedBlocks" ? binaryOptions(args[2]) : null; + return host.fileCall(method, goString(args[0]), value, options, args[3]); + } + globalThis.file = {}; + globalThis.file.download = function(url, path, options) { + if (arguments.length < 2) return {success: false, error: "URL and output path are required"}; + if (!host.networkEnabled) return {success: false, error: "network access unavailable"}; + const opts = binaryOptions(options); + const headers = Object.create(null); + if (opts && isMap(opts.headers)) { + for (const key of Object.keys(opts.headers)) headers[goString(key)] = formatGo(opts.headers[key]); + } + return JSON.parse(host.downloadCall(goString(url), goString(path), opts, JSON.stringify(headers))); + }; + globalThis.file.downloadSegments = function(segments, path, options) { + const failure = (error_type, error) => ({success: false, error, error_type, attempts: 0}); + if (arguments.length < 2) return failure("invalid_request", "segments and output path are required"); + if (!Array.isArray(segments) || !segments.length) return failure("invalid_request", "segments must be a non-empty array"); + const opts = binaryOptions(options); + const common = Object.create(null); + function headersInto(target, source) { + if (source && typeof source === "object" && !Array.isArray(source) + && !ArrayBuffer.isView(source) && !(source instanceof ArrayBuffer) + && !(source instanceof Date) && !(source instanceof Map) && !(source instanceof Set)) { + for (const key of Object.keys(source)) target[goString(key)] = formatGo(source[key]); + } + } + if (host.networkEnabled && opts) headersInto(common, opts.headers); + const specs = []; + for (let index = 0; index < segments.length; index++) { + const item = segments[index]; + const headers = Object.assign(Object.create(null), common); + let url; + if (typeof item === "string") url = item; + else if (item && typeof item === "object" && !Array.isArray(item) + && !ArrayBuffer.isView(item) && !(item instanceof ArrayBuffer) + && !(item instanceof Date) && !(item instanceof Map) && !(item instanceof Set)) { + url = typeof item.url === "string" ? item.url : ""; + if (host.networkEnabled) headersInto(headers, item.headers); + } else return failure("invalid_request", "segment " + index + " must be a URL string or object"); + specs.push({url: goString(url), headers}); + } + return JSON.parse(host.downloadSegmentsCall(JSON.stringify(specs), goString(path), opts)); + }; + for (const method of ["exists", "delete", "read", "readBytes", "write", "writeBytes", "copy", "move", "getSize", "transformPatternedBlocks"]) { + globalThis.file[method] = function() { return fileCall(method, arguments); }; + } + } + const logTypedName = Object.getOwnPropertyDescriptor(Object.getPrototypeOf(Uint8Array.prototype), Symbol.toStringTag).get; + const logArrayBufferLength = Object.getOwnPropertyDescriptor(ArrayBuffer.prototype, "byteLength").get; + const logDateValue = Date.prototype.getTime; + const logMapSize = Object.getOwnPropertyDescriptor(Map.prototype, "size").get; + const logSetSize = Object.getOwnPropertyDescriptor(Set.prototype, "size").get; + const logBoxValues = [Number.prototype.valueOf, Boolean.prototype.valueOf, BigInt.prototype.valueOf, Symbol.prototype.valueOf]; + const logApply = Reflect.apply; + const logSlice = String.prototype.slice; + function logValue(value) { + if (value === null || value === undefined) return "<value>"; + if (byteArrays.has(value)) return "<[]uint8>"; + const opaque = host.logOpaqueType(value); + if (opaque) return opaque; + const kind = typeof value; + if (kind === "bigint") return "<*big.Int>"; + if (kind === "function") return "<func(goja.FunctionCall) goja.Value>"; + if (kind !== "object") return goString(value); + if (Array.isArray(value)) return "<[]interface {}>"; + const typedName = logApply(logTypedName, value, []); + if (typedName) { + const types = {Uint8Array:"uint8", Uint8ClampedArray:"uint8", Int8Array:"int8", Uint16Array:"uint16", Int16Array:"int16", Uint32Array:"uint32", Int32Array:"int32", Float32Array:"float32", Float64Array:"float64", BigInt64Array:"int64", BigUint64Array:"uint64"}; + return "<[]" + types[typedName] + ">"; + } + for (const method of logBoxValues) { + let primitive; + try { primitive = logApply(method, value, []); } catch (_) { continue; } + return typeof primitive === "bigint" ? "<*big.Int>" : goString(value); + } + for (const [method, name] of [[logArrayBufferLength,"goja.ArrayBuffer"], [logDateValue,"time.Time"], [logMapSize,"[][2]interface {}"], [logSetSize,"[]interface {}"]]) { + try { logApply(method, value, []); return "<" + name + ">"; } catch (_) {} + } + return "<map[string]interface {}>"; + } + globalThis.log = {}; + function logArguments(args) { + const values = []; + for (let index = 0; index < Math.min(args.length, 8); index++) { + // Enough UTF-16 units for Go's byte cut, including overflow detection. + values.push(logApply(logSlice, logValue(args[index]), [0, 513])); + } + return values; + } + for (const level of ["debug", "info", "warn", "error"]) { + globalThis.log[level] = function() { + host.extensionLog(level.toUpperCase(), logArguments(arguments), arguments.length); + }; + } + if (host.managedConsole) globalThis.console = {log() { host.extensionLog("", logArguments(arguments), arguments.length); }}; + globalThis.matching = { + compareStrings(first, second) { + return arguments.length < 2 ? 0 : host.compareStrings(goString(first), goString(second)); + }, + compareDuration(first, second, tolerance) { + return arguments.length < 2 ? false : host.compareDuration(+first, +second, tolerance === undefined ? 3000 : +tolerance); + }, + normalizeString(value) { return arguments.length ? host.normalizeMatching(goString(value)) : ""; } + }; + if (host.legacyBackend) { + const mapEntries = Map.prototype.entries; + const setValues = Set.prototype.values; + const mapSize = Object.getOwnPropertyDescriptor(Map.prototype, "size").get; + const setSize = Object.getOwnPropertyDescriptor(Set.prototype, "size").get; + function filenameMetadata(value) { + const seen = new WeakSet(); + function visit(value, depth) { + if (typeof value === "string" || typeof value === "number") return value; + if (value === null || typeof value !== "object" || host.logOpaqueType(value)) return null; + try { return stringApply(boxedNumberValue, value, []); } catch (_) {} + try { stringApply(boxedBooleanValue, value, []); return null; } catch (_) {} + if (value instanceof Date || value instanceof ArrayBuffer + || (ArrayBuffer.isView(value) && !(value instanceof DataView))) return null; + if (seen.has(value)) return null; + if (depth >= 128) throw new TypeError("deeply nested filename metadata"); + seen.add(value); + if (value instanceof Map || value instanceof Set) { + const map = value instanceof Map; + const count = stringApply(map ? mapSize : setSize, value, []); + const iterator = stringApply(map ? mapEntries : setValues, value, []); + for (let index = 0; index < count; index++) { + const entry = iterator.next(); + if (entry.done) break; + if (map) visit(entry.value[0], depth + 1); + visit(map ? entry.value[1] : entry.value, depth + 1); + } + return null; + } + if (Array.isArray(value)) { + const length = value.length; + for (let index = 0; index < length; index++) visit(value[index], depth + 1); + return null; + } + let stringObject; + try { stringObject = stringApply(boxedStringValue, value, []); } catch (_) {} + const result = depth === 0 ? Object.create(null) : null; + for (const key of Object.keys(value)) { + if (stringObject !== undefined && /^(0|[1-9][0-9]*)$/.test(key) && Number(key) < stringObject.length) continue; + // Goja exports ignored values too: preserve getter order, + // errors and shared/cyclic graph visitation without JSON. + const item = visit(value[key], depth + 1); + if (result === null) continue; + if (typeof item === "string") result[key.toWellFormed()] = item.toWellFormed(); + else if (typeof item === "number") { + result[key.toWellFormed()] = Number.isFinite(item) ? item : Number.isNaN(item) ? 0 : item < 0 ? -1e300 : 1e300; + } + } + return result; + } + const result = visit(value, 0); + return isMap(result) ? result : null; + } + globalThis.gobackend = { + sanitizeFilename(value) { return arguments.length ? host.legacySanitize(goString(value)) : ""; }, + buildFilename(template, metadata) { + if (arguments.length < 2) return ""; + const pattern = goString(template); + const fields = filenameMetadata(metadata); + return fields === null ? "" : host.legacyFilename(pattern, serialize(fields)); + }, + getLocalTime() { return JSON.parse(host.legacyLocalTime()); }, + getAudioQuality(path) { + return arguments.length ? JSON.parse(host.legacyQuality(goString(path))) : {error: "file path is required"}; + }, + getLyricsLRC(spotifyID, track, artist, path, duration) { + if (arguments.length < 3) return {error: "spotifyID, trackName, and artistName are required"}; + const id = goString(spotifyID), title = goString(track), name = goString(artist); + const file = path == null ? "" : goString(path); + const value = duration == null ? 0 : goNumberPrimitive(duration); + const milliseconds = typeof value === "string" ? 0 : +value; + const text = typeof value === "string" + ? (typeof duration === "string" ? value : value.trim()).toWellFormed() : null; + return JSON.parse(host.legacyLyrics(id, title, name, file, milliseconds, + text, typeof duration !== "string")); + }, + checkISRCExists(directory, isrc) { + return arguments.length < 2 ? {error: "outputDir and isrc are required"} + : JSON.parse(host.legacyIsrc(false, goString(directory), goString(isrc), "")); + }, + addToISRCIndex(directory, isrc, path) { + return arguments.length < 3 ? {error: "outputDir, isrc, and filePath are required"} + : JSON.parse(host.legacyIsrc(true, goString(directory), goString(isrc), goString(path))); + } + }; + } + globalThis.utils = { + isDownloadCancelled() { return host.downloadCancelled(); }, + isRequestCancelled() { return host.requestCancelled(); }, + randomUserAgent() { return host.randomUserAgent(); }, + appVersion() { return host.appVersion(); }, + appUserAgent() { return host.appUserAgent(); }, + setDownloadStatus(status) { + if (arguments.length && host.downloadItemActive()) host.downloadStatus(goString(status)); + }, + getResolutionRemainingMs() { return host.resolutionRemaining(); }, + encrypt(data, key) { + return arguments.length < 2 ? {success: false, error: "plaintext and key are required"} + : host.cryptoText(false, goString(data), goString(key)); + }, + decrypt(data, key) { + return arguments.length < 2 ? {success: false, error: "ciphertext and key are required"} + : host.cryptoText(true, goString(data), goString(key)); + }, + generateKey(length) { + length = length === undefined ? 32 : Number(length); + if (!Number.isInteger(length) || length < 1 || length > 4096) + return {success: false, error: "key length must be an integer between 1 and 4096 bytes"}; + return host.generateKey(length); + }, + encryptBlockCipher() { return blockTransform("encrypt", arguments); }, + decryptBlockCipher() { return blockTransform("decrypt", arguments); }, + decryptCTRSegments() { return blockTransform("segments", arguments); }, + base64Encode(value) { return arguments.length ? host.base64Encode(goString(value)) : ""; }, + base64Decode(value) { return arguments.length ? host.base64Decode(goString(value), false) : ""; }, + md5(value) { return arguments.length ? host.md5(goString(value)) : ""; }, + sha256(value) { return arguments.length ? host.sha256(goString(value)) : ""; }, + hmacSHA256(message, key) { + return arguments.length < 2 ? "" : host.hmacSHA256(goString(message), goString(key)); + }, + hmacSHA256Base64(message, key) { + return arguments.length < 2 ? "" : host.hmacSHA256Base64(goString(message), goString(key)); + }, + hmacSHA1(key, message) { + function bytes(value) { + if (typeof value === "string") return host.encode(goString(value)); + if (Array.isArray(value)) return Array.from(value, item => typeof item === "number" ? item & 255 : 0); + return null; + } + key = bytes(key); + message = bytes(message); + return key === null || message === null ? emptyBytes() : host.hmacSHA1(key, message); + }, + parseJSON(value) { + if (!arguments.length) return undefined; + try { + // encoding/json rejects overflowing numbers and replaces lone surrogates. + return JSON.parse(serialize(JSON.parse(String(value)))); + } catch (_) { return undefined; } + }, + stringifyJSON(value) { + if (!arguments.length) return ""; + try { return serialize(value); } catch (_) { return ""; } + }, + sleep(milliseconds) { + if (typeof milliseconds !== "number" || !(milliseconds > 0)) return true; + return host.sleep(Math.min(Math.trunc(milliseconds), 300000)); + } + }; + globalThis.btoa = function (value) { + return arguments.length ? host.base64Encode(goString(value)) : ""; + }; + globalThis.atob = function (value) { + return arguments.length ? host.base64Decode(goString(value), true) : ""; + }; + globalThis.TextEncoder = function () { + this.encoding = "utf-8"; + this.encode = function (value) { + return arguments.length ? host.encode(goString(value)) : emptyBytes(); + }; + this.encodeInto = function (value, destination) { + if (arguments.length < 2) return { read: 0, written: 0 }; + const length = host.encode(goString(value)).length; + // Preserve the existing Go host contract, including no destination copy. + return { read: length, written: length }; + }; + }; + globalThis.TextDecoder = function (encoding) { + this.encoding = encoding === undefined ? "utf-8" : goString(encoding); + this.fatal = false; + this.ignoreBOM = false; + this.decode = function (value) { + if (typeof value === "string") return goString(value); + if (!Array.isArray(value) && !(value instanceof Uint8Array)) return ""; + return host.decode(Array.from(value, item => typeof item === "number" ? item & 255 : 0)); + }; + }; + return { + providerHelpers: {goString, isMap, exportValue, serialize}, + registered() { return registered; }, + getExtension() { return globalThis.extension; }, + invokeManaged(action, method, args) { + const extension = globalThis.extension; + let callback; + if (extension !== undefined && (action ? extension : true) && typeof extension[method] === "function") { + callback = () => extension[method].apply(extension, args); + } else if (action && method === "completeGrant" && typeof session !== "undefined" && session && typeof session.completeGrant === "function") { + callback = () => session.completeGrant(); + } + if (!callback) return action ? {success:false, error:"Action function not found: " + method} : {success:true, message:"no " + method + " function"}; + try { + const result = callback(); + if (!action) return {success:true}; + if (result && typeof result.then === "function") return {success:true, pending:true, message:"Action started"}; + if (result !== null && result !== undefined && typeof result === "object" && !Array.isArray(result)) { + const output = {success:true}; + for (const key in result) output[key] = result[key]; + return output; + } + return {success:true, result}; + } catch (error) { return {success:false, error:error.toString()}; } + }, + serialize + }; +}) diff --git a/rust_backend/crates/extensions/src/provider.js b/rust_backend/crates/extensions/src/provider.js new file mode 100644 index 00000000..d2f58dcd --- /dev/null +++ b/rust_backend/crates/extensions/src/provider.js @@ -0,0 +1,352 @@ +(function (host, helpers) { + "use strict"; + const object = Object; + const keys = Object.keys; + const isArray = Array.isArray; + const apply = Reflect.apply; + const stringify = JSON.stringify; + const isView = ArrayBuffer.isView; + const dateType = Date; + const bufferType = ArrayBuffer; + const mapType = Map; + const setType = Set; + const mapEach = Map.prototype.forEach; + const setEach = Set.prototype.forEach; + const stringValue = String.prototype.valueOf; + const numberValue = Number.prototype.valueOf; + const booleanValue = Boolean.prototype.valueOf; + const symbolValue = Symbol.prototype.valueOf; + const {goString, exportValue, serialize} = helpers; + const empty = value => value === undefined || value === null; + const trim = value => host.providerTrim(value); + // Export happens before Go's type assertion. Traverse rejected objects too: + // their getters can change later fields. Shared/cyclic objects are visited + // once per Export, and unsupported JSON values are harmless if discarded. + function exported(value, seen) { + if (typeof value === "symbol") return goString(value); + if (empty(value) || typeof value !== "object" || host.logOpaqueType(value) !== "") return value; + let primitive; + try { primitive = apply(numberValue, value, []); } catch (_) {} + if (typeof primitive === "number") return primitive; + try { primitive = apply(booleanValue, value, []); } catch (_) {} + if (typeof primitive === "boolean") return primitive; + try { primitive = apply(symbolValue, value, []); } catch (_) {} + if (typeof primitive === "symbol") return goString(primitive); + if (value instanceof dateType || value instanceof bufferType || isView(value)) return value; + if (seen === undefined) seen = new mapType(); + if (seen.has(value)) return seen.get(value); + if (isArray(value) || value instanceof setType) { + const result = []; + seen.set(value, result); + if (isArray(value)) { + const length = value.length; + for (let index = 0; index < length; index++) result.push(exported(value[index], seen)); + } else apply(setEach, value, [item => result.push(exported(item, seen))]); + return result; + } + if (value instanceof mapType) { + seen.set(value, value); + apply(mapEach, value, [(item, key) => { exported(key, seen); exported(item, seen); }]); + return value; + } + let stringObject; + try { stringObject = apply(stringValue, value, []); } catch (_) {} + const result = object.create(null); + seen.set(value, result); + for (const key of keys(value)) { + if (stringObject !== undefined && /^(0|[1-9][0-9]*)$/.test(key) && Number(key) < stringObject.length) continue; + result[key] = exported(value[key], seen); + } + return result; + } + function first(value, names) { + for (const name of names) { + const field = value[name]; + if (!empty(field)) return field; + } + } + function text(value, names) { + for (const name of names) { + const field = exported(value[name]); + if (typeof field === "string") return field.toWellFormed(); + } + return ""; + } + function int(value, names, wide = false) { + const field = first(value, names); + return empty(field) ? 0n : BigInt(host.providerInteger(+field, wide)); + } + function map(value, names) { + const field = exported(first(value, names)); + if (empty(field) || !helpers.isMap(field) || host.logOpaqueType(field) !== "") return null; + const result = exportValue(field, []); + return result !== null && typeof result === "object" && keys(result).length ? result : null; + } + function stringMap(value, names) { + const field = first(value, names); + if (empty(field)) return null; + const result = object.create(null); + for (const name of keys(object(field))) { + const child = field[name]; + if (!empty(child)) result[name.toWellFormed()] = goString(child); + } + return keys(result).length ? result : null; + } + function strings(value, names) { + const field = exported(first(value, names)); + if (!isArray(field) || host.logOpaqueType(field) !== "") return []; + const result = []; + for (const item of field) { + const string = typeof item === "string" ? item.toWellFormed() : ""; + if (trim(string)) result.push(trim(string)); + } + return result; + } + function fields(value, schema, required) { + value = object(value); + const result = object.create(null); + for (const [name, kind, ...aliases] of schema) { + const names = [name, ...aliases]; + let field; + switch (kind) { + case "s": field = text(value, names); break; + case "i": field = int(value, names); break; + case "l": field = int(value, names, true); break; + case "b": field = Boolean(first(value, names)); break; + case "m": field = map(value, names); break; + case "M": field = stringMap(value, names); break; + case "a": field = strings(value, names); break; + default: throw Error("unknown provider field kind"); + } + if (required.includes(name) || (field !== "" && field !== 0n && field !== false && field !== null && (!isArray(field) || field.length))) result[name] = field; + } + return result; + } + function array(value, parse) { + if (empty(value)) return []; + const lengthValue = object(value).length; + if (empty(lengthValue)) throw Error("value is not an array"); + const length = Number(BigInt(host.providerInteger(+lengthValue, true))); + if (length <= 0) return []; + const count = Number(BigInt(host.providerInteger(length, false))); + if (count < 0) throw Error("array length exceeds native integer range"); + const result = []; + for (let index = 0; index < count; index++) { + const item = value[index]; + if (!empty(item)) result.push(parse(item)); + } + return result; + } + const trackSchema = [ + ["id","s"],["name","s"],["artists","s"],["album_name","s","albumName"], + ["album_artist","s","albumArtist"],["album_id","s","albumId"],["album_url","s","albumUrl"], + ["artist_id","s","artistId"],["artist_url","s","artistUrl"], + ["external_urls","s","externalUrls","external_url","externalUrl","url"], + ["duration_ms","i","durationMs"],["cover_url","s","coverUrl"],["preview_url","s","previewUrl"], + ["images","s"],["release_date","s","releaseDate"],["track_number","i","trackNumber"], + ["total_tracks","i","totalTracks"],["disc_number","i","discNumber"],["total_discs","i","totalDiscs"], + ["isrc","s"],["provider_id","s","providerId"],["item_type","s","itemType"], + ["album_type","s","albumType"],["explicit","b","is_explicit","isExplicit"],["upc","s","barcode"], + ["tidal_id","s","tidalId"],["qobuz_id","s","qobuzId"],["deezer_id","s","deezerId"], + ["spotify_id","s","spotifyId"],["external_links","M","externalLinks"],["label","s"], + ["copyright","s"],["genre","s"],["composer","s"],["comment","s","comments"], + ["audio_quality","s","audioQuality"],["audio_modes","s","audioModes"] + ]; + const track = value => fields(value, trackSchema, ["id","name","artists","album_name","duration_ms","provider_id"]); + function album(value) { + const tracks = array(first(object(value), ["tracks"]), track); + const result = fields(value, [ + ["id","s"],["name","s"],["artists","s"],["artist_id","s","artistId"], + ["cover_url","s","coverUrl","images"],["header_image","s","headerImage"], + ["header_video","s","headerVideo"],["release_date","s","releaseDate"], + ["total_tracks","i","totalTracks"],["album_type","s","albumType"], + ["audio_traits","a","audioTraits"],["provider_id","s","providerId"] + ], ["id","name","artists","total_tracks","provider_id"]); + result.tracks = tracks; + if (!trim(result.artists)) { + const explicit = tracks.find(item => trim(item.album_artist || "")); + if (explicit) result.artists = trim(explicit.album_artist); + else { + const counts = new Map(); + for (const item of tracks) { + const artist = trim(item.artists); + if (artist) counts.set(artist, (counts.get(artist) || 0) + 1); + } + let count = 0; + result.artists = ""; + for (const [artist, votes] of counts) if (votes > count) { result.artists = artist; count = votes; } + } + } + if (!trim(result.release_date || "")) { + const source = tracks.find(item => trim(item.release_date || "")); + if (source) result.release_date = trim(source.release_date); + else delete result.release_date; + } + if (!result.audio_traits) { + const traits = host.providerAudioTraits(tracks.map(item => item.audio_quality || ""), tracks.map(item => item.audio_modes || "")); + if (traits.length) result.audio_traits = traits; + } + return result; + } + function artist(value) { + const albums = array(first(object(value), ["albums"]), album); + const releases = array(first(object(value), ["releases"]), album); + const tracks = array(first(object(value), ["top_tracks","topTracks","tracks"]), track); + const result = fields(value, [["id","s"],["name","s"],["image_url","s","imageUrl"], + ["header_image","s","headerImage"],["header_video","s","headerVideo"], + ["listeners","i"],["provider_id","s","providerId"]], ["id","name","provider_id"]); + if (albums.length) result.albums = albums; + if (releases.length) result.releases = releases; + if (tracks.length) result.top_tracks = tracks; + return result; + } + function decryption(value) { + if (empty(value)) return null; + const result = fields(value, [["strategy","s"],["key","s"],["iv","s"], + ["input_format","s","inputFormat"],["output_extension","s","outputExtension"],["options","m"]], []); + return keys(result).length ? result : null; + } + function download(value) { + const result = fields(value, [ + ["success","b"],["file_path","s","filePath","path"],["already_exists","b","alreadyExists"], + ["bit_depth","i","bitDepth"],["sample_rate","i","sampleRate"],["audio_codec","s","audioCodec","codec"], + ["duration_ms","i","durationMs"],["error_message","s","errorMessage","error"],["error_type","s","errorType"], + ["retry_after_seconds","i","retryAfterSeconds"],["title","s"],["artist","s"],["album","s"], + ["album_artist","s","albumArtist"],["track_number","i","trackNumber"],["disc_number","i","discNumber"], + ["total_tracks","i","totalTracks"],["total_discs","i","totalDiscs"],["release_date","s","releaseDate"], + ["cover_url","s","coverUrl"],["isrc","s"],["genre","s"],["label","s"],["copyright","s"], + ["composer","s"],["comment","s","comments"],["explicit","b","is_explicit","isExplicit"], + ["album_type","s","albumType"],["upc","s","barcode"],["lyrics_lrc","s","lyricsLrc"], + ["decryption_key","s","decryptionKey"] + ], ["success"]); + const info = decryption(first(object(value), ["decryption"])); + if (info) result.decryption = info; + object.assign(result, fields(value, [["actual_extension","s","actualExtension"], + ["output_extension","s","outputExtension"],["actual_container","s","actualContainer","container"], + ["requires_container_conversion","b","requiresContainerConversion"]], [])); + return result; + } + function url(value) { + const result = fields(value, [["type","s"],["id","s"],["name","s"],["cover_url","s","coverUrl"], + ["header_image","s","headerImage"],["header_video","s","headerVideo"]], ["type"]); + value = object(value); + for (const [name, parse] of [["track",track],["tracks",v => array(v,track)],["album",album],["artist",artist]]) { + const field = first(value, [name]); + if (!empty(field)) { + const parsed = parse(field); + if (!isArray(parsed) || parsed.length) result[name] = parsed; + } + } + return result; + } + function lyrics(value) { + const lines = array(first(object(value), ["lines"]), line => fields(line, [ + ["startTimeMs","l","start_time_ms"],["words","s"],["endTimeMs","l","end_time_ms"] + ], ["startTimeMs","words","endTimeMs"])); + const result = fields(value, [["syncType","s","sync_type"],["instrumental","b"], + ["plainLyrics","s","plain_lyrics"],["provider","s"]], ["syncType","instrumental","plainLyrics","provider"]); + result.lines = lines; + return result; + } + function search(value) { + const raw = first(object(value), ["tracks"]); + const tracks = array(empty(raw) ? value : raw, track); + const total = empty(raw) ? BigInt(tracks.length) : int(object(value), ["total"]); + return {tracks, total: total || BigInt(tracks.length)}; + } + function encode(value) { + // BigInts here are native integer fields created by this parser. Opaque + // extension maps pass through the existing Go export conversion first. + if (typeof value === "bigint") return String(value); + if (isArray(value)) return "[" + value.map(encode).join(",") + "]"; + if (value !== null && typeof value === "object") return "{" + keys(value).map(key => host.quoteJSON(key) + ":" + encode(value[key])).join(",") + "}"; + return serialize(value); + } + const parsers = { + getTrack: track, enrichTrack: track, getAlbum: album, getPlaylist: album, getArtist: artist, + searchTracks: search, customSearch: value => array(value, track), handleUrl: url, + checkAvailability: value => fields(value, [["available","b"],["reason","s"],["track_id","s","trackId"], + ["skip_fallback","b","skipFallback"],["prepared_context","m","preparedContext"]], ["available"]), + download, fetchLyrics: lyrics, + postProcessV2: value => fields(value, [["success","b"],["new_file_path","s","newFilePath"], + ["new_file_uri","s","newFileUri"],["error","s"],["bit_depth","i","bitDepth"],["sample_rate","i","sampleRate"]], ["success"]) + }; + function invoke(method, args) { + const extension = globalThis.extension; + if (empty(extension)) return null; + let name = method; + if (method === "getPlaylist" && typeof extension.getPlaylist !== "function") name = "getAlbum"; + if (method === "postProcessV2" && typeof extension.postProcessV2 !== "function") { + name = "postProcess"; + args = [args[0].path || "", args[1], args[2]]; + } + const callback = extension[name]; + return typeof callback === "function" ? apply(callback, extension, args) : null; + } + function invokeVerified(method, args) { + if (method === "getHomeFeed") { + const extension = globalThis.extension; + if (!empty(extension) && typeof extension[method] === "function") { + return apply(extension[method], extension, args); + } + const callback = globalThis[method]; + return typeof callback === "function" ? apply(callback, globalThis, args) : null; + } + try { return invoke(method, args); } + catch (error) { + const id = method === "enrichTrack" ? null : host.providerPendingVerification(); + if (!empty(id) && !empty(error)) { + let message; + try { + const value = typeof error === "object" || typeof error === "function" ? error.message : error; + if (!empty(value)) message = trim(goString(value)); + } catch (_) {} + if (message === "VERIFY_REQUIRED") { + throw "verification_required: extension '" + id + "' needs signed-session verification: " + goString(error); + } + } + throw error; + } + } + return { + invoke: invokeVerified, + invokeDownload(method, args) { + const extension = globalThis.extension; + if (empty(extension)) return null; + const callback = extension[method]; + if (typeof callback !== "function") return null; + const progress = function (value) { + if (!arguments.length) return; + const percent = BigInt(host.providerInteger(+value, false)); + host.providerProgress(Number(percent < 0n ? 0n : percent > 100n ? 100n : percent)); + }; + return apply(callback, extension, [args[0], args[1], args[2], progress, args[3]]); + }, + invokePostProcess(method, args) { + const result = {}; + try { result.value = invokeVerified(method, args); } + catch (error) { result.failure = goString(error); } + return result; + }, + parsePostProcess(method, result) { + let value; + try { + value = result.failure !== undefined ? {success:false,error:result.failure} + : empty(result.value) ? {success:false,error:"postProcess returned null"} + : parsers.postProcessV2(result.value); + } catch (error) { value = {success:false,error:error.message || goString(error)}; } + return "{\"value\":" + encode(value) + "}"; + }, + parse(method, value) { + if (empty(value)) return "{\"value\":null}"; + if (method === "getHomeFeed") { + // The Go export marshals goja.Value directly: JSON.stringify, + // including toJSON/undefined semantics, without awaiting Promises. + try { return "{\"value\":" + (stringify(value) || "null") + "}"; } + catch (error) { return serialize({parseError: error.message || goString(error)}); } + } + try { return "{\"value\":" + encode(parsers[method](value)) + "}"; } + catch (error) { return serialize({parseError: error.message || goString(error)}); } + } + }; +}) diff --git a/rust_backend/crates/extensions/src/provider.rs b/rust_backend/crates/extensions/src/provider.rs new file mode 100644 index 00000000..a23ccae4 --- /dev/null +++ b/rust_backend/crates/extensions/src/provider.rs @@ -0,0 +1,79 @@ +//! Native numeric and album-quality rules used while reading provider objects. + +pub(crate) fn integer(value: f64, wide: bool) -> String { + let integer = value as i64; + if wide { + integer.to_string() + } else { + (integer as isize).to_string() + } +} + +pub(crate) fn audio_traits(qualities: Vec<String>, modes: Vec<String>) -> Vec<String> { + let mut atmos = false; + let mut hires = false; + let mut lossless = false; + for (quality, mode) in qualities.iter().zip(&modes) { + // Go uses one-rune uppercase mappings, so expanding ß to SS must not + // accidentally make an unrecognized label match LOSSLESS. + let upper = |value: &str| { + value + .chars() + .map(|character| { + let mut mapped = character.to_uppercase(); + let first = mapped.next().unwrap(); + if mapped.next().is_none() { + first + } else { + character + } + }) + .collect::<String>() + }; + let quality = upper(quality); + atmos |= upper(mode).contains("ATMOS") || quality.contains("ATMOS"); + hires |= ["HI_RES", "HIRES", "MASTER", "MQA"] + .iter() + .any(|part| quality.contains(part)); + lossless |= quality.contains("LOSSLESS") || quality.contains("FLAC"); + let lower = spotiflac_core::matching::lowercase(&quality); + let numeric_prefix = |suffix: &str, decimal: bool| { + lower + .find(suffix) + .map(|end| { + let mut start = end; + while start > 0 + && (lower.as_bytes()[start - 1].is_ascii_digit() + || (decimal && lower.as_bytes()[start - 1] == b'.')) + { + start -= 1; + } + &lower[start..end] + }) + .unwrap_or("") + }; + let depth = numeric_prefix("bit", false).parse::<isize>().unwrap_or(0); + let rate = numeric_prefix("khz", true) + .parse::<f64>() + .ok() + .filter(|value| value.is_finite()) + .unwrap_or(0.0); + if depth > 0 { + if depth > 16 || rate > 48.0 { + hires = true; + } else { + lossless = true; + } + } + } + let mut traits = Vec::new(); + if atmos { + traits.push("dolby_atmos".into()); + } + if hires { + traits.push("hi_res_lossless".into()); + } else if lossless { + traits.push("lossless".into()); + } + traits +} diff --git a/rust_backend/crates/extensions/src/redact.rs b/rust_backend/crates/extensions/src/redact.rs new file mode 100644 index 00000000..d3117912 --- /dev/null +++ b/rust_backend/crates/extensions/src/redact.rs @@ -0,0 +1,109 @@ +use regex::Regex; +use std::borrow::Cow; +use std::sync::LazyLock; + +static PATTERNS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| { + // Go's RE2 uses ASCII whitespace and word boundaries. Keep Unicode token + // contents inside the redacted match instead of treating them as delimiters. + vec![ + (r"(?i)(?-u:\b)Authorization(?-u:\b)[\t\n\f\r ]*[:=][\t\n\f\r ]*Bearer[\t\n\f\r ]+[A-Za-z0-9._~+/\-]+=*", "Authorization: Bearer [REDACTED]"), + (r#"(?i)("?(?:access[_\t\n\f\r -]?token|refresh[_\t\n\f\r -]?token|id[_\t\n\f\r -]?token|client[_\t\n\f\r -]?secret|authorization|password|api[_\t\n\f\r -]?key|session[_\t\n\f\r -]?secret|decryption[_\t\n\f\r -]?key|cookie|set-cookie)"?)([\t\n\f\r ]*[:=][\t\n\f\r ]*)("(?:\\.|[^"\\])*"|[^\t\n\f\r ,;}\]]+)"#, "${1}${2}[REDACTED]"), + (r"(?i)([?&](?:access_token|refresh_token|id_token|token|client_secret|api_key|apikey|password|code|grant|sig|signature|x-amz-signature|x-amz-credential|x-amz-security-token|awsaccesskeyid|googleaccessid|policy|key-pair-id)=)[^&\t\n\f\r ]+", "${1}[REDACTED]"), + (r"(?i)(?-u:\b)Bearer[\t\n\f\r ]+[A-Za-z0-9._~+/\-]+=*", "Bearer [REDACTED]"), + (r"(?i)(-decryption_key[\t\n\f\r ]+)[^\t\n\f\r ]+", "${1}[REDACTED]"), + ] + .into_iter() + .map(|(pattern, replacement)| (Regex::new(pattern).expect("static redaction pattern"), replacement)) + .collect() +}); + +fn sanitize_text(mut text: String) -> String { + for (pattern, replacement) in PATTERNS.iter() { + if let Cow::Owned(replaced) = pattern.replace_all(&text, *replacement) { + text = replaced; + } + } + text +} + +pub(crate) fn preview(bytes: &[u8], limit: usize) -> String { + let mut text = sanitize_text(crate::host::decode_go_utf8(bytes)); + if text.len() > limit { + text = crate::host::decode_go_utf8(&text.as_bytes()[..limit]) + "...[truncated]"; + } + text +} + +/// Log arguments can end mid-codepoint after Go's 512-byte truncation. Preserve +/// their raw bytes while matching as Go's regexp UTF-8 decoder would; converting +/// them to replacement characters first would change the later 4000-byte cut. +pub(crate) fn sanitize_bytes(bytes: Vec<u8>) -> Vec<u8> { + let mut bytes = match String::from_utf8(bytes) { + Ok(text) => return sanitize_text(text).into_bytes(), + Err(error) => error.into_bytes(), + }; + for (pattern, replacement) in PATTERNS.iter() { + let (text, invalid) = decoded_positions(&bytes); + let position = |index: usize| index - 2 * invalid.partition_point(|start| *start < index); + let mut output = Vec::with_capacity(bytes.len()); + let mut copied = 0; + for captures in pattern.captures_iter(&text) { + let found = captures.get(0).expect("redaction match"); + let start = position(found.start()); + let end = position(found.end()); + output.extend_from_slice(&bytes[copied..start]); + let mut expanded = String::new(); + // Retained captures are only key names/separators, never secret + // values; they cannot contain an invalid UTF-8 byte. + captures.expand(replacement, &mut expanded); + output.extend_from_slice(expanded.as_bytes()); + copied = end; + } + output.extend_from_slice(&bytes[copied..]); + bytes = output; + } + bytes +} + +fn decoded_positions(mut bytes: &[u8]) -> (String, Vec<usize>) { + let mut text = String::with_capacity(bytes.len()); + let mut invalid = Vec::new(); + while !bytes.is_empty() { + match std::str::from_utf8(bytes) { + Ok(valid) => { + text.push_str(valid); + break; + } + Err(error) => { + let prefix = error.valid_up_to(); + text.push_str(std::str::from_utf8(&bytes[..prefix]).expect("valid UTF-8 prefix")); + invalid.push(text.len()); + text.push('\u{fffd}'); + bytes = &bytes[prefix + 1..]; + } + } + } + (text, invalid) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redaction_preserves_unicode_and_invalid_bytes_around_secret_values() { + for prefix in [b"track: ".as_slice(), "曲名: ".as_bytes(), b"\xff\xe3\x81 "] { + let input = [prefix, b"https://example.test/audio?token=fixture-secret&track=7 password=fixture-password;"].concat(); + let expected = [ + prefix, + b"https://example.test/audio?token=[REDACTED]&track=7 password=[REDACTED];", + ] + .concat(); + assert_eq!(sanitize_bytes(input.clone()), expected); + assert_eq!( + preview(&input, usize::MAX), + crate::host::decode_go_utf8(&expected) + ); + } + } +} diff --git a/rust_backend/crates/extensions/src/repository.rs b/rust_backend/crates/extensions/src/repository.rs new file mode 100644 index 00000000..5552cec1 --- /dev/null +++ b/rust_backend/crates/extensions/src/repository.rs @@ -0,0 +1,549 @@ +//! Native-owned registry cache and verified package downloads. + +mod model; +mod package; +mod urls; + +pub use model::{Registry, RepoExtension, normalize_sha256}; +pub use package::{destination_path, write_verified_package}; +pub use urls::{RegistryLocation, registry_location, require_https}; + +use serde::{Deserialize, Serialize}; +use spotiflac_network::{HttpRequest, NetworkService, NetworkSession}; +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +const MAX_REGISTRY_BYTES: usize = 4 << 20; +const MAX_CACHE_BYTES: u64 = 8 << 20; +const CACHE_TTL: Duration = Duration::from_secs(30 * 60); +const CACHE_FILE: &str = "store_cache.json"; +const REGISTRY_TIMEOUT: Duration = Duration::from_secs(30); +const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(300); + +#[derive(Clone, Debug, thiserror::Error)] +#[error("{0}")] +pub struct RepositoryError(pub String); + +fn error(message: impl Into<String>) -> RepositoryError { + RepositoryError(message.into()) +} +fn cause(context: &str, error: impl std::fmt::Display) -> RepositoryError { + RepositoryError(format!("{context}: {error}")) +} +fn now() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 +} + +#[derive(Clone, Default, Deserialize, Serialize)] +#[serde(default)] +struct Cache { + registry_url: String, + #[serde( + serialize_with = "serialize_registry", + deserialize_with = "deserialize_registry" + )] + registry: Arc<Registry>, + cache_time: i64, + etag: String, + last_modified: String, +} + +fn serialize_registry<S>(registry: &Arc<Registry>, serializer: S) -> Result<S::Ok, S::Error> +where + S: serde::Serializer, +{ + registry.as_ref().serialize(serializer) +} + +fn deserialize_registry<'de, D>(deserializer: D) -> Result<Arc<Registry>, D::Error> +where + D: serde::Deserializer<'de>, +{ + Registry::deserialize(deserializer).map(Arc::new) +} + +#[derive(Default)] +struct Flight { + result: Mutex<Option<Result<Arc<Registry>, RepositoryError>>>, + ready: Condvar, +} + +#[derive(Default)] +struct State { + url: String, + generation: u64, + cache: Option<Arc<Cache>>, + flights: BTreeMap<u64, Arc<Flight>>, +} + +pub struct ExtensionRepository { + network: Arc<NetworkService>, + cache_directory: PathBuf, + _directory_lock: File, + state: Mutex<State>, + pending: Mutex<usize>, + idle: Condvar, + closed: AtomicBool, + parent_closed: Option<Arc<AtomicBool>>, +} + +impl ExtensionRepository { + /// The cache directory belongs to this native object, never to JavaScript. + pub fn with_network( + directory: &Path, + network: Arc<NetworkService>, + ) -> Result<Self, RepositoryError> { + if directory.as_os_str().is_empty() { + return Err(error("repository cache directory is not configured")); + } + fs::create_dir_all(directory).map_err(|e| cause("failed to create repository cache", e))?; + let cache_directory = fs::canonicalize(directory) + .map_err(|e| cause("failed to inspect repository cache", e))?; + let lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags( + (rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32, + ) + .open(cache_directory.join(".backend-repository.lock")) + .map_err(|e| cause("failed to lock repository cache", e))?; + if !lock + .metadata() + .map_err(|e| cause("failed to inspect repository lock", e))? + .is_file() + { + return Err(error("repository cache lock must be a regular file")); + } + lock.try_lock() + .map_err(|_| error("repository cache is already managed"))?; + let cache = (|| { + let file = OpenOptions::new() + .read(true) + .custom_flags( + (rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32, + ) + .open(cache_directory.join(CACHE_FILE)) + .ok()?; + let metadata = file.metadata().ok()?; + if !metadata.is_file() || metadata.len() > MAX_CACHE_BYTES { + return None; + } + let mut bytes = Vec::new(); + file.take(MAX_CACHE_BYTES + 1) + .read_to_end(&mut bytes) + .ok()?; + if bytes.len() as u64 > MAX_CACHE_BYTES { + return None; + } + serde_json::from_slice::<Cache>(&bytes).ok().map(Arc::new) + })(); + Ok(Self { + network, + cache_directory, + _directory_lock: lock, + state: Mutex::new(State { + url: cache + .as_ref() + .map_or_else(String::new, |cache| cache.registry_url.clone()), + cache, + ..State::default() + }), + pending: Mutex::new(0), + idle: Condvar::new(), + closed: AtomicBool::new(false), + parent_closed: None, + }) + } + + pub(crate) fn attach_environment(mut self, closed: Arc<AtomicBool>) -> Self { + self.parent_closed = Some(closed); + self + } + + fn enter(&self) -> Result<Operation<'_>, RepositoryError> { + let mut pending = self.pending.lock().expect("repository operations lock"); + self.check()?; + *pending += 1; + Ok(Operation(self)) + } + + fn check(&self) -> Result<(), RepositoryError> { + if self.closed.load(Ordering::Acquire) + || self + .parent_closed + .as_ref() + .is_some_and(|closed| closed.load(Ordering::Acquire)) + { + Err(error("extension repository closed")) + } else { + Ok(()) + } + } + + fn check_generation(&self, generation: u64) -> Result<(), RepositoryError> { + self.check()?; + if self.state.lock().expect("repository state lock").generation != generation { + Err(error("registry changed while operation was running")) + } else { + Ok(()) + } + } + + pub fn registry_url(&self) -> Result<String, RepositoryError> { + self.check()?; + Ok(self + .state + .lock() + .expect("repository state lock") + .url + .clone()) + } + + pub fn set_registry_url(&self, input: &str) -> Result<(), RepositoryError> { + let _operation = self.enter()?; + let url = match registry_location(input)? { + RegistryLocation::Direct(url) => url, + RegistryLocation::GitHub { owner, repository } => { + let api = format!("https://api.github.com/repos/{owner}/{repository}"); + let session = self.network.native_session(Duration::from_secs(10)); + let response = session.request(request(&api, BTreeMap::new()), || { + self.check().map_err(|e| e.to_string()) + }); + self.check()?; + let branch = response + .ok() + .filter(|response| response.status == 200) + .and_then(|response| { + serde_json::from_slice::<serde_json::Value>(&response.body).ok() + }) + .and_then(|value| { + value["default_branch"] + .as_str() + .filter(|value| !value.is_empty()) + .map(str::to_owned) + }) + .unwrap_or_else(|| "main".into()); + format!( + "https://raw.githubusercontent.com/{owner}/{repository}/{branch}/registry.json" + ) + } + }; + require_https(&url, "registry")?; + let mut state = self.state.lock().expect("repository state lock"); + self.check()?; + if state.url != url { + state.url = url; + self.clear_locked(&mut state); + } + Ok(()) + } + + pub fn clear_registry_url(&self) -> Result<(), RepositoryError> { + self.check()?; + let mut state = self.state.lock().expect("repository state lock"); + state.url.clear(); + self.clear_locked(&mut state); + Ok(()) + } + + pub fn clear_cache(&self) -> Result<(), RepositoryError> { + self.check()?; + self.clear_locked(&mut self.state.lock().expect("repository state lock")); + Ok(()) + } + + fn clear_locked(&self, state: &mut State) { + state.generation = state.generation.wrapping_add(1); + state.cache = None; + state.flights.clear(); + let _ = fs::remove_file(self.cache_directory.join(CACHE_FILE)); + } + + fn save_locked(&self, cache: &Cache) { + // Hold the state lock through publication: a URL change/clear cannot + // race an old snapshot back onto disk. A cache write failure is benign. + let _ = (|| -> Result<(), Box<dyn std::error::Error>> { + let bytes = serde_json::to_vec(cache)?; + if bytes.len() as u64 > MAX_CACHE_BYTES { + return Ok(()); + } + let mut file = tempfile::NamedTempFile::new_in(&self.cache_directory)?; + file.write_all(&bytes)?; + file.as_file().sync_all()?; + file.persist(self.cache_directory.join(CACHE_FILE))?; + Ok(()) + })(); + } + + pub fn fetch(&self, force: bool) -> Result<Arc<Registry>, RepositoryError> { + let _operation = self.enter()?; + let (url, generation, cached, flight, leader) = { + let mut state = self.state.lock().expect("repository state lock"); + if state.url.is_empty() { + return Err(error( + "no registry URL configured. Please add a repository URL first", + )); + } + require_https(&state.url, "registry")?; + if !force + && let Some(cache) = &state.cache + && now().saturating_sub(cache.cache_time) < CACHE_TTL.as_secs() as i64 + { + return Ok(Arc::clone(&cache.registry)); + } + let generation = state.generation; + let leader = !state.flights.contains_key(&generation); + let flight = state.flights.entry(generation).or_default().clone(); + ( + state.url.clone(), + generation, + state.cache.clone(), + flight, + leader, + ) + }; + if !leader { + let mut result = flight.result.lock().expect("repository refresh lock"); + loop { + self.check_generation(generation)?; + if let Some(result) = &*result { + return result.clone(); + } + result = flight + .ready + .wait_timeout(result, Duration::from_millis(10)) + .expect("repository refresh wait") + .0; + } + } + let result = self.refresh(&url, generation, cached); + *flight.result.lock().expect("repository refresh lock") = Some(result.clone()); + flight.ready.notify_all(); + self.state + .lock() + .expect("repository state lock") + .flights + .remove(&generation); + result + } + + fn refresh( + &self, + url: &str, + generation: u64, + cached: Option<Arc<Cache>>, + ) -> Result<Arc<Registry>, RepositoryError> { + let check = || self.check_generation(generation).map_err(|e| e.to_string()); + let session = self.network.native_session(REGISTRY_TIMEOUT); + let mut headers = BTreeMap::new(); + if let Some(cache) = &cached { + if !cache.etag.is_empty() { + headers.insert("If-None-Match".into(), cache.etag.clone()); + } + if !cache.last_modified.is_empty() { + headers.insert("If-Modified-Since".into(), cache.last_modified.clone()); + } + } + let response = read_registry(&session, request(url, headers), &check); + self.check_generation(generation)?; + let (status, headers, body) = match response { + Ok(response) => response, + Err(error) if error.0.starts_with("failed to read registry:") => return Err(error), + Err(error) => { + return cached.map(|cache| Arc::clone(&cache.registry)).ok_or(error); + } + }; + if status != 200 && !(status == 304 && cached.is_some()) { + return cached + .map(|cache| Arc::clone(&cache.registry)) + .ok_or_else(|| error(format!("registry returned HTTP {status}"))); + } + let cache = if status == 304 { + let mut cache = (*cached.expect("304 cache")).clone(); + cache.cache_time = now(); + cache + } else { + let registry = match Registry::parse(&body) { + Ok(registry) => registry, + Err(error) => { + return cached.map(|cache| Arc::clone(&cache.registry)).ok_or(error); + } + }; + let header = |name| { + headers + .get(name) + .and_then(|values| values.first()) + .map_or("", |value| value.trim()) + .to_owned() + }; + Cache { + registry_url: url.into(), + registry: Arc::new(registry), + cache_time: now(), + etag: header("Etag"), + last_modified: header("Last-Modified"), + } + }; + let registry = Arc::clone(&cache.registry); + let mut state = self.state.lock().expect("repository state lock"); + self.check()?; + if state.generation != generation { + return Err(error("registry changed while operation was running")); + } + self.save_locked(&cache); + state.cache = Some(Arc::new(cache)); + Ok(registry) + } + + pub fn extensions( + &self, + force: bool, + installed: &BTreeMap<String, String>, + query: &str, + category: &str, + ) -> Result<String, RepositoryError> { + Ok( + serde_json::to_string(&self.fetch(force)?.responses(installed, query, category)) + .expect("registry JSON"), + ) + } + + pub fn categories(&self) -> Result<Vec<String>, RepositoryError> { + self.check()?; + Ok(["metadata", "download", "utility", "lyrics", "integration"] + .map(str::to_owned) + .to_vec()) + } + + pub fn download(&self, id: &str, directory: &Path) -> Result<PathBuf, RepositoryError> { + let _operation = self.enter()?; + let generation = self.state.lock().expect("repository state lock").generation; + let registry = self.fetch(false)?; + let extension = registry + .extensions + .iter() + .find(|entry| entry.id == id) + .ok_or_else(|| error(format!("extension {id} not found in repo")))?; + require_https(extension.download_url(), "extension download")?; + let destination = destination_path(directory, id, extension.download_url())?; + let check = || self.check_generation(generation).map_err(|e| e.to_string()); + let session = self.network.native_session(DOWNLOAD_TIMEOUT); + let headers = [ + ("Cache-Control".into(), "no-cache".into()), + ("Pragma".into(), "no-cache".into()), + ] + .into(); + let mut stream = session + .open_stream( + request(extension.download_url(), headers), + DOWNLOAD_TIMEOUT, + DOWNLOAD_TIMEOUT, + check, + ) + .map_err(|e| cause("failed to download", e))?; + if stream.response.status != 200 { + return Err(error(format!( + "download returned HTTP {}", + stream.response.status + ))); + } + write_verified_package( + |buffer| stream.read(buffer, check), + &destination, + extension.raw_sha256(), + &check, + )?; + Ok(destination) + } + + pub fn shutdown(&self) { + self.closed.store(true, Ordering::Release); + let pending = self.pending.lock().expect("repository operations lock"); + drop( + self.idle + .wait_while(pending, |pending| *pending > 0) + .expect("repository operations wait"), + ); + } +} + +struct Operation<'a>(&'a ExtensionRepository); + +impl Drop for Operation<'_> { + fn drop(&mut self) { + let mut pending = self.0.pending.lock().expect("repository operations lock"); + *pending -= 1; + self.0.idle.notify_all(); + } +} + +fn request(url: &str, headers: BTreeMap<String, String>) -> HttpRequest { + HttpRequest { + url: url.into(), + method: "GET".into(), + body: String::new(), + headers, + default_json: false, + user_agent: "Go-http-client/1.1".into(), + } +} + +type RegistryResponse = (u16, BTreeMap<String, Vec<String>>, Vec<u8>); + +fn read_registry( + session: &NetworkSession, + request: HttpRequest, + check: &dyn Fn() -> Result<(), String>, +) -> Result<RegistryResponse, RepositoryError> { + let mut stream = session + .open_stream(request, REGISTRY_TIMEOUT, REGISTRY_TIMEOUT, check) + .map_err(|e| cause("failed to fetch registry", e))?; + let status = stream.response.status; + let mut body = Vec::new(); + if status == 200 { + let mut buffer = [0; 64 * 1024]; + loop { + let limit = buffer.len().min(MAX_REGISTRY_BYTES + 1 - body.len()); + let count = stream + .read(&mut buffer[..limit], check) + .map_err(|e| cause("failed to read registry", e))?; + if count == 0 { + break; + } + body.extend_from_slice(&buffer[..count]); + if body.len() > MAX_REGISTRY_BYTES { + return Err(error(format!( + "registry response exceeds {MAX_REGISTRY_BYTES} bytes" + ))); + } + } + } + Ok((status, stream.response.headers, body)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cache_registry_clone_shares_arc_and_round_trips_disk_shape() { + let bytes = br#"{"registry_url":"https://example.test/registry.json","registry":{"version":1,"updated_at":"2026-09-14","extensions":[]},"cache_time":123,"etag":"etag","last_modified":"Mon, 14 Sep 2026 00:00:00 GMT"}"#; + let cache: Cache = serde_json::from_slice(bytes).unwrap(); + let clone = cache.clone(); + + assert!(Arc::ptr_eq(&cache.registry, &clone.registry)); + assert_eq!(serde_json::to_vec(&cache).unwrap(), bytes); + } +} diff --git a/rust_backend/crates/extensions/src/repository/model.rs b/rust_backend/crates/extensions/src/repository/model.rs new file mode 100644 index 00000000..81034384 --- /dev/null +++ b/rust_backend/crates/extensions/src/repository/model.rs @@ -0,0 +1,226 @@ +use super::{RepositoryError, error}; +use crate::environment::compare_versions; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::{Value, json}; +use spotiflac_core::matching::lowercase; +use std::cmp::Ordering; +use std::collections::BTreeMap; + +// Preserve Go's case-insensitive fields, duplicate-field order and null rules. +// A null scalar leaves its previous value; a null slice clears the slice. +trait NullField { + fn clear_null(&mut self) {} +} +impl NullField for String {} +impl NullField for isize {} +impl<T> NullField for Vec<T> { + fn clear_null(&mut self) { + self.clear(); + } +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(transparent)] +pub struct Tags(Vec<String>); + +impl<'de> Deserialize<'de> for Tags { + fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + Ok(Self( + Vec::<Option<String>>::deserialize(deserializer)? + .into_iter() + .map(Option::unwrap_or_default) + .collect(), + )) + } +} + +impl std::ops::Deref for Tags { + type Target = Vec<String>; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl NullField for Tags { + fn clear_null(&mut self) { + self.0.clear(); + } +} + +fn field_matches(key: &str, wire: &str) -> bool { + key.chars() + .map(|value| match value { + 'ſ' => 's', + 'K' => 'k', + value => value.to_ascii_lowercase(), + }) + .eq(wire.chars().map(|value| value.to_ascii_lowercase())) +} + +macro_rules! wire_struct { + ($name:ident {$($field:ident: $kind:ty => $wire:literal),* $(,)?}) => { + #[derive(Clone, Debug, Default, Serialize)] + pub struct $name { + $(#[serde(rename = $wire)] pub $field: $kind,)* + } + impl<'de> Deserialize<'de> for $name { + fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = $name; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a registry object") + } + fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> { + Ok($name::default()) + } + fn visit_map<A: serde::de::MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> { + let mut result = $name::default(); + while let Some(key) = map.next_key::<String>()? { + $(if field_matches(&key, $wire) { + match map.next_value::<Option<$kind>>()? { + Some(value) => result.$field = value, + None => result.$field.clear_null(), + } + continue; + })* + map.next_value::<serde::de::IgnoredAny>()?; + } + Ok(result) + } + } + deserializer.deserialize_any(Visitor) + } + } + }; +} + +wire_struct!(Registry { + version: isize => "version", + updated_at: String => "updated_at", + extensions: Vec<RepoExtension> => "extensions", +}); + +wire_struct!(RepoExtension { + id: String => "id", + name: String => "name", + display_name: String => "display_name", + version: String => "version", + description: String => "description", + download_url: String => "download_url", + icon_url: String => "icon_url", + category: String => "category", + tags: Tags => "tags", + downloads: isize => "downloads", + updated_at: String => "updated_at", + min_app_version: String => "min_app_version", + sha256: String => "sha256", + checksum_sha256: String => "checksum_sha256", + display_name_alt: String => "displayName", + download_url_alt: String => "downloadUrl", + icon_url_alt: String => "iconUrl", + min_app_version_alt: String => "minAppVersion", + checksum_alt: String => "checksumSha256", +}); + +fn first<'a>(values: &[&'a str]) -> &'a str { + values + .iter() + .copied() + .find(|value| !value.is_empty()) + .unwrap_or("") +} + +pub fn normalize_sha256(value: &str) -> String { + let normalized = lowercase(value.trim()); + let normalized = normalized.strip_prefix("sha256:").unwrap_or(&normalized); + if normalized.len() == 64 && normalized.bytes().all(|byte| byte.is_ascii_hexdigit()) { + normalized.into() + } else { + String::new() + } +} + +impl RepoExtension { + pub fn raw_sha256(&self) -> &str { + first(&[ + self.sha256.trim(), + self.checksum_sha256.trim(), + self.checksum_alt.trim(), + ]) + } + + pub fn download_url(&self) -> &str { + first(&[&self.download_url, &self.download_url_alt]) + } + + pub fn response(&self, installed: &BTreeMap<String, String>) -> Value { + let version = installed.get(&self.id); + let mut result = json!({ + "id":self.id,"name":self.name,"version":self.version, + "display_name":first(&[&self.display_name,&self.display_name_alt,&self.name]), + "description":self.description,"download_url":self.download_url(), + "category":self.category,"downloads":self.downloads,"updated_at":self.updated_at, + "is_installed":version.is_some(), + "has_update":version.is_some_and(|installed| compare_versions(&self.version,installed)==Ordering::Greater), + }); + for (key, value) in [ + ("icon_url", first(&[&self.icon_url, &self.icon_url_alt])), + ( + "min_app_version", + first(&[&self.min_app_version, &self.min_app_version_alt]), + ), + ("installed_version", version.map_or("", String::as_str)), + ("sha256", normalize_sha256(self.raw_sha256()).as_str()), + ] { + if !value.is_empty() { + result[key] = value.into(); + } + } + if !self.tags.is_empty() { + result["tags"] = json!(self.tags); + } + result + } +} + +impl Registry { + pub fn parse(body: &[u8]) -> Result<Self, RepositoryError> { + let text = crate::host::decode_go_utf8(body); + let mut registry: Self = serde_json::from_str(&text).map_err(|cause| { + if text.trim_start().starts_with('<') { + error("registry URL returned a web page instead of JSON. Make sure the URL points to a registry.json file or a GitHub repository that contains one") + } else { + error(format!("failed to parse registry: {cause}")) + } + })?; + registry.extensions.retain(|entry| { + entry.raw_sha256().is_empty() || !normalize_sha256(entry.raw_sha256()).is_empty() + }); + Ok(registry) + } + + pub fn responses( + &self, + installed: &BTreeMap<String, String>, + query: &str, + category: &str, + ) -> Vec<Value> { + let query = lowercase(query); + self.extensions + .iter() + .filter(|entry| category.is_empty() || entry.category == category) + .map(|entry| entry.response(installed)) + .filter(|entry| { + query.is_empty() + || ["name", "display_name", "description"] + .iter() + .any(|key| lowercase(entry[key].as_str().unwrap_or("")).contains(&query)) + || entry["tags"].as_array().is_some_and(|tags| { + tags.iter() + .any(|tag| lowercase(tag.as_str().unwrap_or("")).contains(&query)) + }) + }) + .collect() + } +} diff --git a/rust_backend/crates/extensions/src/repository/package.rs b/rust_backend/crates/extensions/src/repository/package.rs new file mode 100644 index 00000000..65f6502d --- /dev/null +++ b/rust_backend/crates/extensions/src/repository/package.rs @@ -0,0 +1,102 @@ +use super::{RepositoryError, cause, error, normalize_sha256}; +use sha2::{Digest, Sha256}; +use spotiflac_core::filename::sanitize_filename; +use spotiflac_core::matching::lowercase; +use spotiflac_network::url::UrlParts; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +const MAX_PACKAGE_BYTES: u64 = 64 << 20; + +pub fn destination_path(directory: &Path, id: &str, url: &str) -> Result<PathBuf, RepositoryError> { + if id.trim().is_empty() { + return Err(error("invalid extension id")); + } + let path = UrlParts::parse(url) + .map_or_else(|| url.into(), |url| crate::host::decode_go_utf8(&url.path)); + let suffix = if lowercase(&path).ends_with(".sflx") { + ".sflx" + } else { + ".spotiflac-ext" + }; + Ok(crate::files::clean( + &directory.join(format!("{}{suffix}", sanitize_filename(id))), + )) +} + +/// Streams to a private sibling file and publishes only verified bytes. An +/// interrupted reader, oversized body or checksum mismatch preserves the old +/// destination. Native callers own the destination directory during this call. +pub fn write_verified_package( + mut read: impl FnMut(&mut [u8]) -> Result<usize, String>, + destination: &Path, + expected: &str, + check: &dyn Fn() -> Result<(), String>, +) -> Result<(), RepositoryError> { + check().map_err(error)?; + let directory = destination + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + fs::create_dir_all(directory) + .map_err(|e| cause("failed to prepare extension download directory", e))?; + let name = destination + .file_name() + .unwrap_or_default() + .to_string_lossy(); + let mut file = tempfile::Builder::new() + .prefix(&format!(".{name}.download-")) + .tempfile_in(directory) + .map_err(|e| cause("failed to create extension download", e))?; + let mut hasher = Sha256::new(); + let mut written = 0u64; + let mut buffer = [0; 64 * 1024]; + loop { + check().map_err(error)?; + let limit = buffer.len().min((MAX_PACKAGE_BYTES + 1 - written) as usize); + let count = read(&mut buffer[..limit]) + .map_err(|e| cause("failed to write extension package", e))?; + if count == 0 { + break; + } + if count > limit { + return Err(error("invalid extension package reader length")); + } + written += count as u64; + if written > MAX_PACKAGE_BYTES { + return Err(error("extension package exceeds the 64 MiB size limit")); + } + file.write_all(&buffer[..count]) + .map_err(|e| cause("failed to write extension package", e))?; + hasher.update(&buffer[..count]); + } + file.as_file() + .sync_all() + .map_err(|e| cause("failed to flush extension package", e))?; + let normalized = normalize_sha256(expected); + if !expected.trim().is_empty() && normalized.is_empty() { + return Err(error( + "registry contains an invalid extension SHA-256 checksum", + )); + } + if !normalized.is_empty() { + let actual = format!("{:x}", hasher.finalize()); + // The checksum is public, but keep the comparison independent of its + // first mismatching byte, matching the Go implementation. + if actual + .bytes() + .zip(normalized.bytes()) + .fold(0, |difference, (a, b)| difference | (a ^ b)) + != 0 + { + return Err(error( + "extension package integrity check failed: SHA-256 mismatch", + )); + } + } + check().map_err(error)?; + file.persist(destination) + .map_err(|e| cause("failed to publish extension package", e))?; + Ok(()) +} diff --git a/rust_backend/crates/extensions/src/repository/urls.rs b/rust_backend/crates/extensions/src/repository/urls.rs new file mode 100644 index 00000000..27ae28bb --- /dev/null +++ b/rust_backend/crates/extensions/src/repository/urls.rs @@ -0,0 +1,49 @@ +use super::{RepositoryError, error}; +use spotiflac_network::url::UrlParts; + +#[derive(Debug, PartialEq, Eq)] +pub enum RegistryLocation { + Direct(String), + GitHub { owner: String, repository: String }, +} + +pub fn registry_location(input: &str) -> Result<RegistryLocation, RepositoryError> { + let input = input.trim(); + if input.is_empty() { + return Err(error("registry URL is empty")); + } + if input.contains("raw.githubusercontent.com") { + return Ok(RegistryLocation::Direct(input.into())); + } + let Some(path) = input + .strip_prefix("https://github.com/") + .or_else(|| input.strip_prefix("http://github.com/")) + else { + return Ok(RegistryLocation::Direct(input.into())); + }; + let mut parts = path.splitn(3, '/'); + let owner = parts.next().unwrap_or(""); + let repository = parts.next().unwrap_or(""); + if owner.is_empty() || repository.is_empty() { + return Err(error( + "invalid GitHub URL: expected github.com/<owner>/<repo>", + )); + } + Ok(RegistryLocation::GitHub { + owner: owner.into(), + repository: repository.strip_suffix(".git").unwrap_or(repository).into(), + }) +} + +pub fn require_https(input: &str, context: &str) -> Result<(), RepositoryError> { + if input.is_empty() { + return Err(error(format!("{context} URL is empty"))); + } + let url = UrlParts::parse(input) + .filter(|url| !url.hostname.is_empty()) + .ok_or_else(|| error(format!("{context} URL is invalid: {input}")))?; + if url.scheme != "https" { + return Err(error(format!("{context} URL must use https: {input}"))); + } + Ok(()) +} diff --git a/rust_backend/crates/extensions/src/resolution.rs b/rust_backend/crates/extensions/src/resolution.rs new file mode 100644 index 00000000..ee60d6f4 --- /dev/null +++ b/rust_backend/crates/extensions/src/resolution.rs @@ -0,0 +1,114 @@ +//! Resolver allowance shared by a download call and its native segment workers. + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +pub(crate) struct ResolutionBudget(Mutex<State>); + +struct State { + remaining: Duration, + started: Instant, + pauses: usize, + charges: usize, +} + +impl State { + fn remaining(&self, now: Instant) -> Duration { + if self.pauses == 0 || self.charges > 0 { + self.remaining + .saturating_sub(now.saturating_duration_since(self.started)) + } else { + self.remaining + } + } + + fn settle(&mut self, now: Instant) { + self.remaining = self.remaining(now); + self.started = now; + } +} + +impl ResolutionBudget { + pub fn new(allowance: Duration) -> Arc<Self> { + Arc::new(Self(Mutex::new(State { + remaining: allowance, + started: Instant::now(), + pauses: 0, + charges: 0, + }))) + } + + pub fn remaining(&self) -> Duration { + self.0 + .lock() + .expect("resolution budget lock") + .remaining(Instant::now()) + } + + pub fn enter(self: &Arc<Self>, charge: bool) -> BudgetGuard { + let mut state = self.0.lock().expect("resolution budget lock"); + state.settle(Instant::now()); + if charge { + state.charges += 1; + } else { + state.pauses += 1; + } + BudgetGuard { + budget: Arc::clone(self), + charge, + } + } +} + +pub(crate) struct BudgetGuard { + budget: Arc<ResolutionBudget>, + charge: bool, +} + +impl Drop for BudgetGuard { + fn drop(&mut self) { + let mut state = self.budget.0.lock().expect("resolution budget lock"); + state.settle(Instant::now()); + if self.charge { + state.charges -= 1; + } else { + state.pauses -= 1; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn charges_override_nested_pauses_and_exhaustion_is_terminal() { + let now = Instant::now(); + let mut state = State { + remaining: Duration::from_secs(60), + started: now, + pauses: 0, + charges: 0, + }; + state.settle(now + Duration::from_secs(10)); + state.pauses = 2; + assert_eq!( + state.remaining(now + Duration::from_secs(3600)), + Duration::from_secs(50) + ); + state.settle(now + Duration::from_secs(3600)); + state.charges = 1; + state.settle(now + Duration::from_secs(3610)); + assert_eq!(state.remaining, Duration::from_secs(40)); + state.charges = 0; + state.settle(now + Duration::from_secs(4000)); + state.pauses = 0; + state.settle(now + Duration::from_secs(4041)); + assert_eq!(state.remaining, Duration::ZERO); + state.pauses = 1; + assert_eq!( + state.remaining(now + Duration::from_secs(5000)), + Duration::ZERO + ); + } +} diff --git a/rust_backend/crates/extensions/src/runtime.rs b/rust_backend/crates/extensions/src/runtime.rs new file mode 100644 index 00000000..27ddb237 --- /dev/null +++ b/rust_backend/crates/extensions/src/runtime.rs @@ -0,0 +1,1458 @@ +use rquickjs::{Coerced, Context, Ctx, FromJs, Function, Object, Persistent, Runtime, Value}; +use spotiflac_core::cancellation::{CancellationError, RequestLease}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Receiver, SyncSender, TrySendError}; +use std::sync::{Arc, Condvar, Mutex, OnceLock, Weak}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +const MAX_INPUT_BYTES: usize = 8 * 1024 * 1024; +const MAX_TIMEOUT_MS: u64 = 300_000; +const QUEUE_CAPACITY: usize = 8; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum LoadMode { + Initialize, + Register, + Validate, +} + +/// Hosts granted by the trusted extension manager after permission validation. +#[derive(Clone)] +pub struct ExtensionServices { + pub legacy_backend: bool, + pub(crate) lyrics: Weak<spotiflac_providers::lyrics::LyricsService>, + pub(crate) lyrics_node: spotiflac_providers::lyrics::CallNode, + pub(crate) lyrics_parent: Option<spotiflac_providers::lyrics::CallNode>, + pub legacy_files: Option<Arc<crate::files::ExtensionFiles>>, + pub isrc: Arc<spotiflac_core::isrc::IndexCache>, + pub logs: Arc<crate::logging::LogBuffer>, + pub extension_id: String, + pub ffmpeg: Arc<crate::ffmpeg::CommandRegistry>, + pub raw_ffmpeg_stub: bool, + pub downloads: Arc<spotiflac_core::downloads::DownloadState>, + pub storage: Option<Arc<crate::storage::ExtensionStore>>, + pub files: Option<Arc<crate::files::ExtensionFiles>>, + pub transfer_policy: crate::transfer_policy::DownloadTransferPolicy, + pub network: Option<Arc<spotiflac_network::NetworkSession>>, + pub auth: Option<Arc<crate::auth::ExtensionAuth>>, + pub(crate) auth_registry: Option<Arc<crate::auth::AuthRegistry>>, + pub session: Option<Arc<crate::signed_session::SignedSessionClient>>, + pub app_version: spotiflac_core::app_version::AppVersion, + pub(crate) parent_closed: Option<Arc<AtomicBool>>, + pub(crate) startup_lease: Option<Arc<RequestLease>>, + pub(crate) initialize_empty_settings: bool, + pub(crate) load_mode: LoadMode, + pub(crate) compiled_source: Option<Arc<CompiledSource>>, +} + +impl Default for ExtensionServices { + fn default() -> Self { + Self { + legacy_backend: false, + lyrics: Weak::new(), + lyrics_node: spotiflac_providers::lyrics::CallGraph::default().node(), + lyrics_parent: None, + legacy_files: None, + isrc: Arc::new(spotiflac_core::isrc::IndexCache::default()), + logs: Arc::new(crate::logging::LogBuffer::default()), + extension_id: String::new(), + ffmpeg: Arc::new(crate::ffmpeg::CommandRegistry::default()), + raw_ffmpeg_stub: false, + downloads: Arc::new(spotiflac_core::downloads::DownloadState::default()), + storage: None, + files: None, + transfer_policy: crate::transfer_policy::DownloadTransferPolicy::default(), + network: None, + auth: None, + auth_registry: None, + session: None, + app_version: Default::default(), + parent_closed: None, + startup_lease: None, + initialize_empty_settings: true, + load_mode: LoadMode::Initialize, + compiled_source: None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ExtensionError { + #[error("extension runtime closed")] + Closed, + #[error("extension runtime busy")] + Busy, + #[error("execution timeout exceeded")] + Timeout, + #[error("{0}")] + Cancelled(CancellationError), + #[error("extension did not call registerExtension()")] + NotRegistered, + #[error("extension function not found: {0}")] + MissingFunction(String), + #[error("invalid extension input: {0}")] + InvalidInput(String), + #[error("{0}")] + Script(String), +} + +#[derive(Clone, Debug)] +pub struct RuntimeLimits { + pub memory_bytes: usize, + pub stack_bytes: usize, + pub timeout_ms: u64, +} + +impl Default for RuntimeLimits { + fn default() -> Self { + Self { + memory_bytes: 64 * 1024 * 1024, + stack_bytes: 512 * 1024, + timeout_ms: 30_000, + } + } +} + +impl RuntimeLimits { + fn validate(&self) -> Result<(), ExtensionError> { + if !(1024 * 1024..=256 * 1024 * 1024).contains(&self.memory_bytes) + || !(64 * 1024..=1024 * 1024).contains(&self.stack_bytes) + { + return Err(ExtensionError::InvalidInput( + "memory must be 1–256 MiB and stack must be 64–1024 KiB".to_owned(), + )); + } + Ok(()) + } + + fn timeout(&self, requested_ms: u64) -> Duration { + let milliseconds = if requested_ms == 0 { + self.timeout_ms.max(1) + } else { + requested_ms + }; + Duration::from_millis(milliseconds.min(MAX_TIMEOUT_MS)) + } +} + +struct Operation { + item_id: String, + deadline: Instant, + lease: Option<Arc<RequestLease>>, + resolution: Option<Arc<crate::resolution::ResolutionBudget>>, +} + +#[derive(Default)] +pub(crate) struct Control { + closed: AtomicBool, + parent_closed: Option<Arc<AtomicBool>>, + operation: Mutex<Option<Operation>>, + waiter: Mutex<Option<Arc<thread::Thread>>>, + #[cfg(test)] + checks: std::sync::atomic::AtomicUsize, +} + +pub(crate) struct ControlWait<'a>(&'a Control); + +impl Drop for ControlWait<'_> { + fn drop(&mut self) { + self.0.waiter.lock().expect("extension waiter lock").take(); + } +} + +impl Control { + pub(crate) fn watch_thread(&self) -> ControlWait<'_> { + let thread = Arc::new(thread::current()); + *self.waiter.lock().expect("extension waiter lock") = Some(Arc::clone(&thread)); + if let Some(lease) = self + .operation + .lock() + .expect("extension operation lock") + .as_ref() + .and_then(|operation| operation.lease.as_ref()) + { + lease.observe_thread(&thread); + } + ControlWait(self) + } + + pub(crate) fn close(&self) { + self.closed.store(true, Ordering::Release); + if let Some(thread) = self.waiter.lock().expect("extension waiter lock").as_ref() { + thread.unpark(); + } + } + + pub(crate) fn deadline_remaining(&self) -> Duration { + self.operation + .lock() + .expect("extension operation lock") + .as_ref() + .map_or(Duration::from_millis(MAX_TIMEOUT_MS), |operation| { + operation.deadline.saturating_duration_since(Instant::now()) + }) + } + + pub(crate) fn request_cancelled(&self) -> bool { + self.operation + .lock() + .expect("extension operation lock") + .as_ref() + .and_then(|operation| operation.lease.as_ref()) + .is_some_and(|lease| lease.active_request_cancelled()) + } + + pub(crate) fn item_id(&self) -> String { + self.operation + .lock() + .expect("extension operation lock") + .as_ref() + .map_or_else(String::new, |operation| operation.item_id.clone()) + } + + pub(crate) fn check(&self) -> Result<(), ExtensionError> { + #[cfg(test)] + self.checks.fetch_add(1, Ordering::Relaxed); + if self.is_closed() { + return Err(ExtensionError::Closed); + } + if let Some(operation) = &*self.operation.lock().expect("extension operation lock") { + if let Some(lease) = &operation.lease { + lease.check_active().map_err(ExtensionError::Cancelled)?; + } + if Instant::now() >= operation.deadline + || operation + .resolution + .as_ref() + .is_some_and(|budget| budget.remaining().is_zero()) + { + return Err(ExtensionError::Timeout); + } + } + Ok(()) + } + + pub(crate) fn resolution(&self) -> Option<Arc<crate::resolution::ResolutionBudget>> { + self.operation + .lock() + .expect("extension operation lock") + .as_ref() + .and_then(|operation| operation.resolution.clone()) + } + + fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + || self + .parent_closed + .as_ref() + .is_some_and(|closed| closed.load(Ordering::Acquire)) + } + + pub(crate) fn sleep(&self, duration: Duration) -> bool { + let started = Instant::now(); + loop { + if self.check().is_err() { + return false; + } + let remaining = duration.saturating_sub(started.elapsed()); + if remaining.is_zero() { + return true; + } + // Core leases and shutdown can be signalled from native threads. + // A short heartbeat also observes deadlines while in a host call. + thread::sleep(remaining.min(Duration::from_millis(10))); + } + } +} + +type Reply = SyncSender<Result<String, ExtensionError>>; + +#[derive(Clone, Copy)] +enum CallKind { + Function, + Lifecycle, + Action, + Provider, + DownloadProvider, + PostProcess, +} + +enum Command { + Call { + method: String, + arguments: String, + operation: Operation, + reply: Reply, + kind: CallKind, + }, + Stop, +} + +/// A stateful VM owned by one dedicated worker. No QuickJS value crosses threads. +/// +/// Calls are serialized and bounded, including queue time, JS jobs, and host +/// utilities. An interrupted VM is retired: callers must reload the extension +/// rather than letting pending callbacks escape into a subsequent request. +pub struct ExtensionRuntime { + pub(crate) compiled_source: Arc<CompiledSource>, + lyrics_node: spotiflac_providers::lyrics::CallNode, + lyrics_parent: Option<spotiflac_providers::lyrics::CallNode>, + downloads: Arc<spotiflac_core::downloads::DownloadState>, + sender: SyncSender<Command>, + control: Arc<Control>, + worker: Mutex<Option<JoinHandle<()>>>, + limits: RuntimeLimits, + session: Option<Arc<crate::signed_session::SignedSessionClient>>, + network: Option<Arc<spotiflac_network::NetworkSession>>, + native_calls: Mutex<usize>, + native_idle: Condvar, +} + +impl ExtensionRuntime { + pub fn load( + source: &str, + settings_json: &str, + limits: RuntimeLimits, + ) -> Result<Self, ExtensionError> { + Self::load_with_services(source, settings_json, limits, ExtensionServices::default()) + } + + pub fn load_with_services( + source: &str, + settings_json: &str, + limits: RuntimeLimits, + mut services: ExtensionServices, + ) -> Result<Self, ExtensionError> { + limits.validate()?; + validate_size(source)?; + let compiled_source = services + .compiled_source + .take() + .filter(|cached| cached.source.as_ref() == source) + .unwrap_or_else(|| { + Arc::new(CompiledSource { + source: source.into(), + bytecode: OnceLock::new(), + }) + }); + services.compiled_source = Some(Arc::clone(&compiled_source)); + let settings = validate_json(settings_json)?; + if !settings.is_object() { + return Err(ExtensionError::InvalidInput( + "settings must be an object".to_owned(), + )); + } + let control = Arc::new(Control { + parent_closed: services.parent_closed.clone(), + ..Control::default() + }); + *control.operation.lock().expect("extension operation lock") = Some(Operation { + item_id: String::new(), + deadline: Instant::now() + limits.timeout(0), + lease: services.startup_lease.clone(), + resolution: None, + }); + let (sender, receiver) = mpsc::sync_channel(QUEUE_CAPACITY); + let (ready, result) = mpsc::sync_channel(1); + let worker_control = Arc::clone(&control); + let worker_limits = limits.clone(); + let source = Arc::clone(&compiled_source.source); + let settings = settings_json.to_owned(); + let session = services.session.clone(); + let network = services.network.clone(); + let downloads = Arc::clone(&services.downloads); + let lyrics_node = services.lyrics_node.clone(); + let lyrics_parent = services.lyrics_parent.clone(); + // Isolated VM loading happens while its primary manager engine is held. + let _dependency = lyrics_parent + .as_ref() + .map(|parent| parent.wait_for(&lyrics_node)) + .transpose() + .map_err(ExtensionError::Script)?; + let worker = thread::Builder::new() + .name("extension-js".to_owned()) + .stack_size(4 * 1024 * 1024) + .spawn(move || { + run_worker( + receiver, + ready, + &worker_control, + &worker_limits, + &source, + &settings, + &services, + ); + worker_control.close(); + }) + .map_err(|error| ExtensionError::Script(format!("start extension worker: {error}")))?; + let runtime = Self { + compiled_source, + lyrics_node, + lyrics_parent, + downloads, + sender, + control, + worker: Mutex::new(Some(worker)), + limits, + session, + network, + native_calls: Mutex::new(0), + native_idle: Condvar::new(), + }; + result.recv().unwrap_or(Err(ExtensionError::Closed))?; + Ok(runtime) + } + + pub fn call( + &self, + method: &str, + arguments_json: &str, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + ) -> Result<String, ExtensionError> { + self.call_with_operation( + method, + arguments_json, + Operation { + item_id: String::new(), + deadline: Instant::now() + self.limits.timeout(timeout_ms), + lease, + resolution: None, + }, + CallKind::Function, + ) + } + + pub(crate) fn managed_call( + &self, + method: &str, + arguments: &str, + action: bool, + timeout_ms: u64, + ) -> Result<String, ExtensionError> { + self.managed_call_with_lease(method, arguments, action, timeout_ms, None) + } + + pub(crate) fn managed_call_with_lease( + &self, + method: &str, + arguments: &str, + action: bool, + timeout_ms: u64, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionError> { + // Initialize/cleanup of a pooled VM also holds the primary engine. + // Its regular download call runs without this dependency. + let _dependency = self + .lyrics_parent + .as_ref() + .map(|parent| parent.wait_for(&self.lyrics_node)) + .transpose() + .map_err(ExtensionError::Script)?; + self.call_with_operation( + method, + arguments, + Operation { + item_id: String::new(), + deadline: Instant::now() + self.limits.timeout(timeout_ms), + lease, + resolution: None, + }, + if action { + CallKind::Action + } else { + CallKind::Lifecycle + }, + ) + } + + /// Read typed provider fields in the owning VM. Go's provider wrappers do + /// not await a returned Promise; the provider object itself is parsed. + pub fn call_provider( + &self, + method: &str, + arguments: &str, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + ) -> Result<String, ExtensionError> { + self.call_provider_operation(method, arguments, lease, timeout_ms, String::new()) + } + + pub(crate) fn call_provider_operation( + &self, + method: &str, + arguments: &str, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + item_id: String, + ) -> Result<String, ExtensionError> { + self.call_with_operation( + method, + arguments, + Operation { + item_id, + deadline: Instant::now() + self.limits.timeout(timeout_ms), + lease, + resolution: None, + }, + CallKind::Provider, + ) + } + + pub(crate) fn is_closed(&self) -> bool { + self.control.is_closed() + } + + pub(crate) fn call_post_process( + &self, + arguments: &str, + timeout_ms: u64, + item_id: &str, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionError> { + self.call_with_operation( + "postProcessV2", + arguments, + Operation { + item_id: item_id.into(), + deadline: Instant::now() + + self + .limits + .timeout(if timeout_ms == 0 { 120_000 } else { timeout_ms }), + lease, + resolution: None, + }, + CallKind::PostProcess, + ) + } + + pub(crate) fn network_session(&self) -> Option<Arc<spotiflac_network::NetworkSession>> { + self.network.clone() + } + + pub(crate) fn call_download_provider( + &self, + arguments: &str, + item_id: &str, + resolution_timeout_ms: u64, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, (ExtensionError, bool)> { + let lease = match lease { + Some(lease) => lease, + None => Arc::new( + self.downloads + .acquire(item_id) + .map_err(|error| (ExtensionError::Cancelled(error), false))?, + ), + }; + if !item_id.is_empty() { + let _ = self.downloads.progress.preparing(item_id, ""); + } + let allowance = if resolution_timeout_ms == 0 { + 60_000 + } else { + resolution_timeout_ms.min(MAX_TIMEOUT_MS) + }; + let budget = crate::resolution::ResolutionBudget::new(Duration::from_millis(allowance)); + self.call_with_operation( + "download", + arguments, + Operation { + item_id: item_id.to_owned(), + deadline: Instant::now() + Duration::from_secs(24 * 60 * 60), + lease: Some(lease), + resolution: Some(Arc::clone(&budget)), + }, + CallKind::DownloadProvider, + ) + .map_err(|error| (error, budget.remaining().is_zero())) + } + + /// Native provider-manager entry point. Resolver/callback/retry time shares + /// one allowance; established native media reads pause only that allowance. + pub fn call_download( + &self, + method: &str, + arguments_json: &str, + lease: Option<Arc<RequestLease>>, + resolution_timeout_ms: u64, + ) -> Result<String, ExtensionError> { + self.call_download_operation( + method, + arguments_json, + lease, + resolution_timeout_ms, + String::new(), + ) + } + + /// The manager starts progress before resolution and completes it after + /// finalization. This call acquires another reference to the same item and + /// binds its identity to the queued command, never to mutable runtime state. + pub fn call_download_for_item( + &self, + method: &str, + arguments_json: &str, + item_id: &str, + resolution_timeout_ms: u64, + ) -> Result<String, ExtensionError> { + if self.control.is_closed() { + return Err(ExtensionError::Closed); + } + let lease = Arc::new( + self.downloads + .acquire(item_id) + .map_err(ExtensionError::Cancelled)?, + ); + self.call_download_operation( + method, + arguments_json, + Some(lease), + resolution_timeout_ms, + item_id.trim().to_owned(), + ) + } + + pub fn download_state(&self) -> Arc<spotiflac_core::downloads::DownloadState> { + Arc::clone(&self.downloads) + } + + fn call_download_operation( + &self, + method: &str, + arguments_json: &str, + lease: Option<Arc<RequestLease>>, + resolution_timeout_ms: u64, + item_id: String, + ) -> Result<String, ExtensionError> { + let allowance = if resolution_timeout_ms == 0 { + 60_000 + } else { + resolution_timeout_ms.min(MAX_TIMEOUT_MS) + }; + self.call_with_operation( + method, + arguments_json, + Operation { + item_id, + deadline: Instant::now() + Duration::from_secs(24 * 60 * 60), + lease, + resolution: Some(crate::resolution::ResolutionBudget::new( + Duration::from_millis(allowance), + )), + }, + CallKind::Function, + ) + } + + fn call_with_operation( + &self, + method: &str, + arguments_json: &str, + operation: Operation, + kind: CallKind, + ) -> Result<String, ExtensionError> { + if self.control.is_closed() { + return Err(ExtensionError::Closed); + } + let arguments = validate_json(arguments_json)?; + if !arguments.is_array() { + return Err(ExtensionError::InvalidInput( + "arguments must be an array".to_owned(), + )); + } + validate_size(method)?; + if let Some(lease) = &operation.lease { + lease.check_active().map_err(ExtensionError::Cancelled)?; + } + let (reply, result) = mpsc::sync_channel(1); + let command = Command::Call { + method: method.to_owned(), + arguments: arguments_json.to_owned(), + operation, + reply, + kind, + }; + self.sender.try_send(command).map_err(|error| match error { + TrySendError::Full(_) => ExtensionError::Busy, + TrySendError::Disconnected(_) => ExtensionError::Closed, + })?; + result.recv().unwrap_or(Err(ExtensionError::Closed)) + } + + /// Session preflight touches thread-safe hosts without occupying the VM. + pub fn preflight_signed_session( + &self, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + ) -> Result<bool, ExtensionError> { + let mut pending = self + .native_calls + .lock() + .expect("extension native calls lock"); + if self.control.is_closed() { + return Err(ExtensionError::Closed); + } + *pending += 1; + drop(pending); + let _operation = NativeOperation(self); + let deadline = Instant::now() + self.limits.timeout(timeout_ms); + let check = || { + if self.control.is_closed() { + return Err(ExtensionError::Closed); + } + if let Some(lease) = &lease { + lease.check_active().map_err(ExtensionError::Cancelled)?; + } + if Instant::now() >= deadline { + return Err(ExtensionError::Timeout); + } + Ok(()) + }; + check()?; + let result = self.session.as_ref().map_or(Ok(false), |session| { + session.preflight(|| check().map_err(|error| error.to_string())) + }); + check()?; + result.map_err(ExtensionError::Script) + } + + pub fn take_verification_url(&self) -> String { + self.session + .as_ref() + .map_or_else(String::new, |session| session.take_verification_url()) + } + + /// Terminal, idempotent interruption, including queued work. Does not run JS. + /// A manager should call the extension's cleanup hook before orderly shutdown. + pub fn shutdown(&self) { + self.control.close(); + let _ = self.sender.try_send(Command::Stop); + if let Some(worker) = self.worker.lock().expect("extension worker lock").take() { + let _ = worker.join(); + } + let mut pending = self + .native_calls + .lock() + .expect("extension native calls lock"); + while *pending > 0 { + pending = self + .native_idle + .wait(pending) + .expect("extension native calls idle"); + } + } +} + +struct NativeOperation<'a>(&'a ExtensionRuntime); +impl Drop for NativeOperation<'_> { + fn drop(&mut self) { + let mut pending = self + .0 + .native_calls + .lock() + .expect("extension native calls lock"); + *pending -= 1; + self.0.native_idle.notify_all(); + } +} + +impl Drop for ExtensionRuntime { + fn drop(&mut self) { + self.shutdown(); + } +} + +fn run_worker( + receiver: Receiver<Command>, + ready: SyncSender<Result<(), ExtensionError>>, + control: &Arc<Control>, + limits: &RuntimeLimits, + source: &str, + settings: &str, + services: &ExtensionServices, +) { + let vm = match Vm::load(control, limits, source, settings, services) { + Ok(vm) => vm, + Err(error) => { + let _ = ready.send(Err(error)); + return; + } + }; + *control.operation.lock().expect("extension operation lock") = None; + if services.load_mode == LoadMode::Validate { + // Validation owns a fresh VM only until cleanup finishes. Keeping both + // steps on this worker avoids a caller/worker round trip and never + // exposes the validation VM to a later provider call. + let result = vm.execute( + control, + "cleanup", + "[]", + Operation { + item_id: String::new(), + deadline: Instant::now() + limits.timeout(0), + lease: None, + resolution: None, + }, + CallKind::Lifecycle, + ); + if let Err(error) = lifecycle_result("cleanup", result) { + let _ = services + .logs + .backend(&format!("[Extension] Cleanup error: {error}")); + } + let _ = ready.send(Ok(())); + return; + } + if ready.send(Ok(())).is_err() { + return; + } + while let Ok(command) = receiver.recv() { + let Command::Call { + method, + arguments, + operation, + reply, + kind, + } = command + else { + break; + }; + let result = vm.execute(control, &method, &arguments, operation, kind); + let _ = reply.send(result); + if control.closed.load(Ordering::Acquire) { + // Dropping receiver rejects all queued callers. No stale jobs survive. + break; + } + } +} + +pub(crate) fn lifecycle_result( + method: &str, + output: Result<String, ExtensionError>, +) -> Result<(), ExtensionError> { + let result: serde_json::Value = serde_json::from_str(&output?) + .map_err(|error| ExtensionError::Script(error.to_string()))?; + if result["success"] == false { + return Err(ExtensionError::Script(format!( + "{method} failed: {}", + result["error"].as_str().unwrap_or("unknown error") + ))); + } + Ok(()) +} + +struct Vm { + gateway: Persistent<Object<'static>>, + context: Context, + // Drop the context and its persistent values before the owning runtime. + _runtime: Runtime, +} + +pub(crate) struct CompiledSource { + source: Arc<str>, + bytecode: OnceLock<Vec<u8>>, +} + +fn builtin_factory<'js>( + ctx: &Ctx<'js>, + source: &'static str, + cache: &OnceLock<Vec<u8>>, +) -> rquickjs::Result<Function<'js>> { + Function::from_js(ctx, eval_cached(ctx, source, cache, true)?) +} + +// Bytecode is compiled locally by this engine. Each VM decodes its own objects; +// installed source caches belong to one primary runtime/install generation. +#[allow(unsafe_code)] +fn eval_cached<'js>( + ctx: &Ctx<'js>, + source: &str, + cache: &OnceLock<Vec<u8>>, + strict: bool, +) -> rquickjs::Result<Value<'js>> { + use rquickjs::qjs; + + let context = ctx.as_raw().as_ptr(); + // Match Ctx::eval's stack bookkeeping before entering QuickJS. + unsafe { qjs::JS_UpdateStackTop(qjs::JS_GetRuntime(context)) }; + let compiled = if let Some(bytes) = cache.get() { + // JS_ReadObject accepts only bytecode produced locally by this engine. + unsafe { + qjs::JS_ReadObject( + context, + bytes.as_ptr(), + bytes.len() as _, + qjs::JS_READ_OBJ_BYTECODE as i32, + ) + } + } else { + let mut source = source.as_bytes().to_vec(); + source.push(0); + let flags = qjs::JS_EVAL_TYPE_GLOBAL + | qjs::JS_EVAL_FLAG_COMPILE_ONLY + | if strict { qjs::JS_EVAL_FLAG_STRICT } else { 0 }; + // The owned value belongs to this context; its RAII wrapper releases it + // on every return path, including compilation/serialization failure. + let compiled = unsafe { + Value::from_raw( + ctx.clone(), + qjs::JS_Eval( + context, + source.as_ptr().cast(), + (source.len() - 1) as _, + if strict { + c"eval_script".as_ptr() + } else { + c"index.js".as_ptr() + }, + flags as i32, + ), + ) + }; + if compiled.is_exception() { + return Err(rquickjs::Error::Exception); + } + let mut len = 0; + let buffer = unsafe { + qjs::JS_WriteObject( + context, + &mut len, + compiled.as_raw(), + qjs::JS_WRITE_OBJ_BYTECODE as i32, + ) + }; + if buffer.is_null() { + // Caching is optional under memory/stack pressure. Keep the already + // compiled script and let execution enforce the VM's limits. + drop(ctx.catch()); + } else { + if len as usize <= MAX_INPUT_BYTES { + let bytes = unsafe { std::slice::from_raw_parts(buffer, len as _).to_vec() }; + let _ = cache.set(bytes); + } + unsafe { qjs::js_free(context, buffer.cast()) }; + } + // Give JS_EvalFunction its own reference; the wrapper still releases + // both its original value and context reference on this branch. + unsafe { qjs::JS_DupValue(context, compiled.as_raw()) } + }; + if unsafe { qjs::JS_IsException(compiled) } { + return Err(rquickjs::Error::Exception); + } + // JS_EvalFunction consumes compiled. Only its owned result gets a wrapper. + let result = unsafe { Value::from_raw(ctx.clone(), qjs::JS_EvalFunction(context, compiled)) }; + if result.is_exception() { + return Err(rquickjs::Error::Exception); + } + Ok(result) +} + +impl Vm { + fn load( + control: &Arc<Control>, + limits: &RuntimeLimits, + source: &str, + settings: &str, + services: &ExtensionServices, + ) -> Result<Self, ExtensionError> { + let runtime = Runtime::new().map_err(|error| ExtensionError::Script(error.to_string()))?; + runtime.set_memory_limit(limits.memory_bytes); + runtime.set_max_stack_size(limits.stack_bytes); + let interrupt = Arc::clone(control); + runtime.set_interrupt_handler(Some(Box::new(move || interrupt.check().is_err()))); + let context = + Context::full(&runtime).map_err(|error| ExtensionError::Script(error.to_string()))?; + let gateway = context.with(|ctx| { + static PRELUDE: OnceLock<Vec<u8>> = OnceLock::new(); + static PROVIDER: OnceLock<Vec<u8>> = OnceLock::new(); + let host = crate::host::register(&ctx, Arc::clone(control), services) + .map_err(|error| script_error(&ctx, control, error))?; + let factory = builtin_factory(&ctx, include_str!("prelude.js"), &PRELUDE) + .map_err(|error| script_error(&ctx, control, error))?; + let gateway: Object = factory + .call((host.clone(),)) + .map_err(|error| script_error(&ctx, control, error))?; + let provider_factory = builtin_factory(&ctx, include_str!("provider.js"), &PROVIDER) + .map_err(|error| script_error(&ctx, control, error))?; + let helpers: Object = gateway + .get("providerHelpers") + .map_err(|error| script_error(&ctx, control, error))?; + let provider: Object = provider_factory + .call((host, helpers)) + .map_err(|error| script_error(&ctx, control, error))?; + gateway + .set("provider", provider) + .map_err(|error| script_error(&ctx, control, error))?; + // Goja compiles extension scripts without forcing strict mode. + let cache = &services + .compiled_source + .as_ref() + .expect("local source cache") + .bytecode; + eval_cached(&ctx, source, cache, false) + .map_err(|error| script_error(&ctx, control, error))?; + drain_jobs(&ctx, control)?; + let registered: Function = gateway + .get("registered") + .map_err(|error| script_error(&ctx, control, error))?; + if !registered + .call::<_, bool>(()) + .map_err(|error| script_error(&ctx, control, error))? + { + return Err(ExtensionError::NotRegistered); + } + Ok(Persistent::save(&ctx, gateway)) + })?; + let vm = Self { + gateway, + context, + _runtime: runtime, + }; + let empty_settings = validate_json(settings)? + .as_object() + .is_some_and(|settings| settings.is_empty()); + if services.load_mode == LoadMode::Initialize + && (services.initialize_empty_settings || !empty_settings) + { + vm.call( + control, + "initialize", + &format!("[{settings}]"), + true, + CallKind::Function, + )?; + } + control.check()?; + Ok(vm) + } + + fn execute( + &self, + control: &Control, + method: &str, + arguments: &str, + operation: Operation, + kind: CallKind, + ) -> Result<String, ExtensionError> { + *control.operation.lock().expect("extension operation lock") = Some(operation); + // A queued cancellation/expiry has not touched the VM and need not retire it. + if let Err(error) = control.check() { + *control.operation.lock().expect("extension operation lock") = None; + return Err(error); + } + let result = self.call(control, method, arguments, false, kind); + // A thrown synchronous error can still leave Promise jobs in the queue. + let jobs = self.context.with(|ctx| drain_jobs(&ctx, control)); + let result = result.and_then(|value| jobs.map(|()| value)); + let interruption = control.check().err(); + *control.operation.lock().expect("extension operation lock") = None; + if interruption.is_some() { + // Callers must see the retired VM before handling its interruption. + control.close(); + } + interruption.map_or(result, Err) + } + + fn call( + &self, + control: &Control, + method: &str, + arguments: &str, + optional: bool, + kind: CallKind, + ) -> Result<String, ExtensionError> { + self.context.with(|ctx| { + let result = (|| -> rquickjs::Result<Option<Value<'_>>> { + let gateway = self.gateway.clone().restore(&ctx)?; + if matches!( + kind, + CallKind::Provider | CallKind::DownloadProvider | CallKind::PostProcess + ) { + let provider: Object = gateway.get("provider")?; + let invoke: Function = + provider.get(if matches!(kind, CallKind::DownloadProvider) { + "invokeDownload" + } else if matches!(kind, CallKind::PostProcess) { + "invokePostProcess" + } else { + "invoke" + })?; + let arguments: rquickjs::Array = ctx.json_parse(arguments)?.get()?; + return Ok(Some(invoke.call((method, arguments))?)); + } + if !matches!(kind, CallKind::Function) { + let invoke: Function = gateway.get("invokeManaged")?; + let arguments: rquickjs::Array = ctx.json_parse(arguments)?.get()?; + return Ok(Some(invoke.call(( + matches!(kind, CallKind::Action), + method, + arguments, + ))?)); + } + let get: Function = gateway.get("getExtension")?; + let extension: Value = get.call(())?; + let Some(object) = extension.as_object() else { + return Ok(None); + }; + let function: Value = object.get(method)?; + let Some(function) = function.as_function() else { + return Ok(None); + }; + let arguments: rquickjs::Array = ctx.json_parse(arguments)?.get()?; + let mut args = rquickjs::function::Args::new(ctx.clone(), arguments.len()); + args.this(object.clone())?; + for argument in arguments.iter::<Value>() { + args.push_arg(argument?)?; + } + Ok(Some(function.call_arg(args)?)) + })() + .map_err(|error| script_error(&ctx, control, error))?; + let Some(mut value) = result else { + return if optional { + Ok("null".to_owned()) + } else { + Err(ExtensionError::MissingFunction(method.to_owned())) + }; + }; + if !optional + && matches!(kind, CallKind::Function) + && let Some(promise) = value.as_promise() + { + value = loop { + control.check()?; + if let Some(result) = promise.result::<Value>() { + break result.map_err(|error| script_error(&ctx, control, error))?; + } + if !ctx.execute_pending_job() { + if ctx.has_exception() { + return Err(script_error(&ctx, control, rquickjs::Error::Exception)); + } + control.sleep(Duration::from_millis(10)); + } + }; + } + drain_jobs(&ctx, control)?; + if optional { + // The Go manager ignores initialize's return value. + return Ok("null".to_owned()); + } + let gateway = self + .gateway + .clone() + .restore(&ctx) + .map_err(|error| script_error(&ctx, control, error))?; + if matches!( + kind, + CallKind::Provider | CallKind::DownloadProvider | CallKind::PostProcess + ) { + let provider: Object = gateway + .get("provider") + .map_err(|error| script_error(&ctx, control, error))?; + let parse: Function = provider + .get(if matches!(kind, CallKind::PostProcess) { + "parsePostProcess" + } else { + "parse" + }) + .map_err(|error| script_error(&ctx, control, error))?; + let json: String = parse + .call((method, value)) + .map_err(|error| script_error(&ctx, control, error))?; + drain_jobs(&ctx, control)?; + control.check()?; + return Ok(json); + } + let serialize: Function = gateway + .get("serialize") + .map_err(|error| script_error(&ctx, control, error))?; + let json: String = serialize + .call((value,)) + .map_err(|error| script_error(&ctx, control, error))?; + // Go export serialization may execute extension-defined getters and jobs. + drain_jobs(&ctx, control)?; + control.check()?; + Ok(json) + }) + } +} + +fn drain_jobs(ctx: &Ctx<'_>, control: &Control) -> Result<(), ExtensionError> { + loop { + control.check()?; + let ran = ctx.execute_pending_job(); + if ctx.has_exception() { + return Err(script_error(ctx, control, rquickjs::Error::Exception)); + } + if !ran { + return Ok(()); + } + } +} + +fn script_error(ctx: &Ctx<'_>, control: &Control, error: rquickjs::Error) -> ExtensionError { + if let Err(interruption) = control.check() { + return interruption; + } + let message = if error.is_exception() { + Coerced::<String>::from_js(ctx, ctx.catch()) + .map(|message| message.0) + .unwrap_or_else(|_| "JavaScript exception".to_owned()) + } else { + error.to_string() + }; + // Even converting a thrown value to text may execute arbitrary JS. + control + .check() + .err() + .unwrap_or(ExtensionError::Script(message)) +} + +fn validate_size(value: &str) -> Result<(), ExtensionError> { + if value.len() > MAX_INPUT_BYTES { + return Err(ExtensionError::InvalidInput( + "input exceeds 8 MiB".to_owned(), + )); + } + Ok(()) +} + +fn validate_json(value: &str) -> Result<serde_json::Value, ExtensionError> { + validate_size(value)?; + serde_json::from_str(value).map_err(|error| ExtensionError::InvalidInput(error.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cached_source_still_obeys_each_vms_memory_and_stack_limits() { + let source = "registerExtension({allocate(){const a=[];while(true)a.push(new Uint8Array(65536));},recurse(){function f(){return 1+f()}return f()}});"; + let primary = ExtensionRuntime::load(source, "{}", RuntimeLimits::default()).unwrap(); + let runtime = ExtensionRuntime::load_with_services( + source, + "{}", + RuntimeLimits { + memory_bytes: 2 << 20, + stack_bytes: 128 << 10, + ..Default::default() + }, + ExtensionServices { + compiled_source: Some(Arc::clone(&primary.compiled_source)), + ..Default::default() + }, + ) + .unwrap(); + for method in ["recurse", "allocate"] { + assert!( + matches!( + runtime.call(method, "[]", None, 1000), + Err(ExtensionError::Script(_)) + ), + "{method}" + ); + } + } + + #[test] + fn compiled_source_reuses_bytecode_but_isolates_state_settings_and_new_source() { + let source = "let n=0,s={};registerExtension({initialize(v){s=v;if(v.spin)while(true){}},next(){return [++n,s.id]}});"; + let primary = + ExtensionRuntime::load(source, "{\"id\":0}", RuntimeLimits::default()).unwrap(); + let cache = Arc::clone(&primary.compiled_source); + assert!(cache.bytecode.get().is_some()); + thread::scope(|scope| { + let workers: Vec<_> = (1..=4) + .map(|id| { + let cache = Arc::clone(&cache); + scope.spawn(move || { + let runtime = ExtensionRuntime::load_with_services( + source, + &format!("{{\"id\":{id}}}"), + RuntimeLimits::default(), + ExtensionServices { + compiled_source: Some(Arc::clone(&cache)), + ..Default::default() + }, + ) + .unwrap(); + assert!(Arc::ptr_eq(&cache, &runtime.compiled_source)); + assert_eq!( + runtime.call("next", "[]", None, 1000).unwrap(), + format!("[1,{id}]") + ); + assert_eq!( + runtime.call("next", "[]", None, 1000).unwrap(), + format!("[2,{id}]") + ); + }) + }) + .collect(); + for worker in workers { + worker.join().unwrap(); + } + }); + assert_eq!(primary.call("next", "[]", None, 1000).unwrap(), "[1,0]"); + let timed_out = ExtensionRuntime::load_with_services( + source, + "{\"spin\":true}", + RuntimeLimits { + timeout_ms: 30, + ..Default::default() + }, + ExtensionServices { + compiled_source: Some(Arc::clone(&cache)), + ..Default::default() + }, + ); + assert!(matches!(timed_out, Err(ExtensionError::Timeout))); + let changed = ExtensionRuntime::load_with_services( + "registerExtension({next(){return 99}});", + "{}", + RuntimeLimits::default(), + ExtensionServices { + compiled_source: Some(Arc::clone(&cache)), + ..Default::default() + }, + ) + .unwrap(); + assert!(!Arc::ptr_eq(&cache, &changed.compiled_source)); + assert_eq!(changed.call("next", "[]", None, 1000).unwrap(), "99"); + } + + #[test] + fn ffmpeg_waits_for_completion_cancellation_or_deadline_without_polling() { + use spotiflac_core::cancellation::{CancellationDomain, CancellationRegistry}; + for mode in [ + "complete", + "cancel", + "release", + "cancel-shutdown", + "close", + "shutdown", + "timeout", + ] { + let registry = crate::ffmpeg::CommandRegistry::default(); + let cancellation = CancellationRegistry::new(CancellationDomain::Download); + let lease = Arc::new(cancellation.acquire("example").unwrap()); + let control = Control::default(); + *control.operation.lock().unwrap() = Some(Operation { + item_id: "example".into(), + deadline: Instant::now() + + if mode == "timeout" { + Duration::from_millis(100) + } else { + Duration::from_secs(1) + }, + lease: Some(Arc::clone(&lease)), + resolution: None, + }); + thread::scope(|scope| { + let worker = scope.spawn(|| { + registry.execute("example", vec![], "in".into(), "out".into(), &control) + }); + let pending: serde_json::Value = + serde_json::from_str(®istry.wait_pending_json(500).unwrap()).unwrap(); + let id = pending[0]["command_id"].as_str().unwrap(); + thread::sleep(Duration::from_millis(45)); + assert!( + control.checks.load(Ordering::Relaxed) <= 3, + "FFmpeg polled during {mode}" + ); + let result = || crate::ffmpeg::CommandResult { + success: true, + output: "done".into(), + error: String::new(), + }; + let began = Instant::now(); + match mode { + "complete" => { + assert!(registry.complete(id, result()).unwrap()); + } + "cancel" => cancellation.cancel("example").unwrap(), + "release" => lease.release(), + "cancel-shutdown" => cancellation.shutdown(), + "close" => control.close(), + "shutdown" => registry.shutdown(), + _ => {} + } + let finished = worker.join().unwrap(); + assert!( + began.elapsed() < Duration::from_millis(300), + "late wake: {mode}" + ); + if mode == "complete" { + assert!(finished.unwrap().success); + } else { + assert!(finished.is_err(), "{mode}"); + } + if mode != "shutdown" { + assert!(!registry.complete(id, result()).unwrap()); + } + }); + } + } + + #[test] + fn interrupted_worker_is_closed_before_publishing_its_reply() { + let services = ExtensionServices::default(); + services.logs.set_enabled(true).unwrap(); + let logs = Arc::clone(&services.logs); + let downloads = Arc::clone(&services.downloads); + let runtime = ExtensionRuntime::load_with_services( + "registerExtension({spin(){log.info('entered');while(true){}}});", + "{}", + RuntimeLimits::default(), + services, + ) + .unwrap(); + // A rendezvous reply holds the worker at publication until we receive. + // This makes the ordering assertion independent of thread scheduling. + let (reply, result) = mpsc::sync_channel(0); + assert!( + runtime + .sender + .send(Command::Call { + method: "spin".into(), + arguments: "[]".into(), + operation: Operation { + item_id: "example-item".into(), + deadline: Instant::now() + Duration::from_secs(10), + lease: Some(Arc::new(downloads.acquire("example-item").unwrap())), + resolution: None, + }, + reply, + kind: CallKind::Function, + }) + .is_ok() + ); + let began = Instant::now(); + while logs.count().unwrap() == 0 { + assert!( + began.elapsed() < Duration::from_secs(2), + "VM did not enter spin" + ); + thread::sleep(Duration::from_millis(1)); + } + downloads.cancel("example-item").unwrap(); + let began = Instant::now(); + while !runtime.is_closed() && began.elapsed() < Duration::from_secs(1) { + thread::sleep(Duration::from_millis(1)); + } + let closed_before_reply = runtime.is_closed(); + let response = result.recv_timeout(Duration::from_secs(2)).unwrap(); + runtime.shutdown(); + assert_eq!( + response, + Err(ExtensionError::Cancelled( + CancellationError::DownloadCancelled + )) + ); + assert!( + closed_before_reply, + "interrupted worker published before closing" + ); + } +} diff --git a/rust_backend/crates/extensions/src/session_host.rs b/rust_backend/crates/extensions/src/session_host.rs new file mode 100644 index 00000000..a17389a4 --- /dev/null +++ b/rust_backend/crates/extensions/src/session_host.rs @@ -0,0 +1,62 @@ +use crate::runtime::{Control, ExtensionServices}; +use rquickjs::{Ctx, Function, Object}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; +use std::sync::Arc; +use zeroize::Zeroizing; + +pub(crate) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + host.set("sessionEnabled", services.session.is_some())?; + if let Some(session) = &services.session { + let session = Arc::clone(session); + host.set( + "sessionCall", + Function::new(ctx.clone(), move |method: String, arguments: String| { + let arguments = Zeroizing::new(arguments); + let check = || control.check().map_err(|error| error.to_string()); + let result = (|| { + check()?; + let args: Vec<Value> = + serde_json::from_str(&arguments).map_err(|error| error.to_string())?; + let string = |index| args.get(index).and_then(Value::as_str).unwrap_or(""); + match method.as_str() { + "status" => session.status(), + "clear" => session.clear().map(|()| json!({"success":true})), + "completeGrant" => session + .complete_grant(string(0), check) + .map(|()| json!({"success":true})), + "signedFetch" => { + if args.len() < 2 { + return Err("method and path are required".into()); + } + let headers: BTreeMap<String, String> = args + .get(3) + .cloned() + .map(serde_json::from_value) + .transpose() + .map_err(|error| error.to_string())? + .unwrap_or_default(); + session.signed_fetch(string(0), string(1), string(2), &headers, check) + } + _ => Err("unknown signed session method".into()), + } + })(); + match result { + Ok(value) => value, + Err(error) if method == "status" => { + json!({"authenticated":false,"error":error}) + } + Err(error) if method == "signedFetch" => json!({"ok":false,"error":error}), + Err(error) => json!({"success":false,"error":error}), + } + .to_string() + })?, + )?; + } + Ok(()) +} diff --git a/rust_backend/crates/extensions/src/signed_session.rs b/rust_backend/crates/extensions/src/signed_session.rs new file mode 100644 index 00000000..63bdef3d --- /dev/null +++ b/rust_backend/crates/extensions/src/signed_session.rs @@ -0,0 +1,10 @@ +//! Gateway signed sessions shared by scope, with isolated extension permissions. + +mod coordinator; +mod exchange; +mod fetch; +pub mod protocol; +mod store; + +pub use coordinator::{SignedRegistry, SignedSessionClient}; +pub use store::{RecordStore, RuntimeHints}; diff --git a/rust_backend/crates/extensions/src/signed_session/coordinator.rs b/rust_backend/crates/extensions/src/signed_session/coordinator.rs new file mode 100644 index 00000000..97f827e0 --- /dev/null +++ b/rust_backend/crates/extensions/src/signed_session/coordinator.rs @@ -0,0 +1,319 @@ +use super::{RecordStore, RuntimeHints, protocol}; +use crate::auth::{AuthRegistry, PendingAuthRequest}; +use crate::manifest::SignedSession; +use serde_json::{Value, json}; +use spotiflac_network::{HttpRequest, HttpResponse, NetworkSession}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; +use zeroize::Zeroizing; + +pub(super) type Check<'a> = &'a dyn Fn() -> Result<(), String>; + +#[derive(Default)] +pub(super) struct Flight { + result: Mutex<Option<Result<(), String>>>, + done: Condvar, +} + +impl Flight { + pub fn finish(&self, result: Result<(), String>) { + *self.result.lock().expect("signed session flight lock") = Some(result); + self.done.notify_all(); + } + + pub fn wait(&self, check: Check<'_>) -> Result<(), String> { + let mut result = self.result.lock().expect("signed session flight lock"); + loop { + check()?; + if let Some(result) = &*result { + return result.clone(); + } + result = self + .done + .wait_timeout(result, Duration::from_millis(10)) + .expect("signed session flight wait") + .0; + } + } +} + +#[derive(Default)] +pub(super) struct State { + pub clear_generation: u64, + pub completed_grant: String, + pub blocked_generation: String, + pub challenge: Option<PendingAuthRequest>, + pub pending_ids: BTreeSet<String>, + pub bootstrap: Option<Arc<Flight>>, + pub refresh: Option<Arc<Flight>>, + pub exchange: Option<Arc<Flight>>, +} + +impl State { + pub fn blocked(&self, record: &protocol::Record) -> bool { + let generation = record.generation(); + !generation.is_empty() && self.blocked_generation == generation + } + + pub fn clear_challenge(&mut self, auth: &AuthRegistry) { + for id in &self.pending_ids { + auth.clear_pending(id); + } + self.pending_ids.clear(); + self.challenge = None; + } + + pub fn remember(&mut self, pending: PendingAuthRequest) { + self.pending_ids.insert(pending.extension_id.clone()); + self.challenge = Some(pending); + } +} + +pub struct SignedRegistry { + root: PathBuf, + pub(super) auth: Arc<AuthRegistry>, + hints: Mutex<RuntimeHints>, + scopes: Mutex<BTreeMap<PathBuf, Arc<Mutex<State>>>>, + pub(super) grants: Mutex<BTreeMap<String, Zeroizing<String>>>, + closed: AtomicBool, +} + +impl SignedRegistry { + pub fn new(root: &Path, auth: Arc<AuthRegistry>) -> Arc<Self> { + Arc::new(Self { + root: root.to_owned(), + auth, + hints: Mutex::default(), + scopes: Mutex::default(), + grants: Mutex::default(), + closed: AtomicBool::new(false), + }) + } + + pub fn set_runtime_state(&self, raw: &str) -> Result<(), String> { + self.check()?; + *self.hints.lock().expect("session hints lock") = RuntimeHints::parse(raw); + Ok(()) + } + + pub fn set_grant(&self, id: &str, grant: &str) -> Result<(), String> { + let mut grants = self.grants.lock().expect("session grants lock"); + self.check()?; + if !id.trim().is_empty() && !grant.trim().is_empty() { + grants.insert(id.trim().into(), Zeroizing::new(grant.trim().into())); + } + Ok(()) + } + + pub fn session( + self: &Arc<Self>, + id: &str, + config: SignedSession, + network: Arc<NetworkSession>, + ) -> Result<Arc<SignedSessionClient>, String> { + self.check()?; + let config = protocol::defaults(config); + let store = RecordStore::open(&self.root, &config)?; + let mut scopes = self.scopes.lock().expect("signed session scopes lock"); + self.check()?; + let scope = Arc::clone(scopes.entry(store.path().to_owned()).or_default()); + Ok(Arc::new(SignedSessionClient { + id: id.to_owned(), + registry: Arc::clone(self), + config, + store, + scope, + network, + verification_url: Mutex::new(String::new()), + })) + } + + pub(super) fn check(&self) -> Result<(), String> { + if self.closed.load(Ordering::Acquire) { + Err("extension environment closed".into()) + } else { + Ok(()) + } + } + + pub fn shutdown(&self) { + self.closed.store(true, Ordering::Release); + self.grants.lock().expect("session grants lock").clear(); + for state in self + .scopes + .lock() + .expect("signed session scopes lock") + .values() + { + state + .lock() + .expect("signed session coordinator lock") + .clear_challenge(&self.auth); + } + } + + pub(crate) fn forget_extension(&self, id: &str) { + self.grants.lock().expect("session grants lock").remove(id); + // Session records can be shared by multiple extensions through their + // manifest namespace. Only remove this extension's pending challenge. + for scope in self + .scopes + .lock() + .expect("signed session scopes lock") + .values() + { + let mut state = scope.lock().expect("signed session coordinator lock"); + state.pending_ids.remove(id); + if state + .challenge + .as_ref() + .is_some_and(|pending| pending.extension_id == id) + { + state.challenge = None; + } + } + } +} + +pub struct SignedSessionClient { + pub(super) id: String, + pub(super) registry: Arc<SignedRegistry>, + pub(super) config: SignedSession, + pub(super) store: RecordStore, + pub(super) scope: Arc<Mutex<State>>, + pub(super) network: Arc<NetworkSession>, + verification_url: Mutex<String>, +} + +impl SignedSessionClient { + pub(super) fn check(&self, check: Check<'_>) -> Result<(), String> { + self.registry.check()?; + check() + } + + pub(super) fn load(&self) -> Result<protocol::Record, String> { + self.store + .load(&self.registry.hints.lock().expect("session hints lock")) + } + + pub fn status(&self) -> Result<Value, String> { + self.registry.check()?; + let state = self.scope.lock().expect("signed session coordinator lock"); + let record = self.load()?; + let blocked = state.blocked(&record); + Ok( + json!({"authenticated":record.usable(self.registry.auth.now()) && !blocked, "verification_required":blocked, + "expires_at":record.expires_at,"install_id":record.install_id,"session_id":record.session_id, + "app_version":self.config.app_version,"platform":self.config.platform}), + ) + } + + pub fn clear(&self) -> Result<(), String> { + self.registry.check()?; + let mut state = self.scope.lock().expect("signed session coordinator lock"); + let mut record = self.load()?; + record.clear(); + self.store.save(&record)?; + state.clear_generation = state.clear_generation.wrapping_add(1); + state.completed_grant.clear(); + state.blocked_generation.clear(); + state.clear_challenge(&self.registry.auth); + self.registry.auth.clear_pending(&self.id); + Ok(()) + } + + pub fn take_verification_url(&self) -> String { + std::mem::take( + &mut self + .verification_url + .lock() + .expect("signed session verification URL lock"), + ) + } + + pub(super) fn verification_required(&self, url: String) -> Value { + if !url.is_empty() { + *self + .verification_url + .lock() + .expect("signed session verification URL lock") = url.clone(); + } + json!({"ok":false,"needsVerification":true,"error":"VERIFY_REQUIRED","open_auth_url":url,"auth_url":url}) + } + + pub(super) fn request( + &self, + method: &str, + url: String, + body: String, + mut headers: BTreeMap<String, String>, + check: Check<'_>, + ) -> Result<HttpResponse, String> { + self.check(check)?; + // Header matching is case-insensitive on the wire. Normalize before + // applying extension overrides so a lowercase override cannot coexist. + headers = headers + .into_iter() + .map(|(key, value)| (key.to_ascii_lowercase(), value)) + .collect(); + headers + .entry("accept".into()) + .or_insert_with(|| "application/json".into()); + self.network.request( + HttpRequest { + url, + method: method.into(), + body, + headers, + default_json: false, + user_agent: format!("SpotiFLAC-Mobile/{}", self.config.app_version), + }, + || self.check(check), + ) + } + + pub(super) fn signed_request( + &self, + record: &protocol::Record, + method: &str, + path: &str, + body: &str, + extras: &BTreeMap<String, String>, + check: Check<'_>, + ) -> Result<HttpResponse, String> { + let url = protocol::endpoint(&self.config, path)?; + let mut headers: BTreeMap<_, _> = protocol::signed_headers( + &self.config, + record, + method, + &url, + body.as_bytes(), + self.registry.auth.now(), + &protocol::random_hex(12)?, + )? + .into_iter() + .map(|(key, value)| (key.to_ascii_lowercase(), value)) + .collect(); + if !body.is_empty() { + headers.insert("content-type".into(), "application/json".into()); + } + for (key, value) in extras { + headers.insert(key.to_ascii_lowercase(), value.clone()); + } + self.request(method, url, body.into(), headers, check) + } + + pub(super) fn wait(&self, delay: Duration, check: Check<'_>) -> Result<(), String> { + let end = Instant::now() + delay; + loop { + self.check(check)?; + let Some(remaining) = end.checked_duration_since(Instant::now()) else { + return Ok(()); + }; + std::thread::sleep(remaining.min(Duration::from_millis(10))); + } + } +} diff --git a/rust_backend/crates/extensions/src/signed_session/exchange.rs b/rust_backend/crates/extensions/src/signed_session/exchange.rs new file mode 100644 index 00000000..a9682dc5 --- /dev/null +++ b/rust_backend/crates/extensions/src/signed_session/exchange.rs @@ -0,0 +1,450 @@ +use super::coordinator::{Check, Flight, SignedSessionClient}; +use super::protocol::{self, Record}; +use crate::auth::{PendingAuthRequest, callback_state}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use spotiflac_network::{query, url::UrlParts}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use zeroize::Zeroizing; + +enum Bootstrap { + Session(Record), + Challenge { + url: String, + callback: String, + state: String, + }, +} + +fn response_record(body: &[u8]) -> Result<Record, String> { + let fields = protocol::object_fields( + body, + &[ + "session_id", + "session_secret", + "expires_at", + "challenge_id", + "challenge_url", + "auth_url", + ], + &[], + &[], + )?; + serde_json::from_value(Value::Object(fields)).map_err(|error| error.to_string()) +} + +impl SignedSessionClient { + pub fn preflight(&self, check: impl Fn() -> Result<(), String>) -> Result<bool, String> { + self.check(&check)?; + { + let state = self.scope.lock().expect("signed session coordinator lock"); + let record = self.load()?; + if record.usable(self.registry.auth.now()) && !state.blocked(&record) { + return Ok(false); + } + } + let url = self.bootstrap(&check)?; + if !url.is_empty() { + return Ok(true); + } + let _state = self.scope.lock().expect("signed session coordinator lock"); + if self.load()?.usable(self.registry.auth.now()) { + Ok(false) + } else { + Err( + "signed-session bootstrap did not return a session or verification challenge" + .into(), + ) + } + } + + pub(super) fn bootstrap(&self, check: Check<'_>) -> Result<String, String> { + loop { + self.check(check)?; + let mut state = self.scope.lock().expect("signed session coordinator lock"); + if let Some(challenge) = state.challenge.as_ref().filter(|challenge| { + !challenge.auth_url.trim().is_empty() + && self.registry.auth.now() - challenge.created_at < 180_000_000_000 + }) { + let mut pending = challenge.clone(); + pending.extension_id = self.id.clone(); + self.registry.auth.register_pending(pending.clone())?; + state.pending_ids.insert(self.id.clone()); + return Ok(pending.auth_url); + } + if state.challenge.is_some() { + state.clear_challenge(&self.registry.auth); + } + if let Some(pending) = self + .registry + .auth + .pending(&self.id) + .filter(|pending| !pending.auth_url.trim().is_empty()) + { + let url = pending.auth_url.clone(); + state.remember(pending); + return Ok(url); + } + let record = self + .load() + .map_err(|error| format!("load signed-session bootstrap state: {error}"))?; + if record.usable(self.registry.auth.now()) && !state.blocked(&record) { + return Ok(String::new()); + } + if let Some(flight) = state.bootstrap.clone() { + drop(state); + flight.wait(&|| self.check(check))?; + continue; + } + let flight = Arc::new(Flight::default()); + state.bootstrap = Some(Arc::clone(&flight)); + let generation = state.clear_generation; + drop(state); + let bootstrap = self.perform_bootstrap(&record, check); + let mut state = self.scope.lock().expect("signed session coordinator lock"); + let result = (|| { + let bootstrap = bootstrap?; + self.check(check)?; + if state.clear_generation != generation { + return Err("signed-session bootstrap was superseded by session clear".into()); + } + let mut latest = self.load()?; + if latest.usable(self.registry.auth.now()) + && !latest.same_session(&record) + && !state.blocked(&latest) + { + state.clear_challenge(&self.registry.auth); + return Ok(String::new()); + } + match bootstrap { + Bootstrap::Session(record) => { + latest.session_id.clone_from(&record.session_id); + latest.session_secret.clone_from(&record.session_secret); + latest.expires_at.clone_from(&record.expires_at); + self.store.save(&latest).map_err(|error| { + format!("save bootstrapped signed session: {error}") + })?; + state.blocked_generation.clear(); + state.clear_challenge(&self.registry.auth); + Ok(String::new()) + } + Bootstrap::Challenge { + url, + callback, + state: nonce, + } => { + let pending = PendingAuthRequest { + extension_id: self.id.clone(), + auth_url: url.clone(), + callback_url: callback, + state: nonce, + created_at: self.registry.auth.now(), + }; + self.registry.auth.register_pending(pending.clone())?; + state.remember(pending); + Ok(url) + } + } + })(); + state.bootstrap = None; + flight.finish(result.as_ref().map(|_| ()).map_err(Clone::clone)); + return result; + } + } + + fn perform_bootstrap(&self, record: &Record, check: Check<'_>) -> Result<Bootstrap, String> { + let url = protocol::endpoint(&self.config, &self.config.endpoints.bootstrap) + .map_err(|error| format!("build signed-session bootstrap URL: {error}"))?; + let url = protocol::with_query( + &url, + &[ + ("app_version", &self.config.app_version), + ("install_id", &record.install_id), + ], + )?; + let mut response = self.request("GET", url.clone(), String::new(), BTreeMap::new(), check); + if response.is_err() { + self.check(check)?; + self.network.reset_connections(); + response = self.request("GET", url.clone(), String::new(), BTreeMap::new(), check); + } + let response = response.map_err(|error| { + format!( + "signed-session bootstrap network request to {} failed: {error}", + UrlParts::parse(&url) + .map(|url| url.authority()) + .unwrap_or_default() + ) + })?; + if !(200..300).contains(&response.status) { + let mut error = format!( + "signed-session bootstrap {}returned HTTP {}", + if response.status >= 500 { + "network request " + } else { + "" + }, + response.status + ); + let retry = protocol::response_retry_after(&response, self.registry.auth.now()); + if retry > 0 { + error += &format!("; retry-after seconds: {retry}"); + } + return Err(error); + } + let body = Zeroizing::new(response.body); + let record = response_record(&body) + .map_err(|error| format!("decode signed-session bootstrap response: {error}"))?; + let boot = protocol::object_fields( + &body, + &["auth_url", "challenge_url", "challenge_id"], + &[], + &[], + ) + .map_err(|error| format!("decode signed-session bootstrap response: {error}"))?; + if !record.session_id.is_empty() + && !record.session_secret.is_empty() + && !record.expires_at.is_empty() + { + return Ok(Bootstrap::Session(record)); + } + let text = |key| boot.get(key).and_then(Value::as_str).unwrap_or(""); + let mut auth_url = text("auth_url").to_owned(); + if auth_url.is_empty() { + auth_url = text("challenge_url").to_owned(); + } + let mut state = callback_state() + .map_err(|error| format!("prepare signed-session callback state: {error}"))?; + if let Some(url) = UrlParts::parse(&auth_url) { + let values = query::parse(&url.raw_query); + let server_state = values + .get(b"state".as_slice()) + .and_then(|values| values.first()) + .map(|bytes| crate::host::decode_go_utf8(bytes)) + .unwrap_or_default(); + if !server_state.trim().is_empty() { + state = server_state.trim().to_owned(); + } else if !auth_url.is_empty() { + auth_url = protocol::with_query(&auth_url, &[("state", &state)])?; + } + } + let callback = protocol::with_query(&self.config.callback_url, &[("state", &state)]) + .map_err(|error| format!("prepare signed-session callback: {error}"))?; + if auth_url.is_empty() && !text("challenge_id").is_empty() { + auth_url = protocol::challenge_url(&self.config, text("challenge_id"), &state) + .unwrap_or_default(); + } + if auth_url.is_empty() { + return Err( + "signed-session bootstrap did not return a session or verification challenge" + .into(), + ); + } + Ok(Bootstrap::Challenge { + url: auth_url, + callback, + state, + }) + } + + pub fn complete_grant( + &self, + grant: &str, + check: impl Fn() -> Result<(), String>, + ) -> Result<(), String> { + self.registry.set_grant(&self.id, grant)?; + let grant = self + .registry + .grants + .lock() + .expect("session grants lock") + .get(&self.id) + .cloned() + .filter(|grant| !grant.is_empty()) + .ok_or("no pending grant")?; + let end = Instant::now() + Duration::from_secs(30); + let check = || { + self.check(&check)?; + if Instant::now() >= end { + Err("context deadline exceeded".into()) + } else { + Ok(()) + } + }; + self.exchange(&grant, &check)?; + let mut grants = self.registry.grants.lock().expect("session grants lock"); + // A newer callback can arrive during HTTP. Do not discard its grant. + if grants + .get(&self.id) + .is_some_and(|pending| pending.as_str() == grant.as_str()) + { + grants.remove(&self.id); + } + self.registry.auth.clear_pending(&self.id); + Ok(()) + } + + fn exchange(&self, grant: &str, check: Check<'_>) -> Result<(), String> { + let generation = self + .scope + .lock() + .expect("signed session coordinator lock") + .clear_generation; + let flight = loop { + self.check(check)?; + let mut state = self.scope.lock().expect("signed session coordinator lock"); + if let Some(flight) = state.exchange.clone() { + drop(state); + // Exchange slots serialize different grants. The preceding + // grant's failure is not the outcome of this new operation. + let _ = flight.wait(&|| self.check(check)); + self.check(check)?; + } else { + let flight = Arc::new(Flight::default()); + state.exchange = Some(Arc::clone(&flight)); + break flight; + } + }; + let result = self.exchange_owned(grant, generation, check); + self.scope + .lock() + .expect("signed session coordinator lock") + .exchange = None; + flight.finish(result.clone()); + result + } + + fn exchange_owned(&self, grant: &str, generation: u64, check: Check<'_>) -> Result<(), String> { + let grant_hash = format!("{:x}", Sha256::digest(grant)); + let record = { + let mut state = self.scope.lock().expect("signed session coordinator lock"); + if state.clear_generation != generation { + return Err("signed-session exchange was superseded by session clear".into()); + } + let record = self.load()?; + if state.completed_grant == grant_hash && record.usable(self.registry.auth.now()) { + state.blocked_generation.clear(); + state.clear_challenge(&self.registry.auth); + return Ok(()); + } + record + }; + let url = protocol::endpoint(&self.config, &self.config.endpoints.exchange)?; + let body = Zeroizing::new(json!({"grant":grant,"install_id":record.install_id,"app_version":self.config.app_version,"platform":self.config.platform}).to_string()); + let mut exchanged = None; + for attempt in 0..3 { + let response = self.request( + "POST", + url.clone(), + body.to_string(), + BTreeMap::from([("content-type".into(), "application/json".into())]), + check, + )?; + let retry = protocol::response_retry_after(&response, self.registry.auth.now()); + if response.status == 429 && attempt < 2 { + self.wait(Duration::from_secs(retry.clamp(1, 300) as u64), check)?; + continue; + } + if !(200..300).contains(&response.status) { + let mut error = format!("session exchange failed: HTTP {}", response.status); + if retry > 0 { + error += &format!("; retry-after seconds: {retry}"); + } + return Err(error); + } + exchanged = Some( + response_record(&Zeroizing::new(response.body)) + .map_err(|error| format!("invalid session exchange response: {error}"))?, + ); + break; + } + let exchanged = exchanged.ok_or("session exchange did not complete")?; + if exchanged.session_id.is_empty() + || exchanged.session_secret.is_empty() + || exchanged.expires_at.is_empty() + { + return Err("session exchange response missing session fields".into()); + } + self.check(check)?; + let mut state = self.scope.lock().expect("signed session coordinator lock"); + if state.clear_generation != generation { + return Err("signed-session exchange was superseded by session clear".into()); + } + let mut latest = self.load()?; + if !(latest.usable(self.registry.auth.now()) + && (state.completed_grant == grant_hash || !latest.same_session(&record))) + { + latest.session_id.clone_from(&exchanged.session_id); + latest.session_secret.clone_from(&exchanged.session_secret); + latest.expires_at.clone_from(&exchanged.expires_at); + self.store.save(&latest)?; + state.completed_grant = grant_hash; + } + state.blocked_generation.clear(); + state.clear_challenge(&self.registry.auth); + Ok(()) + } + + pub(super) fn refresh(&self, check: Check<'_>) -> Result<(), String> { + loop { + self.check(check)?; + let mut state = self.scope.lock().expect("signed session coordinator lock"); + let record = self.load()?; + if !record.refresh_due(&self.config, self.registry.auth.now()) { + return Ok(()); + } + if let Some(flight) = state.refresh.clone() { + drop(state); + flight.wait(&|| self.check(check))?; + continue; + } + let flight = Arc::new(Flight::default()); + state.refresh = Some(Arc::clone(&flight)); + let generation = state.clear_generation; + drop(state); + let response = self.signed_request( + &record, + "POST", + &self.config.endpoints.refresh, + &json!({"install_id":record.install_id}).to_string(), + &BTreeMap::new(), + check, + ); + let mut state = self.scope.lock().expect("signed session coordinator lock"); + let result = (|| { + let response = response?; + if !(200..300).contains(&response.status) { + return Err(format!("session refresh failed: HTTP {}", response.status)); + } + let refreshed = response_record(&Zeroizing::new(response.body))?; + self.check(check)?; + if state.clear_generation != generation { + return Err("signed-session refresh was superseded by session clear".into()); + } + let mut latest = self.load()?; + if latest.same_session(&record) { + let mut changed = false; + for (field, value) in [ + (&mut latest.session_id, &refreshed.session_id), + (&mut latest.session_secret, &refreshed.session_secret), + (&mut latest.expires_at, &refreshed.expires_at), + ] { + if !value.is_empty() && field != value { + field.clone_from(value); + changed = true; + } + } + if changed { + self.store.save(&latest)?; + } + } + Ok(()) + })(); + state.refresh = None; + flight.finish(result.clone()); + return result; + } + } +} diff --git a/rust_backend/crates/extensions/src/signed_session/fetch.rs b/rust_backend/crates/extensions/src/signed_session/fetch.rs new file mode 100644 index 00000000..cf7d5774 --- /dev/null +++ b/rust_backend/crates/extensions/src/signed_session/fetch.rs @@ -0,0 +1,141 @@ +use super::coordinator::SignedSessionClient; +use super::protocol::{self, ErrorContract}; +use serde_json::Value; +use std::collections::BTreeMap; + +impl SignedSessionClient { + pub fn signed_fetch( + &self, + method: &str, + path: &str, + body: &str, + headers: &BTreeMap<String, String>, + check: impl Fn() -> Result<(), String>, + ) -> Result<Value, String> { + self.check(&check)?; + let method = method.trim().to_uppercase(); + let mut record = { + let state = self.scope.lock().expect("signed session coordinator lock"); + let mut record = self.load()?; + let error = if record.session_id.is_empty() || record.session_secret.is_empty() { + Some("signed session is not authenticated") + } else if protocol::parse_time(&record.expires_at) + .is_some_and(|expires| self.registry.auth.now() > expires) + { + record.clear(); + let _ = self.store.save(&record); + Some("signed session expired") + } else { + None + }; + if let Some(error) = error { + drop(state); + let url = self.bootstrap(&check)?; + return if url.is_empty() { + Err(error.into()) + } else { + Ok(self.verification_required(url)) + }; + } + if state.blocked(&record) { + drop(state); + let url = self.bootstrap(&check)?; + if !url.is_empty() { + return Ok(self.verification_required(url)); + } + let state = self.scope.lock().expect("signed session coordinator lock"); + record = self.load()?; + if !record.usable(self.registry.auth.now()) || state.blocked(&record) { + return Err( + "verification_required: signed-session generation is blocked".into(), + ); + } + } + record + }; + if record.refresh_due(&self.config, self.registry.auth.now()) { + let _ = self.refresh(&check); + self.check(&check)?; + let state = self.scope.lock().expect("signed session coordinator lock"); + let latest = self.load()?; + if !latest.usable(self.registry.auth.now()) || state.blocked(&latest) { + drop(state); + let url = self.bootstrap(&check)?; + return if url.is_empty() { + Err("signed session is not authenticated".into()) + } else { + Ok(self.verification_required(url)) + }; + } + record = latest; + } + let mut session_retries = 0; + let mut provider_retries = 0; + let mut request_auth_retry = false; + loop { + let response = self.signed_request(&record, &method, path, body, headers, &check)?; + let contract = ErrorContract::parse(&response.body); + if contract.provider_retry(response.status) { + if provider_retries >= 2 { + return Ok(protocol::response_value(response, self.registry.auth.now())); + } + provider_retries += 1; + self.wait( + protocol::provider_delay(&response, &contract, self.registry.auth.now()), + &check, + )?; + continue; + } + if contract.request_auth_invalid(response.status) { + let _state = self.scope.lock().expect("signed session coordinator lock"); + let latest = self.load()?; + if !request_auth_retry + && latest.usable(self.registry.auth.now()) + && !latest.same_session(&record) + { + request_auth_retry = true; + record = latest; + continue; + } + return Ok(protocol::response_value(response, self.registry.auth.now())); + } + let action = contract.gateway_action(response.status); + if action.is_empty() { + return Ok(protocol::response_value(response, self.registry.auth.now())); + } + let mut state = self.scope.lock().expect("signed session coordinator lock"); + let mut latest = self.load()?; + if latest.usable(self.registry.auth.now()) && !latest.same_session(&record) { + if session_retries >= 1 { + return Err("signed-session retry limit reached".into()); + } + session_retries += 1; + record = latest; + continue; + } + if action == "bootstrap_session" && latest.same_session(&record) { + state.blocked_generation.clear(); + latest.clear(); + self.store.save(&latest)?; + } else if action == "verify" && latest.same_session(&record) { + state.blocked_generation = record.generation(); + } + drop(state); + let url = self.bootstrap(&check)?; + if !url.is_empty() { + return Ok(self.verification_required(url)); + } + let _state = self.scope.lock().expect("signed session coordinator lock"); + let bootstrapped = self.load()?; + if bootstrapped.usable(self.registry.auth.now()) + && !bootstrapped.same_session(&record) + && session_retries < 1 + { + session_retries += 1; + record = bootstrapped; + continue; + } + return Ok(protocol::response_value(response, self.registry.auth.now())); + } + } +} diff --git a/rust_backend/crates/extensions/src/signed_session/protocol.rs b/rust_backend/crates/extensions/src/signed_session/protocol.rs new file mode 100644 index 00000000..4e833086 --- /dev/null +++ b/rust_backend/crates/extensions/src/signed_session/protocol.rs @@ -0,0 +1,589 @@ +use crate::manifest::SignedSession; +use aes_gcm::aead::{OsRng, rand_core::RngCore}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use hmac::{Hmac, Mac}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use spotiflac_network::{HttpResponse, query, url::UrlParts}; +use std::collections::BTreeMap; +use std::time::{Duration, UNIX_EPOCH}; +use time::{Date, Month, OffsetDateTime, PrimitiveDateTime, Time}; +use zeroize::{Zeroize, Zeroizing}; + +pub fn defaults(mut config: SignedSession) -> SignedSession { + for (value, fallback) in [ + (&mut config.app_version, "ext-1.0"), + (&mut config.platform, "extension"), + (&mut config.callback_url, "spotiflac://session-grant"), + (&mut config.scheme_label, "SPOTIFLAC-HMAC-V1"), + (&mut config.header_prefix, "X-Sig-"), + (&mut config.endpoints.bootstrap, "/bootstrap"), + (&mut config.endpoints.challenge, "/challenge"), + (&mut config.endpoints.exchange, "/session/exchange"), + ] { + if value.is_empty() { + *value = fallback.into(); + } + } + if config.time_window_seconds <= 0 { + config.time_window_seconds = 300; + } + config +} + +pub fn namespace(value: &str) -> String { + value + .trim() + .to_lowercase() + .chars() + .filter(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ".-_".contains(*ch)) + .collect::<String>() + .trim_matches(['.', '-', '_']) + .to_owned() +} + +pub fn filename(config: &SignedSession) -> Result<String, String> { + let namespace = namespace(&config.namespace); + if namespace.is_empty() { + return Err("signed session namespace is empty".into()); + } + let scope = [ + namespace.as_str(), + &config.base_url.trim().to_lowercase(), + &config.app_version.trim().to_lowercase(), + &config.platform.trim().to_lowercase(), + ] + .join("\n"); + Ok(format!( + "{namespace}-{}.json", + &hex(&Sha256::digest(scope))[..16] + )) +} + +pub fn endpoint(config: &SignedSession, endpoint: &str) -> Result<String, String> { + let base = UrlParts::parse(&(config.base_url.trim_end_matches('/').to_owned() + "/")) + .filter(|base| base.scheme == "https" && !base.hostname.is_empty()) + .ok_or("invalid signed session baseUrl")?; + let endpoint = endpoint.trim(); + if endpoint.is_empty() { + return Err("signed session endpoint is empty".into()); + } + if endpoint.starts_with("https://") { + return Ok(endpoint.to_owned()); + } + base.resolve(endpoint.trim_start_matches('/')) + .map(|url| url.display_url()) + .ok_or_else(|| "invalid signed session endpoint".into()) +} + +pub fn with_query(input: &str, values: &[(&str, &str)]) -> Result<String, String> { + let mut url = UrlParts::parse(input).ok_or("invalid signed session URL")?; + let mut query = query::parse(&url.raw_query); + for (key, value) in values { + query::set(&mut query, key, value); + } + url.raw_query = query::encode(&query); + Ok(url.display_url()) +} + +pub fn challenge_url( + config: &SignedSession, + challenge: &str, + state: &str, +) -> Result<String, String> { + let callback = with_query( + &config.callback_url, + &[("cb_version", "v2grant"), ("state", state)], + )?; + with_query( + &endpoint(config, &config.endpoints.challenge)?, + &[("id", challenge), ("cb", &callback)], + ) +} + +pub fn parse_time(value: &str) -> Option<i128> { + // Match Go's RFC3339 fallback parser: uppercase T/Z, one- or two-digit + // hours, comma fractions, truncated nanoseconds, and normalized +24:60. + fn number(value: &str, minimum: usize, maximum: usize) -> Option<u32> { + ((minimum..=maximum).contains(&value.len()) + && value.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| value.parse().ok()) + .flatten() + } + let (date, clock) = value.trim().split_once('T')?; + let mut date = date.split('-'); + let year = number(date.next()?, 4, 4)? as i32; + let month = Month::try_from(number(date.next()?, 2, 2)? as u8).ok()?; + let day = number(date.next()?, 2, 2)? as u8; + if date.next().is_some() { + return None; + } + let zone_start = clock.find(['Z', '+', '-'])?; + let (clock, zone) = clock.split_at(zone_start); + let offset = if zone == "Z" { + 0 + } else { + let (sign, zone) = zone.split_at(1); + let (hours, minutes) = zone.split_once(':')?; + let hours = number(hours, 2, 2)?; + let minutes = number(minutes, 2, 2)?; + if hours > 24 || minutes > 60 { + return None; + } + i128::from(hours * 3600 + minutes * 60) + * if sign == "-" { + -1 + } else if sign == "+" { + 1 + } else { + return None; + } + }; + let mut clock = clock.split(':'); + let hours = number(clock.next()?, 1, 2)? as u8; + let minutes = number(clock.next()?, 2, 2)? as u8; + let seconds = clock.next()?; + if clock.next().is_some() { + return None; + } + let (seconds, fraction) = seconds.split_once(['.', ',']).unwrap_or((seconds, "")); + let seconds = number(seconds, 2, 2)? as u8; + let nanos = if fraction.is_empty() { + if value.contains(['.', ',']) { + return None; + } + 0 + } else { + if !fraction.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let digits = &fraction[..fraction.len().min(9)]; + digits.parse::<u32>().ok()? * 10_u32.pow(9 - digits.len() as u32) + }; + let date = Date::from_calendar_date(year, month, day).ok()?; + let time = Time::from_hms_nano(hours, minutes, seconds, nanos).ok()?; + Some( + PrimitiveDateTime::new(date, time) + .assume_utc() + .unix_timestamp_nanos() + - offset * 1_000_000_000, + ) +} + +fn string_or_null<'de, D: serde::Deserializer<'de>>(value: D) -> Result<String, D::Error> { + Ok(Option::<String>::deserialize(value)?.unwrap_or_default()) +} + +// Go's JSON struct decoder accepts ASCII case-insensitive field names and +// treats null scalars as their zero values. All gateway decisions use this path. +pub(super) fn object_fields( + bytes: &[u8], + strings: &[&str], + booleans: &[&str], + integers: &[&str], +) -> Result<serde_json::Map<String, Value>, String> { + struct Fields<'a>([&'a [&'a str]; 3]); + impl<'de> serde::de::Visitor<'de> for Fields<'_> { + type Value = serde_json::Map<String, Value>; + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a signed session object") + } + fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> { + Ok(serde_json::Map::new()) + } + fn visit_map<A: serde::de::MapAccess<'de>>( + self, + mut map: A, + ) -> Result<Self::Value, A::Error> { + let mut result = serde_json::Map::new(); + while let Some(key) = map.next_key::<String>()? { + let key = key.to_ascii_lowercase(); + let Some(kind) = self + .0 + .iter() + .position(|fields| fields.contains(&key.as_str())) + else { + map.next_value::<serde::de::IgnoredAny>()?; + continue; + }; + let value = map.next_value::<Value>()?; + if value.is_null() { + continue; + } + let valid = match kind { + 0 => value.is_string(), + 1 => value.is_boolean(), + _ => value + .as_i64() + .is_some_and(|value| isize::try_from(value).is_ok()), + }; + if !valid { + return Err(serde::de::Error::custom(format!( + "invalid signed session field: {key}" + ))); + } + // Preserve encounter order across duplicate/case-folded keys; + // null leaves the preceding scalar untouched, as in Go. + result.insert(key, value); + } + Ok(result) + } + } + let mut decoder = serde_json::Deserializer::from_slice(bytes); + let result = + serde::Deserializer::deserialize_any(&mut decoder, Fields([strings, booleans, integers])) + .map_err(|error| error.to_string())?; + decoder.end().map_err(|error| error.to_string())?; + Ok(result) +} + +#[derive(Clone, Default, Deserialize, Serialize)] +pub struct Record { + #[serde(default, deserialize_with = "string_or_null")] + pub install_id: String, + #[serde( + default, + skip_serializing_if = "String::is_empty", + deserialize_with = "string_or_null" + )] + pub session_id: String, + #[serde( + default, + skip_serializing_if = "String::is_empty", + deserialize_with = "string_or_null" + )] + pub session_secret: String, + #[serde( + default, + skip_serializing_if = "String::is_empty", + deserialize_with = "string_or_null" + )] + pub expires_at: String, + #[serde( + default, + skip_serializing_if = "String::is_empty", + deserialize_with = "string_or_null" + )] + pub namespace: String, + #[serde( + default, + skip_serializing_if = "String::is_empty", + deserialize_with = "string_or_null" + )] + pub base_url: String, + #[serde( + default, + skip_serializing_if = "String::is_empty", + deserialize_with = "string_or_null" + )] + pub app_version: String, + #[serde( + default, + skip_serializing_if = "String::is_empty", + deserialize_with = "string_or_null" + )] + pub platform: String, +} + +impl Drop for Record { + fn drop(&mut self) { + self.session_secret.zeroize(); + } +} + +impl Record { + pub fn decode(bytes: &[u8]) -> Result<Self, String> { + serde_json::from_value(Value::Object(object_fields( + bytes, + &[ + "install_id", + "session_id", + "session_secret", + "expires_at", + "namespace", + "base_url", + "app_version", + "platform", + ], + &[], + &[], + )?)) + .map_err(|error| error.to_string()) + } + pub fn usable(&self, now: i128) -> bool { + !self.session_id.trim().is_empty() + && !self.session_secret.trim().is_empty() + && parse_time(&self.expires_at).is_none_or(|expires| now < expires) + } + + pub fn same_session(&self, other: &Self) -> bool { + !self.session_id.is_empty() + && self.session_id == other.session_id + && self.session_secret == other.session_secret + } + + pub fn generation(&self) -> String { + if self.session_id.is_empty() || self.session_secret.is_empty() { + return String::new(); + } + hex(&Sha256::digest( + Zeroizing::new(format!("{}\n{}", self.session_id, self.session_secret)).as_bytes(), + )) + } + + pub fn clear(&mut self) { + self.session_id.clear(); + self.session_secret.zeroize(); + self.expires_at.clear(); + } + + pub fn normalize_scope(&mut self, config: &SignedSession) -> bool { + let namespace = namespace(&config.namespace); + if self.namespace == namespace + && self.base_url == config.base_url.trim() + && self.app_version == config.app_version.trim() + && self.platform == config.platform.trim() + { + return false; + } + if !self.namespace.is_empty() + || !self.base_url.is_empty() + || !self.app_version.is_empty() + || !self.platform.is_empty() + { + self.clear(); + } + self.namespace = namespace; + self.base_url = config.base_url.trim().to_owned(); + self.app_version = config.app_version.trim().to_owned(); + self.platform = config.platform.trim().to_owned(); + true + } + + pub fn refresh_due(&self, config: &SignedSession, now: i128) -> bool { + !config.endpoints.refresh.is_empty() + && self.usable(now) + && parse_time(&self.expires_at) + .is_some_and(|expires| now < expires && expires - now <= 3_600_000_000_000) + } +} + +pub fn random_hex(count: usize) -> Result<String, String> { + let mut bytes = Zeroizing::new(vec![0; count]); + OsRng + .try_fill_bytes(&mut bytes) + .map_err(|error| error.to_string())?; + Ok(hex(&bytes)) +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(output, "{byte:02x}"); + } + output +} + +fn hmac(key: &[u8], message: &[u8]) -> String { + let mut mac = Hmac::<Sha256>::new_from_slice(key).expect("HMAC accepts any key length"); + mac.update(message); + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) +} + +pub fn signed_headers( + config: &SignedSession, + record: &Record, + method: &str, + url: &str, + body: &[u8], + now: i128, + nonce: &str, +) -> Result<BTreeMap<String, String>, String> { + let url = UrlParts::parse(url).ok_or("invalid signed session URL")?; + let time = OffsetDateTime::from_unix_timestamp_nanos(now).map_err(|error| error.to_string())?; + let timestamp = format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z", + time.year(), + u8::from(time.month()), + time.day(), + time.hour(), + time.minute(), + time.second(), + time.millisecond() + ); + let hash = hex(&Sha256::digest(body)); + if config.time_window_seconds <= 0 { + return Err("invalid signed session time window".into()); + } + let window = time.unix_timestamp() / config.time_window_seconds as i64; + let rolling = Zeroizing::new(hmac( + record.session_secret.as_bytes(), + format!("{window}:{}", record.session_id).as_bytes(), + )); + let signing = [ + &config.scheme_label, + method, + &url.escaped_path(), + "", + &hash, + ×tamp, + nonce, + &record.session_id, + &config.app_version, + &config.platform, + ] + .join("\n"); + let signature = hmac(rolling.as_bytes(), signing.as_bytes()); + Ok([ + ("Session", record.session_id.clone()), + ("Timestamp", timestamp), + ("Nonce", nonce.to_owned()), + ("Body-SHA256", hash), + ("Signature", signature), + ("App-Version", config.app_version.clone()), + ("Platform", config.platform.clone()), + ] + .into_iter() + .map(|(name, value)| (format!("{}{name}", config.header_prefix), value)) + .collect()) +} + +#[derive(Default, Deserialize, Serialize)] +#[serde(default)] +pub struct ErrorContract { + #[serde(deserialize_with = "string_or_null")] + pub error: String, + #[serde(deserialize_with = "string_or_null")] + pub code: String, + #[serde(deserialize_with = "string_or_null")] + pub origin: String, + #[serde(deserialize_with = "string_or_null")] + pub action: String, + pub retryable: bool, + #[serde(deserialize_with = "string_or_null")] + pub retry_mode: String, + pub retry_after_seconds: isize, +} + +impl ErrorContract { + pub fn parse(body: &[u8]) -> Self { + let mut value: Self = object_fields( + body, + &["error", "code", "origin", "action", "retry_mode"], + &["retryable"], + &["retry_after_seconds"], + ) + .ok() + .and_then(|fields| serde_json::from_value(Value::Object(fields)).ok()) + .unwrap_or_default(); + value.error = value.error.trim().to_owned(); + value.code = value.code.trim().to_uppercase(); + value.origin = value.origin.trim().to_lowercase(); + value.action = value.action.trim().to_lowercase(); + value.retry_mode = value.retry_mode.trim().to_lowercase(); + value.retry_after_seconds = value.retry_after_seconds.max(0); + value + } + + pub fn gateway_action(&self, status: u16) -> &str { + match ( + status, + self.origin.as_str(), + self.code.as_str(), + self.action.as_str(), + ) { + (401, "gateway", "SESSION_INVALID", "bootstrap_session") => "bootstrap_session", + (428, "gateway", "VERIFY_REQUIRED", "verify") => "verify", + _ => "", + } + } + + pub fn provider_retry(&self, status: u16) -> bool { + status == 503 + && self.origin == "provider" + && self.code == "PROVIDER_UNAVAILABLE" + && self.retryable + && self.retry_mode == "same_operation" + } + + pub fn request_auth_invalid(&self, status: u16) -> bool { + status == 403 + && self.origin == "gateway" + && self.code == "REQUEST_AUTH_INVALID" + && self.action.is_empty() + } +} + +pub fn retry_after(value: &str, now: i128) -> i64 { + let value = value.trim(); + if let Ok(seconds) = value.parse::<isize>() { + return seconds.max(0) as i64; + } + httpdate::parse_http_date(value) + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |time| { + ((time.as_nanos() as i128 - now) / 1_000_000_000).max(0) as i64 + }) +} + +pub fn response_retry_after(response: &HttpResponse, now: i128) -> i64 { + response + .headers + .get("Retry-After") + .and_then(|values| values.first()) + .map_or(0, |value| retry_after(value, now)) +} + +pub fn provider_delay(response: &HttpResponse, contract: &ErrorContract, now: i128) -> Duration { + // Go's retry scheduler preserves the fractional part of an HTTP date and + // parses numeric headers without trimming; the response metadata uses a + // separate, whole-second parser above. + if let Some(value) = response + .headers + .get("Retry-After") + .and_then(|values| values.first()) + { + let nanos = if let Ok(seconds) = value.parse::<isize>() { + // time.Duration is an int64 in Go, including multiplication overflow. + i128::from((seconds as i64).wrapping_mul(1_000_000_000)) + } else { + httpdate::parse_http_date(value) + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |time| time.as_nanos() as i128 - now) + }; + if nanos > 0 { + return Duration::from_nanos(nanos.min(120_000_000_000) as u64); + } + } + Duration::from_secs(if contract.retry_after_seconds > 0 { + contract.retry_after_seconds.min(120) as u64 + } else { + 1 + }) +} + +pub fn response_value(response: HttpResponse, now: i128) -> Value { + let contract = ErrorContract::parse(&response.body); + let retry = response_retry_after(&response, now); + let mut result = json!({"statusCode":response.status,"status":response.status,"ok":(200..300).contains(&response.status), + "url":response.url,"body":crate::host::decode_go_utf8(&response.body),"headers":response.headers.into_iter().map(|(key, values)| { + let value = if values.len() == 1 { json!(values[0]) } else { json!(values) }; (key,value) + }).collect::<BTreeMap<_,_>>(),"retryAfterSeconds":if retry>0 {retry} else {contract.retry_after_seconds as i64}}); + if !contract.code.is_empty() || !contract.origin.is_empty() || !contract.action.is_empty() { + for (key, value) in [ + ("error", json!(contract.error)), + ("code", json!(contract.code)), + ("origin", json!(contract.origin)), + ("action", json!(contract.action)), + ("retryable", json!(contract.retryable)), + ("retryMode", json!(contract.retry_mode)), + ] { + result[key] = value; + } + } + result +} diff --git a/rust_backend/crates/extensions/src/signed_session/store.rs b/rust_backend/crates/extensions/src/signed_session/store.rs new file mode 100644 index 00000000..e6a99a89 --- /dev/null +++ b/rust_backend/crates/extensions/src/signed_session/store.rs @@ -0,0 +1,168 @@ +use super::protocol::{self, Record}; +use crate::manifest::SignedSession; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use zeroize::Zeroizing; + +#[derive(Default)] +pub struct RuntimeHints { + default: String, + values: BTreeMap<String, String>, +} + +impl<'de> Deserialize<'de> for RuntimeHints { + fn deserialize<D: serde::Deserializer<'de>>(decoder: D) -> Result<Self, D::Error> { + struct Fields; + impl<'de> serde::de::Visitor<'de> for Fields { + type Value = RuntimeHints; + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a runtime state object") + } + fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> { + Ok(RuntimeHints::default()) + } + fn visit_map<A: serde::de::MapAccess<'de>>( + self, + mut map: A, + ) -> Result<Self::Value, A::Error> { + let mut result = RuntimeHints::default(); + while let Some(key) = map.next_key::<String>()? { + if key.eq_ignore_ascii_case("d") { + if let Some(value) = map.next_value::<Option<String>>()? { + result.default = value; + } + } else if key.eq_ignore_ascii_case("s") { + match map.next_value::<Option<BTreeMap<String, Option<String>>>>()? { + Some(values) => result.values.extend( + values + .into_iter() + .map(|(key, value)| (key, value.unwrap_or_default())), + ), + None => result.values.clear(), + } + } else { + map.next_value::<serde::de::IgnoredAny>()?; + } + } + Ok(result) + } + } + decoder.deserialize_any(Fields) + } +} + +impl RuntimeHints { + pub fn parse(raw: &str) -> Self { + let mut hints: Self = serde_json::from_str(raw).unwrap_or_default(); + hints.default = normalize_hint(&hints.default); + hints.values = hints + .values + .into_iter() + .filter_map(|(path, value)| { + let key = path + .trim() + .trim_end_matches('/') + .rsplit('/') + .next() + .unwrap_or("") + .to_owned(); + let value = normalize_hint(&value); + (!key.is_empty() && key != "." && !value.is_empty()).then_some((key, value)) + }) + .collect(); + hints + } + + fn get(&self, path: &Path) -> &str { + path.file_name() + .and_then(|name| name.to_str()) + .and_then(|name| self.values.get(name)) + .map_or(&self.default, String::as_str) + } +} + +fn normalize_hint(value: &str) -> String { + let value = value.trim().to_lowercase(); + if value.len() == 32 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + value + } else { + String::new() + } +} + +pub struct RecordStore { + path: PathBuf, + config: SignedSession, +} + +impl RecordStore { + pub fn open(root: &Path, config: &SignedSession) -> Result<Self, String> { + let directory = root.join("signed_sessions"); + if directory + .symlink_metadata() + .is_ok_and(|metadata| !metadata.is_dir()) + { + return Err("invalid signed session directory".into()); + } + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder + .create(&directory) + .map_err(|error| error.to_string())?; + let directory = fs::canonicalize(directory).map_err(|error| error.to_string())?; + Ok(Self { + path: directory.join(protocol::filename(config)?), + config: config.clone(), + }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + /// Call while holding this scope's coordinator lock. Go stores this record + /// as private JSON; preserve that format for the eventual ownership cutover. + pub fn load(&self, hints: &RuntimeHints) -> Result<Record, String> { + let mut record = match fs::symlink_metadata(&self.path) { + Ok(metadata) => { + if !metadata.is_file() || metadata.len() > 1024 * 1024 { + return Err("invalid signed session file".into()); + } + let bytes = + Zeroizing::new(fs::read(&self.path).map_err(|error| error.to_string())?); + Record::decode(&bytes) + .map_err(|error| format!("invalid signed session record: {error}"))? + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Record::default(), + Err(error) => return Err(error.to_string()), + }; + let mut changed = false; + if record.install_id.trim().is_empty() { + record.install_id = hints.get(&self.path).to_owned(); + if record.install_id.is_empty() { + record.install_id = protocol::random_hex(16)?; + } + changed = true; + } + if record.normalize_scope(&self.config) { + changed = true; + } + if changed { + self.save(&record)?; + } + Ok(record) + } + + pub fn save(&self, record: &Record) -> Result<(), String> { + let bytes = + Zeroizing::new(serde_json::to_vec_pretty(record).map_err(|error| error.to_string())?); + crate::storage::atomic_write(&self.path, &bytes).map_err(|error| error.to_string()) + } +} diff --git a/rust_backend/crates/extensions/src/storage.rs b/rust_backend/crates/extensions/src/storage.rs new file mode 100644 index 00000000..bf576941 --- /dev/null +++ b/rust_backend/crates/extensions/src/storage.rs @@ -0,0 +1,523 @@ +//! Go-compatible extension persistence. Each file is serialized across runtimes. + +use aes_gcm::aead::{Aead, KeyInit, OsRng, rand_core::RngCore}; +use aes_gcm::{Aes256Gcm, Nonce}; +use base64::Engine; +use base64::alphabet; +use base64::engine::general_purpose::{GeneralPurpose, GeneralPurposeConfig}; +use hmac::{Hmac, Mac}; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock, RwLock, RwLockReadGuard, Weak}; +use std::time::SystemTime; +use zeroize::Zeroizing; + +#[derive(Debug, thiserror::Error)] +pub enum StorageError { + #[error("extension storage closed")] + Closed, + #[error("extension storage master key must be 32 base64-encoded bytes")] + InvalidKey, + #[error("extension storage master key is not configured")] + MissingKey, + #[error("invalid extension ID")] + InvalidExtensionId, + #[error("extension storage path must be a regular file")] + InvalidPath, + #[error("ciphertext too short")] + ShortCiphertext, + #[error("cipher: message authentication failed")] + Authentication, + #[error("failed to generate nonce: {0}")] + Random(String), + #[error("{0}")] + Io(#[from] io::Error), + #[error("{0}")] + Json(#[from] serde_json::Error), + #[error("extension storage must contain a JSON object")] + InvalidObject, +} + +/// The platform supplies this key. Rust never persists it or includes it in Debug. +pub struct StorageMasterKey(Zeroizing<[u8; 32]>); + +impl StorageMasterKey { + pub fn from_base64(encoded: &str) -> Result<Self, StorageError> { + let encoding = GeneralPurpose::new( + &alphabet::STANDARD, + GeneralPurposeConfig::new().with_decode_allow_trailing_bits(true), + ); + let decoded = Zeroizing::new( + encoding + .decode(encoded.replace(['\r', '\n'], "")) + .map_err(|_| StorageError::InvalidKey)?, + ); + let key: [u8; 32] = decoded + .as_slice() + .try_into() + .map_err(|_| StorageError::InvalidKey)?; + Ok(Self(Zeroizing::new(key))) + } + + pub fn derive(&self, extension_id: &str, purpose: &str) -> Zeroizing<[u8; 32]> { + let mut mac = <Hmac<Sha256> as Mac>::new_from_slice(self.0.as_ref()) + .expect("HMAC accepts any key length"); + mac.update(b"SpotiFLAC Mobile extension storage v2\0"); + mac.update(purpose.as_bytes()); + mac.update(&[0]); + mac.update(extension_id.as_bytes()); + Zeroizing::new(mac.finalize().into_bytes().into()) + } +} + +/// Go's wire format: 12-byte nonce, ciphertext, then the 16-byte authentication tag. +pub fn encrypt(plaintext: &[u8], key: &[u8; 32]) -> Result<Vec<u8>, StorageError> { + let mut nonce = [0u8; 12]; + OsRng + .try_fill_bytes(&mut nonce) + .map_err(|error| StorageError::Random(error.to_string()))?; + let cipher = Aes256Gcm::new(key.into()); + let ciphertext = cipher + .encrypt(Nonce::from_slice(&nonce), plaintext) + .map_err(|_| StorageError::Authentication)?; + let mut result = Vec::with_capacity(nonce.len() + ciphertext.len()); + result.extend_from_slice(&nonce); + result.extend_from_slice(&ciphertext); + Ok(result) +} + +pub fn decrypt(ciphertext: &[u8], key: &[u8; 32]) -> Result<Zeroizing<Vec<u8>>, StorageError> { + if ciphertext.len() < 12 { + return Err(StorageError::ShortCiphertext); + } + let cipher = Aes256Gcm::new(key.into()); + let plaintext = cipher + .decrypt(Nonce::from_slice(&ciphertext[..12]), &ciphertext[12..]) + .map_err(|_| StorageError::Authentication)?; + Ok(Zeroizing::new(plaintext)) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreKind { + Storage, + Credentials, + Settings, +} + +impl StoreKind { + fn filename(self) -> &'static str { + match self { + Self::Storage => "storage.json", + Self::Credentials => ".credentials.enc", + Self::Settings => "settings.enc", + } + } + + fn purpose(self) -> &'static str { + match self { + Self::Storage => "", + Self::Credentials => "credentials", + Self::Settings => "settings", + } + } +} + +#[derive(PartialEq, Eq)] +struct Identity { + length: u64, + modified: Option<SystemTime>, + #[cfg(unix)] + inode: u64, +} + +#[derive(Default)] +struct FileCache { + snapshot: Option<Map<String, Value>>, + identity: Option<Identity>, + key_tag: Option<[u8; 32]>, +} + +type SharedFile = Arc<Mutex<FileCache>>; + +fn shared_file(path: &Path) -> SharedFile { + static FILES: OnceLock<Mutex<HashMap<PathBuf, Weak<Mutex<FileCache>>>>> = OnceLock::new(); + let mut files = FILES + .get_or_init(Mutex::default) + .lock() + .expect("extension file registry lock"); + if let Some(file) = files.get(path).and_then(Weak::upgrade) { + return file; + } + files.retain(|_, file| file.strong_count() > 0); + let file = Arc::new(Mutex::new(FileCache::default())); + files.insert(path.to_owned(), Arc::downgrade(&file)); + file +} + +/// Stores for one canonical data directory. Instances share locks and immutable +/// snapshots, while values returned to callers are independent deep copies. +pub struct ExtensionStore { + directory: PathBuf, + extension_id: String, + master_key: Option<Arc<StorageMasterKey>>, + credentials_tag: OnceLock<Option<[u8; 32]>>, + settings_tag: OnceLock<Option<[u8; 32]>>, + storage: SharedFile, + credentials: SharedFile, + settings: SharedFile, + salt: SharedFile, + closed: RwLock<bool>, +} + +impl ExtensionStore { + pub fn open( + directory: &Path, + extension_id: &str, + master_key: Option<Arc<StorageMasterKey>>, + ) -> Result<Self, StorageError> { + if !valid_extension_id(extension_id) { + return Err(StorageError::InvalidExtensionId); + } + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(directory)?; + let directory = fs::canonicalize(directory)?; + Ok(Self { + storage: shared_file(&directory.join(StoreKind::Storage.filename())), + credentials: shared_file(&directory.join(StoreKind::Credentials.filename())), + settings: shared_file(&directory.join(StoreKind::Settings.filename())), + salt: shared_file(&directory.join(".cred_salt")), + directory, + extension_id: extension_id.to_owned(), + master_key, + credentials_tag: OnceLock::new(), + settings_tag: OnceLock::new(), + closed: RwLock::new(false), + }) + } + + fn file(&self, kind: StoreKind) -> &SharedFile { + match kind { + StoreKind::Storage => &self.storage, + StoreKind::Credentials => &self.credentials, + StoreKind::Settings => &self.settings, + } + } + + fn key(&self, kind: StoreKind) -> Result<Zeroizing<[u8; 32]>, StorageError> { + Ok(self + .master_key + .as_ref() + .ok_or(StorageError::MissingKey)? + .derive(&self.extension_id, kind.purpose())) + } + + fn cache_tag(&self, kind: StoreKind) -> Option<[u8; 32]> { + let cache = match kind { + StoreKind::Storage => return None, + StoreKind::Credentials => &self.credentials_tag, + StoreKind::Settings => &self.settings_tag, + }; + *cache.get_or_init(|| { + self.master_key.as_ref().map(|key| { + Sha256::digest(key.derive(&self.extension_id, kind.purpose()).as_ref()).into() + }) + }) + } + + fn refresh<'a>( + &self, + kind: StoreKind, + cache: &'a mut FileCache, + ) -> Result<&'a Map<String, Value>, StorageError> { + let path = self.directory.join(kind.filename()); + let identity = file_identity(&path)?; + let key_tag = self.cache_tag(kind); + if cache.snapshot.is_none() + || cache.identity != identity + || cache.key_tag != key_tag + || (kind == StoreKind::Settings && identity.is_none()) + { + let snapshot = match read_optional(&path)? { + Some(data) if kind == StoreKind::Storage => parse_object(&data)?, + Some(data) => { + let key = self.key(kind)?; + let plaintext = match decrypt(&data, &key) { + Ok(plaintext) => plaintext, + Err(error) if kind == StoreKind::Credentials => { + let legacy = self.legacy_credentials_key()?; + let plaintext = decrypt(&data, &legacy).map_err(|_| error)?; + // Validate before replacing a legacy file. Corrupt data is preserved. + parse_object(&plaintext)?; + atomic_write(&path, &encrypt(&plaintext, &key)?)?; + plaintext + } + Err(error) => return Err(error), + }; + parse_object(&plaintext)? + } + None if kind == StoreKind::Settings => self.migrate_settings(&path)?, + None => Map::new(), + }; + cache.identity = file_identity(&path)?; + cache.key_tag = key_tag; + cache.snapshot = Some(snapshot); + } + Ok(cache.snapshot.as_ref().expect("loaded storage snapshot")) + } + + fn legacy_credentials_key(&self) -> Result<Zeroizing<[u8; 32]>, StorageError> { + let _guard = self.salt.lock().expect("extension salt lock"); + let path = self.directory.join(".cred_salt"); + let salt = match read_optional(&path)? { + Some(salt) if salt.len() == 32 => salt, + _ => { + let mut salt = vec![0u8; 32]; + OsRng + .try_fill_bytes(&mut salt) + .map_err(|error| StorageError::Random(error.to_string()))?; + atomic_write(&path, &salt)?; + salt + } + }; + let mut hash = Sha256::new(); + hash.update(self.extension_id.as_bytes()); + hash.update(&salt); + Ok(Zeroizing::new(hash.finalize().into())) + } + + fn migrate_settings(&self, path: &Path) -> Result<Map<String, Value>, StorageError> { + let legacy = self.directory.join("settings.json"); + let Some(data) = read_optional(&legacy)? else { + return Ok(Map::new()); + }; + let snapshot = parse_object(&data)?; + let key = self.key(StoreKind::Settings)?; + atomic_write(path, &encrypt(&data, &key)?)?; + fs::remove_file(legacy)?; + Ok(snapshot) + } + + pub fn get(&self, kind: StoreKind, key: &str) -> Result<Option<Value>, StorageError> { + let _access = self.access()?; + let mut cache = self.file(kind).lock().expect("extension storage lock"); + Ok(self.refresh(kind, &mut cache)?.get(key).cloned()) + } + + pub fn all(&self, kind: StoreKind) -> Result<Map<String, Value>, StorageError> { + let _access = self.access()?; + let mut cache = self.file(kind).lock().expect("extension storage lock"); + Ok(self.refresh(kind, &mut cache)?.clone()) + } + + pub fn set(&self, kind: StoreKind, key: &str, value: Value) -> Result<(), StorageError> { + self.mutate(kind, |snapshot| { + snapshot.insert(key.to_owned(), value); + true + }) + } + + pub fn remove(&self, kind: StoreKind, key: &str) -> Result<(), StorageError> { + self.mutate(kind, |snapshot| { + let existed = snapshot.remove(key).is_some(); + // Go credentials.remove persists even if the key did not exist. + existed || kind != StoreKind::Storage + }) + } + + pub fn replace(&self, kind: StoreKind, value: Map<String, Value>) -> Result<(), StorageError> { + self.mutate(kind, |snapshot| { + *snapshot = value; + true + }) + } + + fn mutate( + &self, + kind: StoreKind, + mutate: impl FnOnce(&mut Map<String, Value>) -> bool, + ) -> Result<(), StorageError> { + let _access = self.access()?; + let mut cache = self.file(kind).lock().expect("extension storage lock"); + let mut snapshot = self.refresh(kind, &mut cache)?.clone(); + if !mutate(&mut snapshot) { + return Ok(()); + } + let data = Zeroizing::new(serde_json::to_vec(&snapshot)?); + let path = self.directory.join(kind.filename()); + if kind == StoreKind::Storage { + atomic_write(&path, &data)?; + } else { + let key = self.key(kind)?; + atomic_write(&path, &encrypt(&data, &key)?)?; + } + cache.identity = file_identity(&path)?; + cache.key_tag = self.cache_tag(kind); + cache.snapshot = Some(snapshot); + Ok(()) + } + + fn access(&self) -> Result<RwLockReadGuard<'_, bool>, StorageError> { + let guard = self + .closed + .read() + .expect("extension storage lifecycle lock"); + if *guard { + Err(StorageError::Closed) + } else { + Ok(guard) + } + } + + /// Wait for current reads/writes, then prevent stale handles from accessing + /// a replacement installation at the same data path. + pub(crate) fn close(&self) { + *self + .closed + .write() + .expect("extension storage lifecycle lock") = true; + } +} + +pub(crate) fn valid_extension_id(id: &str) -> bool { + (1..=128).contains(&id.len()) + && (id.as_bytes()[0].is_ascii_lowercase() || id.as_bytes()[0].is_ascii_digit()) + && id.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte) + }) +} + +fn file_identity(path: &Path) -> Result<Option<Identity>, StorageError> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if !metadata.is_file() { + return Err(StorageError::InvalidPath); + } + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + Ok(Some(Identity { + length: metadata.len(), + modified: metadata.modified().ok(), + #[cfg(unix)] + inode: metadata.ino(), + })) +} + +fn read_optional(path: &Path) -> Result<Option<Vec<u8>>, StorageError> { + if file_identity(path)?.is_none() { + return Ok(None); + } + Ok(Some(fs::read(path)?)) +} + +fn parse_object(data: &[u8]) -> Result<Map<String, Value>, StorageError> { + match serde_json::from_slice(data)? { + Value::Object(value) => Ok(value), + Value::Null => Ok(Map::new()), + _ => Err(StorageError::InvalidObject), + } +} + +pub(crate) fn atomic_write(path: &Path, data: &[u8]) -> Result<(), StorageError> { + file_identity(path)?; + let directory = path.parent().ok_or(StorageError::InvalidPath)?; + let mut file = tempfile::Builder::new() + .prefix(".spotiflac-") + .tempfile_in(directory)?; + file.write_all(data)?; + file.as_file().sync_all()?; + file.persist(path).map_err(|error| error.error)?; + // Directory fsync is unavailable on some filesystems; rename remains atomic. + if let Ok(directory) = File::open(directory) { + let _ = directory.sync_all(); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retired_store_cannot_read_or_overwrite_a_reinstalled_extension() { + let directory = tempfile::tempdir().unwrap(); + let key = Arc::new( + StorageMasterKey::from_base64("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=").unwrap(), + ); + let old = ExtensionStore::open( + directory.path(), + "example.extension", + Some(Arc::clone(&key)), + ) + .unwrap(); + old.set(StoreKind::Storage, "old", true.into()).unwrap(); + old.close(); + fs::remove_file(directory.path().join("storage.json")).unwrap(); + let replacement = + ExtensionStore::open(directory.path(), "example.extension", Some(key)).unwrap(); + for kind in [ + StoreKind::Storage, + StoreKind::Credentials, + StoreKind::Settings, + ] { + replacement.set(kind, "new", true.into()).unwrap(); + assert!(matches!(old.get(kind, "new"), Err(StorageError::Closed))); + assert!(matches!(old.all(kind), Err(StorageError::Closed))); + assert!(matches!( + old.set(kind, "stale", true.into()), + Err(StorageError::Closed) + )); + assert!(matches!(old.remove(kind, "new"), Err(StorageError::Closed))); + assert_eq!( + replacement.all(kind).unwrap(), + Map::from_iter([("new".into(), true.into())]) + ); + } + } + + #[test] + fn encrypted_snapshots_are_scoped_to_master_key() { + let directory = tempfile::tempdir().unwrap(); + let key = Arc::new( + StorageMasterKey::from_base64("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=").unwrap(), + ); + let other_key = Arc::new( + StorageMasterKey::from_base64("AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=").unwrap(), + ); + let store = ExtensionStore::open( + directory.path(), + "example.extension", + Some(Arc::clone(&key)), + ) + .unwrap(); + store + .set(StoreKind::Credentials, "token", "secret".into()) + .unwrap(); + store + .set(StoreKind::Settings, "theme", "dark".into()) + .unwrap(); + + let other = + ExtensionStore::open(directory.path(), "example.extension", Some(other_key)).unwrap(); + for kind in [StoreKind::Credentials, StoreKind::Settings] { + assert!(matches!(other.all(kind), Err(StorageError::Authentication))); + } + + let missing = ExtensionStore::open(directory.path(), "example.extension", None).unwrap(); + for kind in [StoreKind::Credentials, StoreKind::Settings] { + assert!(matches!(missing.all(kind), Err(StorageError::MissingKey))); + } + } +} diff --git a/rust_backend/crates/extensions/src/transfer_policy.rs b/rust_backend/crates/extensions/src/transfer_policy.rs new file mode 100644 index 00000000..a6019982 --- /dev/null +++ b/rust_backend/crates/extensions/src/transfer_policy.rs @@ -0,0 +1,133 @@ +//! Manifest-declared download policy. Execution belongs to the download manager. + +use serde::Serialize; +use serde_json::{Map, Value}; +use std::collections::BTreeSet; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DownloadTransferPolicy { + pub max_attempts: i64, + pub initial_retry_delay_ms: i64, + pub max_retry_delay_ms: i64, + pub resume_policy: String, + pub persistent_checkpoint: bool, + pub refresh_stream_on_status: BTreeSet<i64>, + pub max_parallel_segments: i64, + pub max_concurrent_downloads: i64, +} + +impl Default for DownloadTransferPolicy { + fn default() -> Self { + Self { + max_attempts: 3, + initial_retry_delay_ms: 500, + max_retry_delay_ms: 8000, + resume_policy: "none".to_owned(), + persistent_checkpoint: false, + refresh_stream_on_status: BTreeSet::from([401, 403]), + max_parallel_segments: 3, + max_concurrent_downloads: 3, + } + } +} + +fn number(value: Option<&Value>, fallback: i64) -> i64 { + value.and_then(Value::as_f64).map_or(fallback, rounded_int) +} + +pub(crate) fn rounded_int(value: f64) -> i64 { + let rounded = value.round(); + // Go uses the target's conversion: ARM saturates, while x86 returns + // MinInt on overflow. Keep the native Go int width on ARM32 as well. + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + if rounded >= isize::MAX as f64 || rounded < isize::MIN as f64 { + return isize::MIN as i64; + } + (rounded as isize) as i64 +} + +impl DownloadTransferPolicy { + pub fn from_capabilities(capabilities: &Map<String, Value>) -> Self { + let mut policy = Self::default(); + let Some(config) = capabilities + .get("downloadTransfer") + .and_then(Value::as_object) + else { + return policy; + }; + policy.max_attempts = number(config.get("maxAttempts"), policy.max_attempts).clamp(1, 8); + policy.initial_retry_delay_ms = number( + config.get("initialRetryDelayMs"), + policy.initial_retry_delay_ms, + ) + .clamp(100, 30_000); + policy.max_retry_delay_ms = + number(config.get("maxRetryDelayMs"), policy.max_retry_delay_ms) + .clamp(policy.initial_retry_delay_ms, 120_000); + if let Some(resume) = config.get("resumePolicy").and_then(Value::as_str) { + let resume = resume.trim().to_lowercase(); + if matches!(resume.as_str(), "none" | "validated") { + policy.resume_policy = resume; + } + } + policy.persistent_checkpoint = config.get("persistentCheckpoint") + == Some(&Value::Bool(true)) + && policy.resume_policy == "validated"; + if let Some(statuses) = config + .get("refreshStreamOnStatus") + .and_then(Value::as_array) + { + let statuses: BTreeSet<_> = statuses + .iter() + .map(|value| number(Some(value), 0)) + .filter(|value| (400..=599).contains(value)) + .collect(); + if !statuses.is_empty() { + policy.refresh_stream_on_status = statuses; + } + } + policy.max_parallel_segments = number( + config.get("maxParallelSegments"), + policy.max_parallel_segments, + ) + .clamp(1, 8); + policy.max_concurrent_downloads = number( + config.get("maxConcurrentDownloads"), + policy.max_concurrent_downloads, + ) + .clamp(1, 3); + policy + } +} + +pub(crate) fn validate(capabilities: &Map<String, Value>) -> Result<(), String> { + let Some(raw) = capabilities.get("downloadTransfer") else { + return Ok(()); + }; + let config = raw.as_object().ok_or("must be an object")?; + if let Some(resume) = config.get("resumePolicy") + && !matches!(resume.as_str(), Some("none" | "validated")) + { + return Err("resumePolicy must be 'none' or 'validated'".to_owned()); + } + if let Some(checkpoint) = config.get("persistentCheckpoint") + && !checkpoint.is_boolean() + { + return Err("persistentCheckpoint must be a boolean".to_owned()); + } + for key in [ + "maxAttempts", + "initialRetryDelayMs", + "maxRetryDelayMs", + "maxParallelSegments", + "maxConcurrentDownloads", + ] { + if let Some(value) = config.get(key) + && number(Some(value), -1) < 0 + { + return Err(format!("{key} must be a non-negative number")); + } + } + Ok(()) +} diff --git a/rust_backend/crates/extensions/src/utility_host.rs b/rust_backend/crates/extensions/src/utility_host.rs new file mode 100644 index 00000000..f27540a2 --- /dev/null +++ b/rust_backend/crates/extensions/src/utility_host.rs @@ -0,0 +1,123 @@ +use crate::runtime::{Control, ExtensionServices}; +use rquickjs::{Ctx, Function, Object, Value}; +use spotiflac_core::matching; +pub(crate) use spotiflac_network::random_user_agent; +use std::sync::Arc; + +pub(crate) fn register<'js>( + ctx: &Ctx<'js>, + host: &Object<'js>, + control: Arc<Control>, + services: &ExtensionServices, +) -> rquickjs::Result<()> { + let logs = Arc::clone(&services.logs); + let id = services.extension_id.clone(); + host.set( + "managedConsole", + services.load_mode != crate::runtime::LoadMode::Initialize, + )?; + let auth = services.auth_registry.clone(); + let provider_id = services.extension_id.clone(); + host.set( + "providerPendingVerification", + Function::new(ctx.clone(), move || { + auth.as_ref() + .filter(|auth| auth.has_fresh_challenge(&provider_id)) + .map(|_| provider_id.clone()) + })?, + )?; + let progress = Arc::clone(&services.downloads.progress); + let progress_control = Arc::clone(&control); + host.set( + "providerProgress", + Function::new(ctx.clone(), move |percent: i32| { + let id = progress_control.item_id(); + if !id.is_empty() && progress_control.check().is_ok() { + let _ = progress.set_progress(&id, f64::from(percent) / 100.0, 0, 0); + } + })?, + )?; + host.set( + "providerInteger", + Function::new(ctx.clone(), crate::provider::integer)?, + )?; + host.set( + "providerTrim", + Function::new(ctx.clone(), |value: String| value.trim().to_owned())?, + )?; + host.set( + "providerAudioTraits", + Function::new(ctx.clone(), crate::provider::audio_traits)?, + )?; + host.set( + "extensionLog", + Function::new( + ctx.clone(), + move |level: String, values: Vec<String>, count: usize| { + logs.extension(&id, &level, values, count) + }, + )?, + )?; + host.set( + "logOpaqueType", + Function::new(ctx.clone(), |value: Value<'js>| { + if value.is_proxy() { + "<goja.Proxy>" + } else if value.is_promise() { + "<*goja.Promise>" + } else { + "" + } + })?, + )?; + let comparison = Arc::clone(&control); + host.set( + "compareStrings", + Function::new(ctx.clone(), move |first: String, second: String| { + // The call boundary turns cancellation/timeout into its typed error. + matching::compare_strings(&first, &second, &|| { + comparison.check().map_err(|e| e.to_string()) + }) + .unwrap_or(0.0) + })?, + )?; + host.set( + "normalizeMatching", + Function::new(ctx.clone(), |value: String| matching::normalize(&value))?, + )?; + host.set( + "compareDuration", + Function::new(ctx.clone(), |first: f64, second: f64, tolerance: f64| { + // Goja's ToInteger clips infinities and overflow to the i64 endpoints. + matching::compare_duration(first as i64, second as i64, tolerance as i64) + })?, + )?; + let downloads = Arc::clone(&services.downloads); + let download_control = Arc::clone(&control); + host.set( + "downloadCancelled", + Function::new(ctx.clone(), move || { + let id = download_control.item_id(); + !id.is_empty() && downloads.cancellation.is_cancelled(&id).unwrap_or(false) + })?, + )?; + host.set( + "requestCancelled", + Function::new(ctx.clone(), move || control.request_cancelled())?, + )?; + let version = services.app_version.clone(); + host.set( + "appVersion", + Function::new(ctx.clone(), move || version.get())?, + )?; + let version = services.app_version.clone(); + host.set( + "appUserAgent", + Function::new(ctx.clone(), move || version.user_agent())?, + )?; + host.set( + "randomUserAgent", + Function::new(ctx.clone(), random_user_agent)?, + )?; + Ok(()) +} diff --git a/rust_backend/crates/extensions/tests/optimization_fetch.rs b/rust_backend/crates/extensions/tests/optimization_fetch.rs new file mode 100644 index 00000000..23498854 --- /dev/null +++ b/rust_backend/crates/extensions/tests/optimization_fetch.rs @@ -0,0 +1,260 @@ +use serde_json::{Value, json}; +use spotiflac_extensions::RuntimeLimits; +use spotiflac_extensions::environment::ExtensionEnvironment; +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, mpsc}; +use std::thread; +use std::time::{Duration, Instant}; + +const KEY: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + +struct Request { + method: String, + path: String, + headers: BTreeMap<String, String>, + body: Vec<u8>, +} + +struct Server { + base: String, + requests: mpsc::Receiver<Request>, + stop: Arc<AtomicBool>, + worker: Option<thread::JoinHandle<()>>, +} + +impl Server { + fn new(responses: &[(&str, &[u8])]) -> Self { + let responses: BTreeMap<_, _> = responses + .iter() + .map(|(path, body)| (path.to_string(), body.to_vec())) + .collect(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let stop = Arc::new(AtomicBool::new(false)); + let stopped = stop.clone(); + let (sent, requests) = mpsc::channel(); + let worker = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(10); + while !stopped.load(Ordering::Acquire) && Instant::now() < deadline { + let (mut stream, _) = match listener.accept() { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(1)); + continue; + } + Err(error) => panic!("fetch fixture accept: {error}"), + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut bytes = Vec::new(); + let header_end = loop { + if let Some(index) = bytes.windows(4).position(|part| part == b"\r\n\r\n") { + break index + 4; + } + let mut chunk = [0; 4096]; + let count = stream.read(&mut chunk).unwrap(); + assert!(count > 0 && bytes.len() + count <= 16384); + bytes.extend_from_slice(&chunk[..count]); + }; + let mut lines = std::str::from_utf8(&bytes[..header_end]) + .unwrap() + .split("\r\n"); + let mut first = lines.next().unwrap().split_whitespace(); + let method = first.next().unwrap().to_owned(); + let path = first.next().unwrap().to_owned(); + let headers: BTreeMap<_, _> = lines + .filter_map(|line| line.split_once(':')) + .map(|(key, value)| (key.to_ascii_lowercase(), value.trim().to_owned())) + .collect(); + let length = headers + .get("content-length") + .map(|value| value.parse::<usize>().unwrap()) + .unwrap_or(0); + assert!(length <= 1 << 20); + while bytes.len() - header_end < length { + let mut chunk = [0; 16384]; + let count = stream.read(&mut chunk).unwrap(); + assert!(count > 0 && bytes.len() + count <= (1 << 20) + header_end); + bytes.extend_from_slice(&chunk[..count]); + } + let response = responses.get(&path).map(Vec::as_slice).unwrap_or(b"{}"); + sent.send(Request { + method, + path, + headers, + body: bytes[header_end..].to_vec(), + }) + .unwrap(); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + response.len() + ) + .unwrap(); + stream.write_all(response).unwrap(); + } + }); + Self { + base, + requests, + stop, + worker: Some(worker), + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +fn manifest() -> String { + json!({"name":"example.fetch","version":"1","description":"Generic fetch fixture","type":["metadata_provider"], + "permissions":{"network":["127.0.0.1"],"allowHttp":true}}).to_string() +} + +#[test] +fn fetch_json_preserves_go_numbers_normalization_and_fresh_objects() { + let server = Server::new(&[ + ("/numbers", b"[-0,-0.0,9007199254740993,9223372036854775807,18446744073709551615,1.7976931348623157e308,5e-324,-1e-4000]"), + ("/overflow", br#"{"x":1e400,"x":0}"#), + ("/malformed", b"{\"value\":\"\xff\xe2\x82\"}"), + ("/surrogates", br#"{"\ud800":1,"\ufffd":2,"\ud800":3,"value":"\udfff","pair":"\ud83d\ude00"}"#), + ("/object", br#"{"__proto__":{"polluted":true},"nested":{"n":1},"items":[1,2]}"#), + ("/invalid", br#"{"x":1,}"#), + ("/trailing", b"{}{}"), + ("/string", br#"{"x":"[1e400] \" { "}"#), + ("/null", b"null"), + ]); + let directory = tempfile::tempdir().unwrap(); + let environment = ExtensionEnvironment::new(directory.path(), KEY, "1").unwrap(); + environment.set_allow_private_network(true).unwrap(); + let runtime = environment.load(&manifest(), r#"registerExtension({run(base){ + const numbers = fetch(base+'/numbers').json(); + const malformed = fetch(base+'/malformed').json(); + const surrogates = fetch(base+'/surrogates').json(); + const response = fetch(base+'/object'); + const first = response.json(); + first.nested.n = 9; first.items.push(3); first.extra = true; + const second = response.json(); + return { + zero: Object.is(numbers[0],-0) && Object.is(numbers[1],-0) && Object.is(numbers[7],-0), + numbers: numbers[2]===9007199254740992 && numbers[3]===9223372036854775808 + && numbers[4]===18446744073709551616 && numbers[5]===Number.MAX_VALUE && numbers[6]===Number.MIN_VALUE, + overflow: fetch(base+'/overflow').json()===undefined, + malformed: malformed.value==='\ufffd\ufffd\ufffd', + surrogates: surrogates['\ufffd']===3 && surrogates.value==='\ufffd' && surrogates.pair==='\ud83d\ude00', + proto: Object.prototype.hasOwnProperty.call(second,'__proto__') + && second.__proto__.polluted===true && Object.getPrototypeOf(second)===Object.prototype && ({}).polluted===undefined, + fresh: first!==second && first.nested!==second.nested && second.nested.n===1 && second.items.length===2 && second.extra===undefined, + invalid: fetch(base+'/invalid').json()===undefined && fetch(base+'/trailing').json()===undefined, + string: fetch(base+'/string').json().x==='[1e400] " { ', + null: fetch(base+'/null').json()===null + }; + }});"#, RuntimeLimits::default()).unwrap(); + let result = runtime + .call("run", &json!([server.base]).to_string(), None, 5000) + .unwrap(); + let result: Value = serde_json::from_str(&result).unwrap(); + assert_eq!(result.as_object().unwrap().len(), 10); + for (name, value) in result.as_object().unwrap() { + assert_eq!(value, true, "{name}: {result}"); + } + environment.shutdown(); +} + +#[test] +fn direct_http_arguments_preserve_body_bytes_getters_headers_and_content_type() { + let server = Server::new(&[]); + let directory = tempfile::tempdir().unwrap(); + let environment = ExtensionEnvironment::new(directory.path(), KEY, "1").unwrap(); + environment.set_allow_private_network(true).unwrap(); + let runtime = environment.load(&manifest(), r#"registerExtension({run(base){ + const events = []; + const url = {toString(){events.push('url');return base+'/large';}}; + const body = { + get z(){events.push('body.z');return undefined;}, + get a(){events.push('body.a');return '\ud800';}, + get blob(){events.push('body.blob');return 'x'.repeat(262144);} + }; + const headers = { + get 'X-Zero'(){events.push('header.zero');return -0;}, + get 'X-List'(){events.push('header.list');return ['one',2];}, + get 'X-Null'(){events.push('header.null');return null;} + }; + const options = { + get method(){events.push('method');return 'post';}, + get body(){events.push('body');return body;}, + get headers(){events.push('headers');return headers;} + }; + const large = fetch(url,options); + const empty = http.post(base+'/empty',undefined); + const emptyFetch = fetch(base+'/empty-fetch',{method:'POST'}); + const override = fetch(base+'/override',{method:'POST',body:'raw\0\ud800',headers:{'Content-Type':'application/custom'}}); + return {events,statuses:[large.status,empty.statusCode,emptyFetch.status,override.status]}; + }});"#, RuntimeLimits::default()).unwrap(); + let result = runtime + .call("run", &json!([server.base]).to_string(), None, 5000) + .unwrap(); + let result: Value = serde_json::from_str(&result).unwrap(); + assert_eq!( + result["events"], + json!([ + "url", + "method", + "method", + "body", + "body.z", + "body.a", + "body.blob", + "headers", + "header.zero", + "header.list", + "header.null" + ]) + ); + assert_eq!(result["statuses"], json!([200, 200, 200, 200])); + let requests: Vec<_> = (0..4) + .map(|_| { + server + .requests + .recv_timeout(Duration::from_secs(2)) + .unwrap() + }) + .collect(); + assert!(requests.iter().all(|request| request.method == "POST")); + let large = &requests[0]; + assert_eq!(large.path, "/large"); + let expected = format!( + "{{\"a\":\"\u{fffd}\",\"blob\":\"{}\",\"z\":null}}", + "x".repeat(262144) + ); + assert_eq!(large.body, expected.as_bytes()); + assert_eq!(large.headers["x-zero"], "-0"); + assert_eq!(large.headers["x-list"], "[one 2]"); + assert_eq!(large.headers["x-null"], "<nil>"); + assert_eq!(large.headers["content-type"], "application/json"); + assert_eq!(requests[1].path, "/empty"); + assert!(requests[1].body.is_empty()); + assert_eq!(requests[1].headers["content-type"], "application/json"); + assert_eq!(requests[2].path, "/empty-fetch"); + assert!(requests[2].body.is_empty()); + assert!(!requests[2].headers.contains_key("content-type")); + assert_eq!(requests[3].path, "/override"); + assert_eq!(requests[3].body, b"raw\0\xef\xbf\xbd"); + assert_eq!(requests[3].headers["content-type"], "application/custom"); + environment.shutdown(); +} diff --git a/rust_backend/crates/extensions/tests/optimization_redirects.rs b/rust_backend/crates/extensions/tests/optimization_redirects.rs new file mode 100644 index 00000000..19882be7 --- /dev/null +++ b/rust_backend/crates/extensions/tests/optimization_redirects.rs @@ -0,0 +1,257 @@ +use spotiflac_network::policy::NetworkPermissions; +use spotiflac_network::{HttpRequest, NetworkService}; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::thread; +use std::time::{Duration, Instant}; + +struct Server { + base: String, + connections: Arc<AtomicUsize>, + redirected: mpsc::Receiver<()>, + stop: Arc<AtomicBool>, + worker: Option<thread::JoinHandle<()>>, +} + +fn pause(stop: &AtomicBool, duration: Duration) -> bool { + let deadline = Instant::now() + duration; + while Instant::now() < deadline { + if stop.load(Ordering::Acquire) { + return false; + } + thread::sleep(Duration::from_millis(2)); + } + !stop.load(Ordering::Acquire) +} + +fn connection(mut stream: TcpStream, stop: &AtomicBool, redirected: mpsc::Sender<()>) { + stream + .set_read_timeout(Some(Duration::from_millis(25))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_millis(100))) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + let mut request = Vec::new(); + while Instant::now() < deadline && !stop.load(Ordering::Acquire) { + let mut bytes = [0; 1024]; + match stream.read(&mut bytes) { + Ok(0) => break, + Ok(count) => request.extend_from_slice(&bytes[..count]), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => + { + continue; + } + Err(_) => break, + } + assert!(request.len() <= 8192); + if !request.windows(4).any(|part| part == b"\r\n\r\n") { + continue; + } + let path = std::str::from_utf8(&request) + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .to_owned(); + request.clear(); + if path.starts_with("/final") { + if path == "/final-cancel" && !pause(stop, Duration::from_secs(1)) { + break; + } + if stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: keep-alive\r\n\r\nok", + ) + .is_err() + { + break; + } + continue; + } + let final_path = if path == "/cancel" { + "/final-cancel" + } else { + "/final" + }; + let length = if path == "/huge" { 64 << 20 } else { 32 }; + if write!(stream, "HTTP/1.1 302 Found\r\nLocation: {final_path}\r\nContent-Length: {length}\r\nConnection: keep-alive\r\n\r\n").is_err() { + break; + } + if path == "/short" { + if stream.write_all(&[b'x'; 32]).is_err() { + break; + } + let _ = redirected.send(()); + continue; + } + if stream.write_all(b"x").is_err() { + break; + } + let _ = redirected.send(()); + if path == "/huge" { + let _ = stream.write_all(&[b'x'; 65535]); + pause(stop, Duration::from_secs(1)); + break; + } + let delay = if path == "/split" { + Duration::from_millis(10) + } else { + Duration::from_secs(1) + }; + if !pause(stop, delay) || stream.write_all(&[b'x'; 31]).is_err() { + break; + } + } +} + +impl Server { + fn new() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + listener.set_nonblocking(true).unwrap(); + let connections = Arc::new(AtomicUsize::new(0)); + let count = connections.clone(); + let stop = Arc::new(AtomicBool::new(false)); + let stopped = stop.clone(); + let (redirect, redirected) = mpsc::channel(); + let worker = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(5); + let mut workers = Vec::new(); + while Instant::now() < deadline && !stopped.load(Ordering::Acquire) { + match listener.accept() { + Ok((stream, _)) => { + assert!(count.fetch_add(1, Ordering::AcqRel) < 4); + let stopped = stopped.clone(); + let redirect = redirect.clone(); + workers.push(thread::spawn(move || { + connection(stream, &stopped, redirect) + })); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(1)); + } + Err(error) => panic!("redirect fixture accept: {error}"), + } + } + stopped.store(true, Ordering::Release); + for worker in workers { + worker.join().unwrap(); + } + }); + Self { + base, + connections, + redirected, + stop, + worker: Some(worker), + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +fn request(url: String) -> HttpRequest { + serde_json::from_value(serde_json::json!({"url":url})).unwrap() +} + +#[test] +fn redirect_bodies_report_connection_reuse_without_waiting_for_unbounded_bodies() { + for route in ["short", "split", "delayed", "huge"] { + let server = Server::new(); + let service = NetworkService::new().unwrap(); + service.set_allow_private_network(true); + let session = service.session( + NetworkPermissions { + domains: vec!["127.0.0.1".into()], + allow_http: true, + }, + Duration::from_secs(3), + ); + let began = Instant::now(); + let response = session + .request(request(format!("{}/{route}", server.base)), || Ok(())) + .unwrap(); + let elapsed = began.elapsed(); + assert_eq!(response.status, 200); + assert_eq!(response.body, b"ok"); + let connections = server.connections.load(Ordering::Acquire); + assert_eq!( + connections, + if matches!(route, "short" | "split") { + 1 + } else { + 2 + }, + "{route}" + ); + eprintln!( + "redirect route={route} connections={connections} elapsed_us={}", + elapsed.as_micros() + ); + if matches!(route, "delayed" | "huge") { + assert!( + elapsed < Duration::from_millis(750), + "{route} waited for an unnecessary redirect body: {elapsed:?}" + ); + } + } +} + +#[test] +fn cancellation_interrupts_redirect_body_or_followup_without_waiting_for_eof() { + let server = Server::new(); + let service = NetworkService::new().unwrap(); + service.set_allow_private_network(true); + let session = service.session( + NetworkPermissions { + domains: vec!["127.0.0.1".into()], + allow_http: true, + }, + Duration::from_secs(3), + ); + let cancelled = Arc::new(AtomicBool::new(false)); + let active = cancelled.clone(); + let input = request(format!("{}/cancel", server.base)); + let (done, result) = mpsc::channel(); + let worker = thread::spawn(move || { + let response = session.request(input, || { + if active.load(Ordering::Acquire) { + Err("redirect fixture cancelled".into()) + } else { + Ok(()) + } + }); + let _ = done.send(response); + }); + server + .redirected + .recv_timeout(Duration::from_secs(2)) + .unwrap(); + let began = Instant::now(); + cancelled.store(true, Ordering::Release); + let failure = result + .recv_timeout(Duration::from_millis(750)) + .unwrap() + .unwrap_err(); + worker.join().unwrap(); + assert_eq!(failure, "redirect fixture cancelled"); + eprintln!( + "redirect route=cancel connections={} elapsed_us={}", + server.connections.load(Ordering::Acquire), + began.elapsed().as_micros() + ); +} diff --git a/rust_backend/crates/extensions/tests/optimization_segments.rs b/rust_backend/crates/extensions/tests/optimization_segments.rs new file mode 100644 index 00000000..f72102ee --- /dev/null +++ b/rust_backend/crates/extensions/tests/optimization_segments.rs @@ -0,0 +1,260 @@ +#![cfg(unix)] + +use serde_json::{Value, json}; +use spotiflac_core::cancellation::{CancellationDomain, CancellationError, CancellationRegistry}; +use spotiflac_extensions::environment::ExtensionEnvironment; +use spotiflac_extensions::{ExtensionError, RuntimeLimits}; +use std::fs; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::{Mutex, mpsc}; +use std::thread; +use std::time::{Duration, Instant}; + +const SEGMENTS: usize = 2048; +const WINDOW: usize = 4; +static SERIAL: Mutex<()> = Mutex::new(()); + +struct Server { + base: String, + arrivals: mpsc::Receiver<usize>, + closed: mpsc::Receiver<()>, + control: mpsc::Sender<bool>, + worker: Option<thread::JoinHandle<()>>, +} + +impl Server { + fn new() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + listener.set_nonblocking(true).unwrap(); + let (sent, arrivals) = mpsc::channel(); + let (close, closed) = mpsc::channel(); + let (control, commands) = mpsc::channel(); + let worker = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(20); + let mut held: Option<TcpStream> = None; + let mut requests = 0; + while Instant::now() < deadline { + match commands.try_recv() { + Ok(true) => { + let mut stream = held.take().expect("segment zero is held"); + stream.set_nonblocking(false).unwrap(); + stream.write_all(&[0; 3]).unwrap(); + } + Ok(false) | Err(mpsc::TryRecvError::Disconnected) => break, + Err(mpsc::TryRecvError::Empty) => {} + } + if let Some(stream) = &mut held { + match stream.read(&mut [0]) { + Ok(0) => { + held.take(); + let _ = close.send(()); + } + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + ) => + { + held.take(); + let _ = close.send(()); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {} + result => panic!("unexpected held segment read: {result:?}"), + } + } + let (mut stream, _) = match listener.accept() { + Ok(connection) => connection, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if requests == SEGMENTS && held.is_none() { + break; + } + thread::sleep(Duration::from_millis(1)); + continue; + } + Err(error) => panic!("accept failed: {error}"), + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut request = Vec::new(); + while !request.windows(4).any(|part| part == b"\r\n\r\n") { + let mut bytes = [0; 1024]; + let count = stream.read(&mut bytes).unwrap(); + assert!(count > 0 && request.len() + count <= 8192); + request.extend_from_slice(&bytes[..count]); + } + let index: usize = std::str::from_utf8(&request) + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .trim_start_matches('/') + .parse() + .unwrap(); + assert!(index < SEGMENTS); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n") + .unwrap(); + if index == 0 { + stream.write_all(&[0]).unwrap(); + stream.set_nonblocking(true).unwrap(); + held = Some(stream); + } else { + stream.write_all(&(index as u32).to_be_bytes()).unwrap(); + } + requests += 1; + if sent.send(index).is_err() { + break; + } + } + }); + Self { + base, + arrivals, + closed, + control, + worker: Some(worker), + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + let _ = self.control.send(false); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +fn stalled_window(cancel: bool) { + let _serial = SERIAL.lock().unwrap_or_else(|error| error.into_inner()); + let server = Server::new(); + let directory = tempfile::tempdir().unwrap(); + let environment = ExtensionEnvironment::new( + directory.path(), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "1", + ) + .unwrap(); + environment.set_allow_private_network(true).unwrap(); + let manifest = json!({"name":"example.segments","version":"1","description":"Generic segment fixture", + "type":["download_provider"],"permissions":{"file":true,"network":["127.0.0.1"],"allowHttp":true}}).to_string(); + let runtime = environment + .load( + &manifest, + r#"registerExtension({run(base,count,parallel){ + return file.downloadSegments(Array.from({length:count},(_,index)=>base+'/'+index), + 'result.bin',{maxParallel:parallel,maxAttempts:1}); + }});"#, + RuntimeLimits::default(), + ) + .unwrap(); + let root = directory.path().join("example.segments"); + let target = root.join("result.bin"); + fs::write(&target, b"original").unwrap(); + let descriptors = fs::read_dir("/dev/fd").unwrap().count(); + let registry = CancellationRegistry::new(CancellationDomain::Download); + let lease = std::sync::Arc::new(registry.acquire("window").unwrap()); + let arguments = json!([server.base, SEGMENTS, WINDOW]).to_string(); + let (done, result) = mpsc::channel(); + let worker = thread::spawn(move || { + let value = runtime.call_download("run", &arguments, Some(lease), 10_000); + let _ = done.send(value); + }); + let mut initial = Vec::new(); + for _ in 0..WINDOW { + initial.push( + server + .arrivals + .recv_timeout(Duration::from_secs(5)) + .unwrap(), + ); + } + initial.sort_unstable(); + assert_eq!(initial, (0..WINDOW).collect::<Vec<_>>()); + assert_eq!( + server.arrivals.recv_timeout(Duration::from_millis(150)), + Err(mpsc::RecvTimeoutError::Timeout) + ); + let staged = fs::read_dir(&root) + .unwrap() + .filter(|entry| { + entry + .as_ref() + .unwrap() + .file_name() + .to_string_lossy() + .contains(".segment.") + }) + .count(); + assert_eq!( + staged, WINDOW, + "active and completed segment files share the window" + ); + assert!( + fs::read_dir("/dev/fd").unwrap().count() <= descriptors + WINDOW * 4 + 16, + "open descriptors must depend on the window, not the segment count" + ); + if cancel { + registry.cancel("window").unwrap(); + } else { + server.control.send(true).unwrap(); + } + let outcome = result.recv_timeout(Duration::from_secs(15)).unwrap(); + worker.join().unwrap(); + if cancel { + assert_eq!( + outcome, + Err(ExtensionError::Cancelled( + CancellationError::DownloadCancelled + )) + ); + server.closed.recv_timeout(Duration::from_secs(2)).unwrap(); + assert_eq!(fs::read(&target).unwrap(), b"original"); + assert!(matches!( + server.arrivals.try_recv(), + Err(mpsc::TryRecvError::Empty) + )); + } else { + let value: Value = serde_json::from_str(&outcome.unwrap()).unwrap(); + assert_eq!(value["success"], true); + let expected: Vec<_> = (0..SEGMENTS as u32).flat_map(u32::to_be_bytes).collect(); + assert_eq!(fs::read(&target).unwrap(), expected); + let mut remaining: Vec<_> = (WINDOW..SEGMENTS) + .map(|_| { + server + .arrivals + .recv_timeout(Duration::from_secs(2)) + .unwrap() + }) + .collect(); + remaining.sort_unstable(); + assert_eq!(remaining, (WINDOW..SEGMENTS).collect::<Vec<_>>()); + } + assert!(fs::read_dir(&root).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .contains(".partial") + })); + environment.shutdown(); +} + +#[test] +fn stalled_first_segment_bounds_files_and_requests_then_assembles_in_order() { + stalled_window(false); +} + +#[test] +fn cancellation_of_stalled_segment_window_joins_workers_and_preserves_output() { + stalled_window(true); +} diff --git a/rust_backend/crates/mobile/Cargo.toml b/rust_backend/crates/mobile/Cargo.toml new file mode 100644 index 00000000..e464503a --- /dev/null +++ b/rust_backend/crates/mobile/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "spotiflac-mobile" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[lib] +crate-type = ["lib", "cdylib", "staticlib"] + +[dependencies] +spotiflac-core.workspace = true +spotiflac-extensions.workspace = true +thiserror.workspace = true +uniffi.workspace = true +serde_json.workspace = true +zeroize.workspace = true +rustix.workspace = true + +[lints] +workspace = true diff --git a/rust_backend/crates/mobile/src/cancellation.rs b/rust_backend/crates/mobile/src/cancellation.rs new file mode 100644 index 00000000..ff0531cf --- /dev/null +++ b/rust_backend/crates/mobile/src/cancellation.rs @@ -0,0 +1,101 @@ +use spotiflac_core::cancellation as core; +use std::sync::Arc; + +#[derive(uniffi::Enum)] +pub enum CancellationDomain { + Download, + ExtensionRequest, +} + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum CancellationError { + #[error("download cancelled")] + DownloadCancelled, + #[error("extension request cancelled")] + ExtensionRequestCancelled, + #[error("cancellation registry closed")] + RegistryClosed, + #[error("request lease released")] + LeaseReleased, +} + +impl From<core::CancellationError> for CancellationError { + fn from(error: core::CancellationError) -> Self { + match error { + core::CancellationError::DownloadCancelled => Self::DownloadCancelled, + core::CancellationError::ExtensionRequestCancelled => Self::ExtensionRequestCancelled, + core::CancellationError::RegistryClosed => Self::RegistryClosed, + core::CancellationError::LeaseReleased => Self::LeaseReleased, + } + } +} + +#[derive(uniffi::Object)] +pub struct CancellationRegistry { + inner: core::CancellationRegistry, +} + +#[uniffi::export] +impl CancellationRegistry { + #[uniffi::constructor] + pub fn new(domain: CancellationDomain) -> Self { + let domain = match domain { + CancellationDomain::Download => core::CancellationDomain::Download, + CancellationDomain::ExtensionRequest => core::CancellationDomain::ExtensionRequest, + }; + Self { + inner: core::CancellationRegistry::new(domain), + } + } + + pub fn acquire(&self, id: String) -> Result<Arc<RequestLease>, CancellationError> { + Ok(Arc::new(RequestLease { + inner: Arc::new(self.inner.acquire(&id)?), + })) + } + + pub fn cancel(&self, id: String) -> Result<(), CancellationError> { + self.inner.cancel(&id).map_err(Into::into) + } + + pub fn cancel_active(&self) -> Result<Vec<String>, CancellationError> { + self.inner.cancel_active().map_err(Into::into) + } + + pub fn is_cancelled(&self, id: String) -> Result<bool, CancellationError> { + self.inner.is_cancelled(&id).map_err(Into::into) + } + + pub fn reset_if_idle(&self, id: String) -> Result<(), CancellationError> { + self.inner.reset_if_idle(&id).map_err(Into::into) + } + + pub fn shutdown(&self) { + self.inner.shutdown(); + } +} + +#[derive(uniffi::Object)] +pub struct RequestLease { + pub(crate) inner: Arc<core::RequestLease>, +} + +#[uniffi::export] +impl RequestLease { + pub fn is_cancelled(&self) -> Result<bool, CancellationError> { + self.inner.is_cancelled().map_err(Into::into) + } + + pub fn check_active(&self) -> Result<(), CancellationError> { + self.inner.check_active().map_err(Into::into) + } + + pub fn wait_cancelled(&self, timeout_ms: i64) -> Result<bool, CancellationError> { + self.inner.wait_cancelled(timeout_ms).map_err(Into::into) + } + + pub fn release(&self) { + self.inner.release(); + } +} diff --git a/rust_backend/crates/mobile/src/extensions.rs b/rust_backend/crates/mobile/src/extensions.rs new file mode 100644 index 00000000..490663c7 --- /dev/null +++ b/rust_backend/crates/mobile/src/extensions.rs @@ -0,0 +1,395 @@ +use crate::cancellation::RequestLease; +use spotiflac_extensions::environment::{ + EnvironmentError, ExtensionEnvironment as CoreEnvironment, +}; +use spotiflac_extensions::{ExtensionError, ExtensionRuntime, RuntimeLimits}; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +/// Keeps operation directories accessible until native finalization finishes. +/// Explicit close and object destruction both release only these grants. +#[derive(uniffi::Object)] +pub struct DownloadDirectoryScope { + grants: Mutex<Option<Vec<spotiflac_extensions::files::TemporaryGrant>>>, +} + +#[uniffi::export] +impl DownloadDirectoryScope { + pub fn release(&self) { + self.grants + .lock() + .expect("download directory scope lock") + .take(); + } +} + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum JsExtensionError { + #[error("extension runtime closed")] + Closed, + #[error("extension runtime busy")] + Busy, + #[error("execution timeout exceeded")] + Timeout, + #[error("{0}")] + Cancelled(String), + #[error("extension did not call registerExtension()")] + NotRegistered, + #[error("extension function not found: {0}")] + MissingFunction(String), + #[error("invalid extension input: {0}")] + InvalidInput(String), + #[error("{0}")] + Script(String), + #[error("extension environment closed")] + EnvironmentClosed, + #[error("{0}")] + Storage(String), + #[error("{0}")] + Manifest(String), + #[error("{0}")] + Gate(String), + #[error("{0}")] + Network(String), + #[error("{0}")] + Auth(String), + #[error("{0}")] + Index(String), +} + +impl From<EnvironmentError> for JsExtensionError { + fn from(error: EnvironmentError) -> Self { + match error { + EnvironmentError::Closed => Self::EnvironmentClosed, + EnvironmentError::Runtime(error) => error.into(), + EnvironmentError::Storage(error) => Self::Storage(error.to_string()), + EnvironmentError::Manifest(error) => Self::Manifest(error.to_string()), + EnvironmentError::Gate(message) => Self::Gate(message), + EnvironmentError::Network(error) => Self::Network(error.to_string()), + EnvironmentError::Auth(error) => Self::Auth(error), + EnvironmentError::Index(error) => Self::Index(error), + } + } +} + +impl From<ExtensionError> for JsExtensionError { + fn from(error: ExtensionError) -> Self { + match error { + ExtensionError::Closed => Self::Closed, + ExtensionError::Busy => Self::Busy, + ExtensionError::Timeout => Self::Timeout, + ExtensionError::Cancelled(error) => Self::Cancelled(error.to_string()), + ExtensionError::NotRegistered => Self::NotRegistered, + ExtensionError::MissingFunction(method) => Self::MissingFunction(method), + ExtensionError::InvalidInput(message) => Self::InvalidInput(message), + ExtensionError::Script(message) => Self::Script(message), + } + } +} + +/// Migration runtime. Calls must run off the UI thread. The complete extension +/// manager and network/file/auth hosts are not yet wired into the mobile app. +#[derive(uniffi::Object)] +pub struct JsExtension { + inner: Arc<ExtensionRuntime>, +} + +#[uniffi::export] +impl JsExtension { + pub fn download_state(&self) -> Arc<crate::progress::DownloadState> { + Arc::new(crate::progress::DownloadState { + inner: self.inner.download_state(), + }) + } + + #[uniffi::constructor] + pub fn new( + source: String, + settings_json: String, + timeout_ms: u64, + ) -> Result<Self, JsExtensionError> { + let limits = RuntimeLimits { + timeout_ms: if timeout_ms == 0 { 30_000 } else { timeout_ms }, + ..RuntimeLimits::default() + }; + Ok(Self { + inner: Arc::new(ExtensionRuntime::load(&source, &settings_json, limits)?), + }) + } + + pub fn call( + &self, + method: String, + arguments_json: String, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + ) -> Result<String, JsExtensionError> { + let lease = lease.map(|lease| Arc::clone(&lease.inner)); + self.inner + .call(&method, &arguments_json, lease, timeout_ms) + .map_err(Into::into) + } + + pub fn preflight_signed_session( + &self, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + ) -> Result<bool, JsExtensionError> { + self.inner + .preflight_signed_session(lease.map(|lease| Arc::clone(&lease.inner)), timeout_ms) + .map_err(Into::into) + } + + pub fn call_download( + &self, + method: String, + arguments_json: String, + lease: Option<Arc<RequestLease>>, + resolution_timeout_ms: u64, + ) -> Result<String, JsExtensionError> { + self.inner + .call_download( + &method, + &arguments_json, + lease.map(|lease| Arc::clone(&lease.inner)), + resolution_timeout_ms, + ) + .map_err(Into::into) + } + + pub fn take_verification_url(&self) -> String { + self.inner.take_verification_url() + } + + pub fn call_download_for_item( + &self, + method: String, + arguments_json: String, + item_id: String, + resolution_timeout_ms: u64, + ) -> Result<String, JsExtensionError> { + self.inner + .call_download_for_item(&method, &arguments_json, &item_id, resolution_timeout_ms) + .map_err(Into::into) + } + + pub fn shutdown(&self) { + self.inner.shutdown(); + } +} + +#[derive(uniffi::Object)] +pub struct ExtensionEnvironment { + pub(crate) inner: Arc<CoreEnvironment>, +} + +#[uniffi::export] +impl ExtensionEnvironment { + pub fn log_buffer(&self) -> Arc<crate::logging::LogBuffer> { + Arc::new(crate::logging::LogBuffer { + inner: self.inner.log_buffer(), + }) + } + + pub fn ffmpeg_commands(&self) -> Arc<crate::ffmpeg::FfmpegCommands> { + Arc::new(crate::ffmpeg::FfmpegCommands { + inner: self.inner.ffmpeg_commands(), + }) + } + + pub fn download_state(&self) -> Arc<crate::progress::DownloadState> { + Arc::new(crate::progress::DownloadState { + inner: self.inner.download_state(), + }) + } + + #[uniffi::constructor] + pub fn new( + data_directory: String, + master_key: String, + app_version: String, + ) -> Result<Self, JsExtensionError> { + let master_key = zeroize::Zeroizing::new(master_key); + Ok(Self { + inner: Arc::new(CoreEnvironment::new( + Path::new(&data_directory), + &master_key, + &app_version, + )?), + }) + } + + pub fn load( + &self, + manifest_json: String, + source: String, + timeout_ms: u64, + ) -> Result<Arc<JsExtension>, JsExtensionError> { + let limits = RuntimeLimits { + timeout_ms: if timeout_ms == 0 { 30_000 } else { timeout_ms }, + ..RuntimeLimits::default() + }; + Ok(Arc::new(JsExtension { + inner: self.inner.load(&manifest_json, &source, limits)?, + })) + } + + pub fn settings(&self, extension_id: String) -> Result<String, JsExtensionError> { + Ok(serde_json::Value::Object(self.inner.settings(&extension_id)?).to_string()) + } + + pub fn set_setting( + &self, + extension_id: String, + key: String, + value_json: String, + ) -> Result<(), JsExtensionError> { + let value = serde_json::from_str(&value_json) + .map_err(|error| JsExtensionError::InvalidInput(error.to_string()))?; + self.inner + .set_setting(&extension_id, &key, value) + .map_err(Into::into) + } + + pub fn remove_setting( + &self, + extension_id: String, + key: String, + ) -> Result<(), JsExtensionError> { + self.inner + .remove_setting(&extension_id, &key) + .map_err(Into::into) + } + + pub fn set_allow_private_network(&self, allow: bool) -> Result<(), JsExtensionError> { + self.inner + .set_allow_private_network(allow) + .map_err(Into::into) + } + + pub fn set_network_compatibility_options( + &self, + allow_http: bool, + insecure_tls: bool, + ) -> Result<(), JsExtensionError> { + self.inner + .set_network_compatibility_options(allow_http, insecure_tls) + .map_err(Into::into) + } + + pub fn cleanup_connections(&self) -> Result<(), JsExtensionError> { + self.inner.cleanup_connections().map_err(Into::into) + } + + pub fn set_allowed_download_directories( + &self, + directories: Vec<String>, + ) -> Result<(), JsExtensionError> { + self.inner + .set_allowed_download_directories( + &directories + .into_iter() + .map(std::path::PathBuf::from) + .collect::<Vec<_>>(), + ) + .map_err(Into::into) + } + + pub fn grant_download_directories( + &self, + directories: Vec<String>, + ) -> Result<Arc<DownloadDirectoryScope>, JsExtensionError> { + let grants = directories + .iter() + .map(|path| { + self.inner + .grant_temporary_download_directory(Path::new(path)) + }) + .collect::<Result<Vec<_>, _>>() + .map_err(JsExtensionError::Storage)?; + Ok(Arc::new(DownloadDirectoryScope { + grants: Mutex::new(Some(grants)), + })) + } + + pub fn pending_auth(&self, extension_id: String) -> Result<String, JsExtensionError> { + Ok(self + .inner + .pending_auth(&extension_id)? + .map(|pending| serde_json::to_string(&pending).expect("pending auth JSON")) + .unwrap_or_default()) + } + + pub fn set_runtime_state(&self, raw: String) -> Result<(), JsExtensionError> { + self.inner.set_runtime_state(&raw).map_err(Into::into) + } + + pub fn set_session_grant( + &self, + extension_id: String, + grant: String, + ) -> Result<(), JsExtensionError> { + let grant = zeroize::Zeroizing::new(grant); + self.inner + .set_session_grant(&extension_id, &grant) + .map_err(Into::into) + } + + pub fn all_pending_auth(&self) -> Result<String, JsExtensionError> { + Ok(serde_json::to_string(&self.inner.all_pending_auth()?).expect("pending auth JSON")) + } + + pub fn clear_pending_auth(&self, extension_id: String) -> Result<(), JsExtensionError> { + self.inner + .clear_pending_auth(&extension_id) + .map_err(Into::into) + } + + pub fn resolve_callback_state(&self, state: String) -> Result<String, JsExtensionError> { + self.inner + .resolve_callback_state(&state, false) + .map_err(Into::into) + } + + pub fn consume_callback_state(&self, state: String) -> Result<String, JsExtensionError> { + self.inner + .resolve_callback_state(&state, true) + .map_err(Into::into) + } + + pub fn set_auth_code( + &self, + extension_id: String, + code: String, + ) -> Result<(), JsExtensionError> { + let code = zeroize::Zeroizing::new(code); + self.inner + .set_auth_code(&extension_id, &code) + .map_err(Into::into) + } + + pub fn set_auth_tokens( + &self, + extension_id: String, + access_token: String, + refresh_token: String, + expires_in: i64, + ) -> Result<(), JsExtensionError> { + let access_token = zeroize::Zeroizing::new(access_token); + let refresh_token = zeroize::Zeroizing::new(refresh_token); + self.inner + .set_auth_tokens(&extension_id, &access_token, &refresh_token, expires_in) + .map_err(Into::into) + } + + pub fn is_authenticated(&self, extension_id: String) -> Result<bool, JsExtensionError> { + self.inner + .is_authenticated(&extension_id) + .map_err(Into::into) + } + + pub fn shutdown(&self) { + self.inner.shutdown(); + } +} diff --git a/rust_backend/crates/mobile/src/ffmpeg.rs b/rust_backend/crates/mobile/src/ffmpeg.rs new file mode 100644 index 00000000..1733de20 --- /dev/null +++ b/rust_backend/crates/mobile/src/ffmpeg.rs @@ -0,0 +1,64 @@ +use spotiflac_extensions::ffmpeg::{CommandRegistry, CommandResult, RegistryClosed}; +use std::sync::Arc; + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum FfmpegError { + #[error("FFmpeg command registry closed")] + Closed, +} + +impl From<RegistryClosed> for FfmpegError { + fn from(_: RegistryClosed) -> Self { + Self::Closed + } +} + +/// All handles returned by one environment refer to its single command queue. +#[derive(uniffi::Object)] +pub struct FfmpegCommands { + pub(crate) inner: Arc<CommandRegistry>, +} + +#[uniffi::export] +impl FfmpegCommands { + pub fn pending(&self) -> Result<String, FfmpegError> { + self.inner.pending_json().map_err(Into::into) + } + + pub fn wait_pending(&self, timeout_ms: i64) -> Result<String, FfmpegError> { + self.inner.wait_pending_json(timeout_ms).map_err(Into::into) + } + + pub fn get_command(&self, command_id: String) -> Result<String, FfmpegError> { + Ok(self + .inner + .get(&command_id)? + .map_or_else(String::new, |command| { + serde_json::to_string(&command).expect("FFmpeg command JSON") + })) + } + + pub fn complete( + &self, + command_id: String, + success: bool, + output: String, + error: String, + ) -> Result<bool, FfmpegError> { + self.inner + .complete( + &command_id, + CommandResult { + success, + output, + error, + }, + ) + .map_err(Into::into) + } + + pub fn shutdown(&self) { + self.inner.shutdown(); + } +} diff --git a/rust_backend/crates/mobile/src/filename.rs b/rust_backend/crates/mobile/src/filename.rs new file mode 100644 index 00000000..40db7cd5 --- /dev/null +++ b/rust_backend/crates/mobile/src/filename.rs @@ -0,0 +1,12 @@ +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum FilenameError { + #[error("{message}")] + Invalid { message: String }, +} + +#[uniffi::export] +pub fn build_filename(template: String, metadata_json: String) -> Result<String, FilenameError> { + spotiflac_core::filename::build_filename_json(&template, &metadata_json) + .map_err(|message| FilenameError::Invalid { message }) +} diff --git a/rust_backend/crates/mobile/src/index.rs b/rust_backend/crates/mobile/src/index.rs new file mode 100644 index 00000000..e25bd19d --- /dev/null +++ b/rust_backend/crates/mobile/src/index.rs @@ -0,0 +1,69 @@ +use crate::cancellation::RequestLease; +use crate::extensions::{ExtensionEnvironment, JsExtensionError}; +use std::sync::Arc; + +fn check(lease: &Option<Arc<RequestLease>>) -> Result<(), String> { + lease.as_ref().map_or(Ok(()), |lease| { + lease + .inner + .check_active() + .map_err(|error| error.to_string()) + }) +} + +#[uniffi::export] +impl ExtensionEnvironment { + pub fn prebuild_isrc_index( + &self, + directory: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<(), JsExtensionError> { + self.inner + .prebuild_isrc_index(&directory, &|| check(&lease)) + .map_err(Into::into) + } + + pub fn check_isrc_exists( + &self, + directory: String, + isrc: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, JsExtensionError> { + self.inner + .check_isrc_exists(&directory, &isrc, &|| check(&lease)) + .map_err(Into::into) + } + + pub fn add_to_isrc_index( + &self, + directory: String, + isrc: String, + path: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<(), JsExtensionError> { + self.inner + .add_to_isrc_index(&directory, &isrc, &path, &|| check(&lease)) + .map_err(Into::into) + } + + pub fn check_files_exist_parallel( + &self, + directory: String, + tracks_json: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, JsExtensionError> { + self.inner + .check_files_exist_parallel(&directory, &tracks_json, &|| check(&lease)) + .map_err(Into::into) + } + + pub fn invalidate_isrc_cache(&self, directory: String) -> Result<(), JsExtensionError> { + self.inner + .invalidate_isrc_cache(&directory) + .map_err(Into::into) + } + + pub fn check_file_exists(&self, path: String) -> Result<bool, JsExtensionError> { + self.inner.check_file_exists(&path).map_err(Into::into) + } +} diff --git a/rust_backend/crates/mobile/src/lib.rs b/rust_backend/crates/mobile/src/lib.rs new file mode 100644 index 00000000..96272226 --- /dev/null +++ b/rust_backend/crates/mobile/src/lib.rs @@ -0,0 +1,21 @@ +//! Migration APIs. Instances here are not connected to the app's Go-owned work. + +mod cancellation; +mod extensions; +mod ffmpeg; +mod filename; +mod index; +mod logging; +mod lyrics; +mod manager; +mod metadata; +mod progress; +mod repository; +mod tags; + +uniffi::setup_scaffolding!(); + +#[uniffi::export] +pub fn sanitize_filename(filename: String) -> String { + spotiflac_core::filename::sanitize_filename(&filename) +} diff --git a/rust_backend/crates/mobile/src/logging.rs b/rust_backend/crates/mobile/src/logging.rs new file mode 100644 index 00000000..539cd95c --- /dev/null +++ b/rust_backend/crates/mobile/src/logging.rs @@ -0,0 +1,64 @@ +use spotiflac_extensions::logging::{LogBuffer as CoreLogBuffer, LogClosed}; +use std::sync::Arc; + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum LogError { + #[error("log buffer closed")] + Closed, +} + +impl From<LogClosed> for LogError { + fn from(_: LogClosed) -> Self { + Self::Closed + } +} + +#[derive(uniffi::Object)] +pub struct LogBuffer { + pub(crate) inner: Arc<CoreLogBuffer>, +} + +#[uniffi::export] +impl LogBuffer { + #[uniffi::constructor] + pub fn new() -> Self { + Self { + inner: Arc::new(CoreLogBuffer::default()), + } + } + + pub fn set_enabled(&self, enabled: bool) -> Result<(), LogError> { + self.inner.set_enabled(enabled).map_err(Into::into) + } + pub fn is_enabled(&self) -> Result<bool, LogError> { + self.inner.is_enabled().map_err(Into::into) + } + pub fn add(&self, level: String, tag: String, message: String) -> Result<(), LogError> { + self.inner.add(&level, &tag, &message).map_err(Into::into) + } + pub fn backend(&self, message: String) -> Result<(), LogError> { + self.inner.backend(&message).map_err(Into::into) + } + pub fn all(&self) -> Result<String, LogError> { + self.inner.all().map_err(Into::into) + } + pub fn since(&self, index: i64) -> Result<String, LogError> { + self.inner.since(index).map_err(Into::into) + } + pub fn clear(&self) -> Result<(), LogError> { + self.inner.clear().map_err(Into::into) + } + pub fn count(&self) -> Result<u64, LogError> { + self.inner.count().map_err(Into::into) + } + pub fn shutdown(&self) { + self.inner.shutdown(); + } +} + +impl Default for LogBuffer { + fn default() -> Self { + Self::new() + } +} diff --git a/rust_backend/crates/mobile/src/lyrics.rs b/rust_backend/crates/mobile/src/lyrics.rs new file mode 100644 index 00000000..e3260526 --- /dev/null +++ b/rust_backend/crates/mobile/src/lyrics.rs @@ -0,0 +1,117 @@ +use crate::cancellation::RequestLease; +use crate::manager::{ExtensionManager, ExtensionManagerError}; +use spotiflac_extensions::backend::LyricsRequest as CoreRequest; +use std::sync::Arc; + +#[derive(uniffi::Record)] +pub struct LyricsRequest { + pub spotify_id: String, + pub track: String, + pub artist: String, + pub file_path: String, + pub duration_ms: i64, +} + +impl From<LyricsRequest> for CoreRequest { + fn from(request: LyricsRequest) -> Self { + Self { + spotify_id: request.spotify_id, + track: request.track, + artist: request.artist, + file_path: request.file_path, + duration_ms: request.duration_ms, + } + } +} + +fn check(lease: &Option<Arc<RequestLease>>) -> Result<(), String> { + lease.as_ref().map_or(Ok(()), |lease| { + lease + .inner + .check_active() + .map_err(|error| error.to_string()) + }) +} + +/// Lyrics use the installed manager's root/cache. Native file access shares +/// environment grants; the caller retains platform access through this call. +#[uniffi::export] +impl ExtensionManager { + pub fn embed_lyrics_to_file( + &self, + path: String, + lyrics: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .embed_lyrics_to_file(&path, &lyrics, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_lyrics_lrc( + &self, + request: LyricsRequest, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_lyrics_lrc(&request.into(), &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_lyrics_lrc_with_source( + &self, + request: LyricsRequest, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_lyrics_lrc_with_source(&request.into(), &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn fetch_and_save_lyrics( + &self, + request: LyricsRequest, + output_path: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<(), ExtensionManagerError> { + self.inner + .fetch_and_save_lyrics(&request.into(), &output_path, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_lyrics_providers_json(&self) -> Result<String, ExtensionManagerError> { + self.inner + .get_lyrics_providers_json() + .map_err(ExtensionManagerError::Operation) + } + + pub fn set_lyrics_providers_json( + &self, + providers_json: String, + ) -> Result<(), ExtensionManagerError> { + self.inner + .set_lyrics_providers_json(&providers_json) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_available_lyrics_providers_json(&self) -> Result<String, ExtensionManagerError> { + self.inner + .get_available_lyrics_providers_json() + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_lyrics_fetch_options_json(&self) -> Result<String, ExtensionManagerError> { + self.inner + .get_lyrics_fetch_options_json() + .map_err(ExtensionManagerError::Operation) + } + + pub fn set_lyrics_fetch_options_json( + &self, + options_json: String, + ) -> Result<(), ExtensionManagerError> { + self.inner + .set_lyrics_fetch_options_json(&options_json) + .map_err(ExtensionManagerError::Operation) + } +} diff --git a/rust_backend/crates/mobile/src/manager.rs b/rust_backend/crates/mobile/src/manager.rs new file mode 100644 index 00000000..8d133473 --- /dev/null +++ b/rust_backend/crates/mobile/src/manager.rs @@ -0,0 +1,414 @@ +use crate::cancellation::RequestLease; +use crate::extensions::ExtensionEnvironment; +use spotiflac_extensions::RuntimeLimits; +use spotiflac_extensions::backend::Backend as CoreManager; +use spotiflac_extensions::manager::ManagerError; +use std::path::Path; +use std::sync::Arc; + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum ExtensionManagerError { + #[error("{0}")] + Operation(String), +} + +impl From<ManagerError> for ExtensionManagerError { + fn from(error: ManagerError) -> Self { + Self::Operation(error.to_string()) + } +} + +/// Owns installed source, data, and JavaScript workers. Use a background native +/// thread for operations that read packages or run extension code. +#[derive(uniffi::Object)] +pub struct ExtensionManager { + pub(crate) inner: CoreManager, +} + +#[uniffi::export] +impl ExtensionManager { + #[uniffi::constructor] + pub fn new( + source_directory: String, + data_directory: String, + master_key: String, + app_version: String, + timeout_ms: u64, + ) -> Result<Self, ExtensionManagerError> { + Self::with_lyrics_settings( + source_directory, + data_directory, + master_key, + app_version, + timeout_ms, + "[]".into(), + "{}".into(), + ) + } + + #[uniffi::constructor] + pub fn with_lyrics_settings( + source_directory: String, + data_directory: String, + master_key: String, + app_version: String, + timeout_ms: u64, + providers_json: String, + options_json: String, + ) -> Result<Self, ExtensionManagerError> { + let key = zeroize::Zeroizing::new(master_key); + let limits = RuntimeLimits { + timeout_ms: if timeout_ms == 0 { 30_000 } else { timeout_ms }, + ..RuntimeLimits::default() + }; + Ok(Self { + inner: CoreManager::with_lyrics_settings( + Path::new(&source_directory), + Path::new(&data_directory), + &key, + &app_version, + limits, + &providers_json, + &options_json, + )?, + }) + } + + pub fn environment(&self) -> Arc<ExtensionEnvironment> { + Arc::new(ExtensionEnvironment { + inner: self.inner.environment(), + }) + } + + pub fn get_app_version(&self) -> Result<String, ExtensionManagerError> { + self.inner + .get_app_version() + .map_err(ExtensionManagerError::Operation) + } + + pub fn set_app_version(&self, version: String) -> Result<(), ExtensionManagerError> { + self.inner + .set_app_version(&version) + .map_err(ExtensionManagerError::Operation) + } + + pub fn install(&self, package_path: String) -> Result<String, ExtensionManagerError> { + self.inner + .install(Path::new(&package_path)) + .map_err(Into::into) + } + + pub fn upgrade(&self, package_path: String) -> Result<String, ExtensionManagerError> { + self.inner + .upgrade(Path::new(&package_path)) + .map_err(Into::into) + } + + pub fn check_upgrade(&self, package_path: String) -> Result<String, ExtensionManagerError> { + self.inner + .check_upgrade(Path::new(&package_path)) + .map_err(Into::into) + } + + pub fn installed(&self) -> Result<String, ExtensionManagerError> { + self.inner.installed().map_err(Into::into) + } + + pub fn load_all(&self) -> Result<String, ExtensionManagerError> { + self.inner.load_all().map_err(Into::into) + } + + pub fn set_enabled( + &self, + extension_id: String, + enabled: bool, + ) -> Result<(), ExtensionManagerError> { + self.inner + .set_enabled(&extension_id, enabled) + .map_err(Into::into) + } + + pub fn initialize( + &self, + extension_id: String, + settings_json: String, + ) -> Result<(), ExtensionManagerError> { + let settings = serde_json::from_str(&settings_json).map_err(|error| { + ExtensionManagerError::Operation(format!("invalid settings: {error}")) + })?; + self.inner + .initialize(&extension_id, settings) + .map_err(Into::into) + } + + pub fn cleanup(&self, extension_id: String) -> Result<(), ExtensionManagerError> { + self.inner.cleanup(&extension_id).map_err(Into::into) + } + + pub fn update_settings( + &self, + extension_id: String, + settings_json: String, + ) -> Result<(), ExtensionManagerError> { + let settings = serde_json::from_str(&settings_json).map_err(|error| { + ExtensionManagerError::Operation(format!("invalid settings: {error}")) + })?; + self.inner + .update_settings(&extension_id, settings) + .map_err(Into::into) + } + + pub fn download_by_strategy( + &self, + request_json: String, + ) -> Result<String, ExtensionManagerError> { + self.inner + .download_by_strategy(&request_json, &|| Ok(())) + .map_err(ExtensionManagerError::Operation) + } + + pub fn download_with_extensions_json( + &self, + request_json: String, + ) -> Result<String, ExtensionManagerError> { + self.inner + .download_with_extensions_json(&request_json, &|| Ok(())) + .map_err(ExtensionManagerError::Operation) + } + + pub fn check_extension_health_json( + &self, + extension_id: String, + ) -> Result<String, ExtensionManagerError> { + self.inner + .check_extension_health_json(&extension_id, &|| Ok(())) + .map_err(ExtensionManagerError::Operation) + } + + pub fn download( + &self, + extension_id: String, + request_json: String, + resolution_timeout_ms: u64, + ) -> Result<String, ExtensionManagerError> { + let request = serde_json::from_str(&request_json).map_err(|error| { + ExtensionManagerError::Operation(format!("invalid download request: {error}")) + })?; + self.inner + .download(&extension_id, request, resolution_timeout_ms) + .map_err(Into::into) + } + + pub fn release_idle_download_runtimes(&self) -> Result<u64, ExtensionManagerError> { + self.inner + .release_idle_download_runtimes() + .map_err(Into::into) + } + + pub fn release_memory(&self, under_pressure: bool) -> Result<(), ExtensionManagerError> { + self.inner + .release_memory(under_pressure) + .map_err(ExtensionManagerError::Operation) + } + + pub fn post_process( + &self, + extension_id: String, + input_json: String, + metadata_json: String, + hook_id: String, + timeout_ms: u64, + ) -> Result<String, ExtensionManagerError> { + let input = serde_json::from_str(&input_json).map_err(|error| { + ExtensionManagerError::Operation(format!("invalid post-process input: {error}")) + })?; + let metadata = serde_json::from_str(&metadata_json).map_err(|error| { + ExtensionManagerError::Operation(format!("invalid post-process metadata: {error}")) + })?; + self.inner + .post_process(&extension_id, input, metadata, &hook_id, timeout_ms) + .map_err(Into::into) + } + + pub fn run_post_processing( + &self, + input_json: String, + metadata_json: String, + timeout_ms: u64, + ) -> Result<String, ExtensionManagerError> { + // Preserve the permissive Go JSON wrapper used by PlatformBridge. + self.inner + .run_post_processing( + serde_json::from_str(&input_json).unwrap_or_default(), + serde_json::from_str(&metadata_json).unwrap_or_default(), + timeout_ms, + ) + .map_err(Into::into) + } + + pub fn check_availability( + &self, + extension_id: String, + request_json: String, + timeout_ms: u64, + ) -> Result<String, ExtensionManagerError> { + let request = serde_json::from_str(&request_json).map_err(|error| { + ExtensionManagerError::Operation(format!("invalid availability request: {error}")) + })?; + self.inner + .check_availability(&extension_id, request, timeout_ms) + .map_err(Into::into) + } + + pub fn enrich_track( + &self, + extension_id: String, + track_json: String, + item_id: String, + timeout_ms: u64, + ) -> Result<String, ExtensionManagerError> { + self.inner + .enrich_track(&extension_id, &track_json, &item_id, timeout_ms) + .map_err(Into::into) + } + + pub fn search_metadata_provider( + &self, + extension_id: String, + query: String, + limit: i64, + timeout_ms: u64, + ) -> Result<String, ExtensionManagerError> { + self.inner + .search_metadata_provider(&extension_id, &query, limit as isize, timeout_ms) + .map_err(Into::into) + } + + pub fn search_metadata_providers( + &self, + query: String, + limit: i64, + include_extensions: bool, + item_id: String, + timeout_ms: u64, + ) -> Result<String, ExtensionManagerError> { + self.inner + .search_metadata_providers( + &query, + limit as isize, + include_extensions, + &item_id, + timeout_ms, + ) + .map_err(Into::into) + } + + pub fn invoke_action( + &self, + extension_id: String, + action: String, + ) -> Result<String, ExtensionManagerError> { + self.inner + .invoke_action(&extension_id, &action) + .map_err(Into::into) + } + + pub fn call( + &self, + extension_id: String, + method: String, + arguments_json: String, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + ) -> Result<String, ExtensionManagerError> { + self.inner + .call( + &extension_id, + &method, + &arguments_json, + lease.map(|lease| Arc::clone(&lease.inner)), + timeout_ms, + ) + .map_err(Into::into) + } + + pub fn provider_ids(&self, kind: String) -> Result<Vec<String>, ExtensionManagerError> { + self.inner.provider_ids(&kind).map_err(Into::into) + } + + pub fn provider_call( + &self, + extension_id: String, + method: String, + arguments_json: String, + lease: Option<Arc<RequestLease>>, + timeout_ms: u64, + ) -> Result<String, ExtensionManagerError> { + self.inner + .provider_call( + &extension_id, + &method, + &arguments_json, + lease.map(|lease| Arc::clone(&lease.inner)), + timeout_ms, + ) + .map_err(Into::into) + } + + pub fn set_provider_priority( + &self, + kind: String, + ids: Vec<String>, + ) -> Result<(), ExtensionManagerError> { + self.inner + .set_provider_priority(&kind, ids) + .map_err(Into::into) + } + + pub fn set_fallback_providers( + &self, + ids: Option<Vec<String>>, + ) -> Result<(), ExtensionManagerError> { + self.inner.set_fallback_providers(ids).map_err(Into::into) + } + + pub fn provider_priorities(&self) -> Result<String, ExtensionManagerError> { + self.inner.provider_priorities().map_err(Into::into) + } + + pub fn fallback_allowed(&self, extension_id: String) -> Result<bool, ExtensionManagerError> { + self.inner + .fallback_allowed(&extension_id) + .map_err(Into::into) + } + + pub fn find_url_handler(&self, url: String) -> Result<Option<String>, ExtensionManagerError> { + self.inner.find_url_handler(&url).map_err(Into::into) + } + + pub fn get_extension_pending_auth_json( + &self, + extension_id: String, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_extension_pending_auth_json(&extension_id) + .map_err(ExtensionManagerError::Operation) + } + + pub fn unload(&self, extension_id: String) -> Result<(), ExtensionManagerError> { + self.inner.unload(&extension_id).map_err(Into::into) + } + + pub fn unload_all(&self) -> Result<(), ExtensionManagerError> { + self.inner.unload_all().map_err(Into::into) + } + + pub fn remove(&self, extension_id: String) -> Result<(), ExtensionManagerError> { + self.inner.remove(&extension_id).map_err(Into::into) + } + + pub fn shutdown(&self) { + self.inner.shutdown(); + } +} diff --git a/rust_backend/crates/mobile/src/metadata.rs b/rust_backend/crates/mobile/src/metadata.rs new file mode 100644 index 00000000..605cbaa8 --- /dev/null +++ b/rust_backend/crates/mobile/src/metadata.rs @@ -0,0 +1,393 @@ +use crate::cancellation::RequestLease; +use crate::manager::{ExtensionManager, ExtensionManagerError}; +use std::sync::Arc; + +#[derive(uniffi::Record)] +pub struct DeezerResource { + pub resource_type: String, + pub resource_id: String, +} + +fn check(lease: &Option<Arc<RequestLease>>) -> Result<(), String> { + lease.as_ref().map_or(Ok(()), |lease| { + lease + .inner + .check_active() + .map_err(|error| error.to_string()) + }) +} + +/// Metadata, platform resolution and installed extensions share one root. +/// Run network operations on a native background thread and retain any lease. +#[uniffi::export] +impl ExtensionManager { + pub fn enrich_track_json( + &self, + extension_id: String, + track_json: String, + ) -> Result<String, ExtensionManagerError> { + self.inner + .enrich_track_json(&extension_id, &track_json) + .map_err(ExtensionManagerError::Operation) + } + + pub fn handle_url_json(&self, url: String) -> Result<String, ExtensionManagerError> { + self.inner + .handle_url_json(&url) + .map_err(ExtensionManagerError::Operation) + } + + pub fn custom_search_json( + &self, + extension_id: String, + query: String, + options_json: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .custom_search_json( + &extension_id, + &query, + &options_json, + lease.map(|lease| Arc::clone(&lease.inner)), + ) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_extension_home_feed_json( + &self, + extension_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_extension_home_feed_json( + &extension_id, + lease.map(|lease| Arc::clone(&lease.inner)), + ) + .map_err(ExtensionManagerError::Operation) + } + + pub fn find_collection_across_extensions_json( + &self, + request_json: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .find_collection_across_extensions_json(&request_json, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn fetch_music_brainz_genre_by_isrc( + &self, + isrc: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .fetch_music_brainz_genre_by_isrc(&isrc, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn fetch_music_brainz_album_artist_by_isrc( + &self, + isrc: String, + album_name: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .fetch_music_brainz_album_artist_by_isrc(&isrc, &album_name, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_provider_metadata_json( + &self, + provider_id: String, + resource_type: String, + resource_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_provider_metadata_json(&provider_id, &resource_type, &resource_id, &|| { + check(&lease) + }) + .map_err(ExtensionManagerError::Operation) + } + + pub fn set_metadata_language(&self, tag: String) -> Result<(), ExtensionManagerError> { + self.inner + .set_metadata_language(&tag) + .map_err(ExtensionManagerError::Operation) + } + + pub fn set_song_link_region(&self, region: String) -> Result<(), ExtensionManagerError> { + self.inner + .set_song_link_region(®ion) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_song_link_region(&self) -> Result<String, ExtensionManagerError> { + self.inner + .get_song_link_region() + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_track_cache_size(&self) -> Result<u64, ExtensionManagerError> { + self.inner + .get_track_cache_size() + .map_err(ExtensionManagerError::Operation) + } + + pub fn clear_track_id_cache(&self) -> Result<(), ExtensionManagerError> { + self.inner + .clear_track_id_cache() + .map_err(ExtensionManagerError::Operation) + } + + pub fn parse_deezer_url(&self, url: String) -> Result<DeezerResource, ExtensionManagerError> { + let (resource_type, resource_id) = self + .inner + .parse_deezer_url(&url) + .map_err(ExtensionManagerError::Operation)?; + Ok(DeezerResource { + resource_type, + resource_id, + }) + } + + pub fn search_deezer( + &self, + query: String, + track_limit: i64, + artist_limit: i64, + filter: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .search_deezer( + &query, + track_limit as isize, + artist_limit as isize, + &filter, + &|| check(&lease), + ) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_deezer_metadata( + &self, + resource_type: String, + resource_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_deezer_metadata(&resource_type, &resource_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_deezer_extended_metadata( + &self, + track_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_deezer_extended_metadata(&track_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn search_deezer_by_isrc( + &self, + isrc: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .search_deezer_by_isrc(&isrc, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn search_deezer_by_isrc_for_item_id( + &self, + isrc: String, + item_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .search_deezer_by_isrc_for_item_id(&isrc, &item_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn convert_spotify_to_deezer( + &self, + resource_type: String, + spotify_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .convert_spotify_to_deezer(&resource_type, &spotify_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_track_platform_links_json( + &self, + spotify_id: String, + isrc: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_track_platform_links_json(&spotify_id, &isrc, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn check_track_availability_json( + &self, + spotify_id: String, + isrc: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .check_track_availability_json(&spotify_id, &isrc, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn check_album_availability_json( + &self, + spotify_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .check_album_availability_json(&spotify_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn check_availability_from_deezer_json( + &self, + track_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .check_availability_from_deezer_json(&track_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn check_availability_by_platform_json( + &self, + platform: String, + resource_type: String, + resource_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .check_availability_by_platform_json(&platform, &resource_type, &resource_id, &|| { + check(&lease) + }) + .map_err(ExtensionManagerError::Operation) + } + + pub fn check_availability_from_url_json( + &self, + url: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .check_availability_from_url_json(&url, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_streaming_urls_json( + &self, + spotify_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_streaming_urls_json(&spotify_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_deezer_id_from_spotify( + &self, + spotify_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_deezer_id_from_spotify(&spotify_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_deezer_album_id_from_spotify( + &self, + spotify_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_deezer_album_id_from_spotify(&spotify_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_youtube_url_from_spotify( + &self, + spotify_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_youtube_url_from_spotify(&spotify_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_spotify_id_from_deezer_track( + &self, + track_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_spotify_id_from_deezer_track(&track_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_tidal_url_from_deezer_track( + &self, + track_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_tidal_url_from_deezer_track(&track_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_amazon_url_from_deezer_track( + &self, + track_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_amazon_url_from_deezer_track(&track_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_youtube_url_from_deezer_track( + &self, + track_id: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .get_youtube_url_from_deezer_track(&track_id, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn preview_reenrich_file( + &self, + request_json: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .preview_reenrich_file(&request_json, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } + + pub fn reenrich_file( + &self, + request_json: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .reenrich_file(&request_json, &|| check(&lease)) + .map_err(ExtensionManagerError::Operation) + } +} diff --git a/rust_backend/crates/mobile/src/progress.rs b/rust_backend/crates/mobile/src/progress.rs new file mode 100644 index 00000000..664303c2 --- /dev/null +++ b/rust_backend/crates/mobile/src/progress.rs @@ -0,0 +1,204 @@ +use crate::cancellation::{CancellationError, RequestLease}; +use spotiflac_core::downloads::DownloadState as CoreState; +use spotiflac_core::progress::{ProgressError, ProgressSubscription}; +use std::sync::Arc; + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum DownloadProgressError { + #[error("download progress registry closed")] + Closed, + #[error("download progress subscription closed")] + SubscriptionClosed, +} + +impl From<ProgressError> for DownloadProgressError { + fn from(error: ProgressError) -> Self { + match error { + ProgressError::Closed => Self::Closed, + ProgressError::SubscriptionClosed => Self::SubscriptionClosed, + } + } +} + +/// Share one instance between the native download manager and its listeners. +#[derive(uniffi::Object)] +pub struct DownloadState { + pub(crate) inner: Arc<CoreState>, +} + +#[uniffi::export] +impl DownloadState { + #[uniffi::constructor] + pub fn new() -> Self { + Self { + inner: Arc::new(CoreState::default()), + } + } + + pub fn acquire(&self, item_id: String) -> Result<Arc<RequestLease>, CancellationError> { + Ok(Arc::new(RequestLease { + inner: Arc::new(self.inner.acquire(&item_id)?), + })) + } + + pub fn cancel_download(&self, item_id: String) -> Result<(), CancellationError> { + self.inner.cancel(&item_id).map_err(Into::into) + } + + pub fn cancel_active_downloads(&self) -> Result<Vec<String>, CancellationError> { + self.inner.cancel_active().map_err(Into::into) + } + + pub fn reset_download_cancel(&self, item_id: String) -> Result<(), CancellationError> { + self.inner + .cancellation + .reset_if_idle(&item_id) + .map_err(Into::into) + } + + pub fn is_cancelled(&self, item_id: String) -> Result<bool, CancellationError> { + self.inner + .cancellation + .is_cancelled(&item_id) + .map_err(Into::into) + } + + pub fn init_item_progress(&self, item_id: String) -> Result<(), DownloadProgressError> { + self.inner.progress.start(&item_id).map_err(Into::into) + } + + pub fn clear_item_progress(&self, item_id: String) -> Result<(), DownloadProgressError> { + self.inner.progress.remove(&item_id).map_err(Into::into) + } + + pub fn clear_all_progress(&self) -> Result<(), DownloadProgressError> { + self.inner.progress.clear().map_err(Into::into) + } + + pub fn set_preparing( + &self, + item_id: String, + stage: String, + ) -> Result<(), DownloadProgressError> { + self.inner + .progress + .preparing(&item_id, &stage) + .map_err(Into::into) + } + + pub fn set_downloading(&self, item_id: String) -> Result<(), DownloadProgressError> { + self.inner + .progress + .downloading(&item_id) + .map_err(Into::into) + } + + pub fn set_total(&self, item_id: String, total: i64) -> Result<(), DownloadProgressError> { + self.inner + .progress + .set_total(&item_id, total) + .map_err(Into::into) + } + + pub fn set_received( + &self, + item_id: String, + received: i64, + ) -> Result<(), DownloadProgressError> { + self.inner + .progress + .set_received(&item_id, received) + .map_err(Into::into) + } + + pub fn set_received_with_speed( + &self, + item_id: String, + received: i64, + speed: f64, + ) -> Result<(), DownloadProgressError> { + self.inner + .progress + .set_received_with_speed(&item_id, received, speed) + .map_err(Into::into) + } + + pub fn set_progress( + &self, + item_id: String, + progress: f64, + received: i64, + total: i64, + ) -> Result<(), DownloadProgressError> { + self.inner + .progress + .set_progress(&item_id, progress, received, total) + .map_err(Into::into) + } + + pub fn set_finalizing(&self, item_id: String) -> Result<(), DownloadProgressError> { + self.inner.progress.finalizing(&item_id).map_err(Into::into) + } + + pub fn complete_item(&self, item_id: String) -> Result<(), DownloadProgressError> { + self.inner.progress.complete(&item_id).map_err(Into::into) + } + + pub fn item_progress(&self, item_id: String) -> Result<String, DownloadProgressError> { + self.inner.progress.item(&item_id).map_err(Into::into) + } + + pub fn all_progress(&self) -> Result<String, DownloadProgressError> { + self.inner.progress.snapshot().map_err(Into::into) + } + + pub fn progress_delta(&self, since: i64) -> Result<String, DownloadProgressError> { + self.inner.progress.delta(since).map_err(Into::into) + } + + pub fn wait_progress_delta( + &self, + since: i64, + timeout_ms: i64, + ) -> Result<String, DownloadProgressError> { + self.inner + .progress + .wait_delta(since, timeout_ms) + .map_err(Into::into) + } + + pub fn subscribe_progress( + &self, + ) -> Result<Arc<DownloadProgressSubscription>, DownloadProgressError> { + Ok(Arc::new(DownloadProgressSubscription { + inner: self.inner.progress.subscribe()?, + })) + } + + pub fn shutdown(&self) { + self.inner.shutdown(); + } +} + +impl Default for DownloadState { + fn default() -> Self { + Self::new() + } +} + +#[derive(uniffi::Object)] +pub struct DownloadProgressSubscription { + inner: ProgressSubscription, +} + +#[uniffi::export] +impl DownloadProgressSubscription { + pub fn wait_delta(&self, since: i64, timeout_ms: i64) -> Result<String, DownloadProgressError> { + self.inner.wait_delta(since, timeout_ms).map_err(Into::into) + } + + pub fn stop(&self) { + self.inner.close(); + } +} diff --git a/rust_backend/crates/mobile/src/repository.rs b/rust_backend/crates/mobile/src/repository.rs new file mode 100644 index 00000000..3a681067 --- /dev/null +++ b/rust_backend/crates/mobile/src/repository.rs @@ -0,0 +1,105 @@ +use crate::manager::ExtensionManager; +use spotiflac_extensions::repository::{ExtensionRepository as CoreRepository, RepositoryError}; +use std::path::Path; +use std::sync::Arc; + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum ExtensionRepositoryError { + #[error("{0}")] + Operation(String), +} + +impl From<RepositoryError> for ExtensionRepositoryError { + fn from(error: RepositoryError) -> Self { + Self::Operation(error.to_string()) + } +} + +/// Retains the manager for current installed versions and shares its network +/// policy. Registry and package IO must run on a native background thread. +#[derive(uniffi::Object)] +pub struct ExtensionRepository { + inner: CoreRepository, + manager: Arc<ExtensionManager>, +} + +#[uniffi::export] +impl ExtensionRepository { + #[uniffi::constructor] + pub fn new( + manager: Arc<ExtensionManager>, + cache_directory: String, + ) -> Result<Self, ExtensionRepositoryError> { + let inner = manager + .inner + .environment() + .repository(Path::new(&cache_directory))?; + Ok(Self { inner, manager }) + } + + pub fn registry_url(&self) -> Result<String, ExtensionRepositoryError> { + self.inner.registry_url().map_err(Into::into) + } + + pub fn set_registry_url(&self, url: String) -> Result<(), ExtensionRepositoryError> { + self.inner.set_registry_url(&url).map_err(Into::into) + } + + pub fn clear_registry_url(&self) -> Result<(), ExtensionRepositoryError> { + self.inner.clear_registry_url().map_err(Into::into) + } + + pub fn extensions(&self, force_refresh: bool) -> Result<String, ExtensionRepositoryError> { + self.search_inner(force_refresh, "", "") + } + + pub fn search( + &self, + query: String, + category: String, + ) -> Result<String, ExtensionRepositoryError> { + self.search_inner(false, &query, &category) + } + + pub fn categories(&self) -> Result<Vec<String>, ExtensionRepositoryError> { + self.inner.categories().map_err(Into::into) + } + + pub fn clear_cache(&self) -> Result<(), ExtensionRepositoryError> { + self.inner.clear_cache().map_err(Into::into) + } + + pub fn download( + &self, + extension_id: String, + destination_directory: String, + ) -> Result<String, ExtensionRepositoryError> { + self.inner + .download(&extension_id, Path::new(&destination_directory)) + .map(|path| path.to_string_lossy().into_owned()) + .map_err(Into::into) + } + + pub fn shutdown(&self) { + self.inner.shutdown(); + } +} + +impl ExtensionRepository { + fn search_inner( + &self, + force: bool, + query: &str, + category: &str, + ) -> Result<String, ExtensionRepositoryError> { + let installed = self + .manager + .inner + .installed_versions() + .map_err(|e| ExtensionRepositoryError::Operation(e.to_string()))?; + self.inner + .extensions(force, &installed, query, category) + .map_err(Into::into) + } +} diff --git a/rust_backend/crates/mobile/src/tags.rs b/rust_backend/crates/mobile/src/tags.rs new file mode 100644 index 00000000..f8759453 --- /dev/null +++ b/rust_backend/crates/mobile/src/tags.rs @@ -0,0 +1,355 @@ +use crate::cancellation::RequestLease; +use crate::manager::{ExtensionManager, ExtensionManagerError}; +use spotiflac_core::tags; +use std::collections::BTreeMap; +use std::fs::{File, OpenOptions}; +use std::sync::Arc; + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum AudioTagsError { + #[error("{message}")] + Read { message: String }, +} + +impl From<String> for AudioTagsError { + fn from(message: String) -> Self { + Self::Read { message } + } +} + +#[uniffi::export] +impl ExtensionManager { + pub fn set_library_cover_cache_directory( + &self, + directory: String, + ) -> Result<(), ExtensionManagerError> { + self.inner + .set_library_cover_cache_directory(&directory) + .map_err(ExtensionManagerError::Operation) + } + + pub fn read_audio_metadata( + &self, + path: String, + hint: String, + cache_key: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .read_audio_metadata(&path, &hint, &cache_key, &|| check_lease(lease.as_deref())) + .map(|value| value.to_string()) + .map_err(ExtensionManagerError::Operation) + } + + pub fn parse_cue_file_json( + &self, + path: String, + audio_directory: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .parse_cue_file_json(&path, &audio_directory, &|| check_lease(lease.as_deref())) + .map(|value| value.to_string()) + .map_err(ExtensionManagerError::Operation) + } + + #[allow(clippy::too_many_arguments)] + pub fn scan_cue_file_for_library( + &self, + path: String, + audio_directory: String, + virtual_prefix: String, + mod_time: i64, + cache_key: String, + scan_time: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .scan_cue_file_for_library( + &path, + &audio_directory, + &virtual_prefix, + mod_time, + &cache_key, + &scan_time, + &|| check_lease(lease.as_deref()), + ) + .map(|value| value.to_string()) + .map_err(ExtensionManagerError::Operation) + } + + pub fn scan_library_folder( + &self, + folder: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .scan_library_folder(&folder, &|| check_lease(lease.as_deref())) + .map(|value| value.to_string()) + .map_err(ExtensionManagerError::Operation) + } + + pub fn scan_library_folder_incremental( + &self, + folder: String, + existing_json: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .scan_library_folder_incremental(&folder, &existing_json, &|| { + check_lease(lease.as_deref()) + }) + .map(|value| value.to_string()) + .map_err(ExtensionManagerError::Operation) + } + + pub fn scan_library_folder_incremental_from_snapshot( + &self, + folder: String, + snapshot: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .scan_library_folder_incremental_from_snapshot(&folder, &snapshot, &|| { + check_lease(lease.as_deref()) + }) + .map(|value| value.to_string()) + .map_err(ExtensionManagerError::Operation) + } + + pub fn scan_library_folder_to_ndjson_file( + &self, + folder: String, + output: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<u64, ExtensionManagerError> { + self.inner + .scan_library_folder_to_ndjson_file(&folder, &output, &|| check_lease(lease.as_deref())) + .map_err(ExtensionManagerError::Operation) + } + + pub fn get_library_scan_progress(&self) -> Result<String, ExtensionManagerError> { + self.inner + .get_library_scan_progress() + .map(|value| value.to_string()) + .map_err(ExtensionManagerError::Operation) + } + + pub fn cancel_library_scan(&self) -> Result<(), ExtensionManagerError> { + self.inner + .cancel_library_scan() + .map_err(ExtensionManagerError::Operation) + } + + pub fn rewrite_split_artist_tags( + &self, + path: String, + artist: String, + album_artist: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .rewrite_split_artist_tags(&path, &artist, &album_artist, &|| { + check_lease(lease.as_deref()) + }) + .map_err(ExtensionManagerError::Operation) + } + + pub fn extract_cover_to_file( + &self, + audio_path: String, + output_path: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<(), ExtensionManagerError> { + self.inner + .extract_cover_to_file(&audio_path, &output_path, &|| check_lease(lease.as_deref())) + .map_err(ExtensionManagerError::Operation) + } + + pub fn save_cover_to_cache_with_hint_and_key( + &self, + audio_path: String, + hint: String, + cache_directory: String, + cache_key: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .save_cover_to_cache_with_hint_and_key( + &audio_path, + &hint, + &cache_directory, + &cache_key, + &|| check_lease(lease.as_deref()), + ) + .map_err(ExtensionManagerError::Operation) + } + + pub fn download_cover_to_file_sized( + &self, + url: String, + output_path: String, + max_dimension: i64, + lease: Option<Arc<RequestLease>>, + ) -> Result<(), ExtensionManagerError> { + self.inner + .download_cover_to_file_sized(&url, &output_path, max_dimension, &|| { + check_lease(lease.as_deref()) + }) + .map_err(ExtensionManagerError::Operation) + } + + pub fn edit_file_metadata( + &self, + path: String, + metadata_json: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .edit_file_metadata(&path, &metadata_json, &|| check_lease(lease.as_deref())) + .map(|result| result.to_string()) + .map_err(ExtensionManagerError::Operation) + } + + pub fn write_m4a_freeform_tags( + &self, + path: String, + metadata_json: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + let check = || check_lease(lease.as_deref()); + check().map_err(ExtensionManagerError::Operation)?; + let fields = decode_fields(&metadata_json).map_err(|error| { + ExtensionManagerError::Operation(format!("invalid metadata JSON: {error}")) + })?; + self.inner + .edit_m4a_freeform(&path, &fields, false, &check) + .map_err(|error| { + ExtensionManagerError::Operation(format!( + "failed to write M4A freeform tags: {error}" + )) + })?; + Ok(r#"{"success":true,"method":"native_m4a_freeform"}"#.into()) + } + + pub fn ensure_ac4_config( + &self, + path: String, + reference: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + self.inner + .ensure_ac4_config(&path, &reference, &|| check_lease(lease.as_deref())) + .map_err(|error| { + ExtensionManagerError::Operation(format!( + "failed to finalize AC-4 container: {error}" + )) + })?; + Ok(r#"{"success":true}"#.into()) + } + + pub fn write_ac4_metadata( + &self, + path: String, + metadata_json: String, + cover_path: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<String, ExtensionManagerError> { + let handled = self + .inner + .write_ac4_metadata(&path, &metadata_json, &cover_path, &|| { + check_lease(lease.as_deref()) + }) + .map_err(|error| { + ExtensionManagerError::Operation(format!("failed to write AC-4 metadata: {error}")) + })?; + Ok(serde_json::json!({"success":true,"handled":handled}).to_string()) + } + + /// Atomic audio tag editing on the root owner's granted file paths. + pub fn edit_audio_tags( + &self, + path: String, + format: String, + fields_json: String, + lease: Option<Arc<RequestLease>>, + ) -> Result<(), ExtensionManagerError> { + let check = || check_lease(lease.as_deref()); + check().map_err(ExtensionManagerError::Operation)?; + let fields = decode_fields(&fields_json).map_err(ExtensionManagerError::Operation)?; + self.inner + .edit_audio_tags(&path, &format, &fields, &check) + .map_err(ExtensionManagerError::Operation) + } +} + +fn decode_fields(json: &str) -> Result<BTreeMap<String, String>, String> { + let fields: Option<BTreeMap<String, Option<String>>> = + serde_json::from_str(json).map_err(|error| error.to_string())?; + Ok(fields + .unwrap_or_default() + .into_iter() + .map(|(key, value)| (key, value.unwrap_or_default())) + .collect()) +} + +/// Read only the tag fields. Native callers retain ownership of any platform +/// descriptor referenced by `path` until this synchronous operation returns. +#[uniffi::export] +pub fn read_audio_tags( + path: String, + format: String, + lease: Option<Arc<RequestLease>>, +) -> Result<String, AudioTagsError> { + let check = || check_lease(lease.as_deref()); + check()?; + let mut file = open_audio_file(&path)?; + let metadata = tags::read_audio_tags(&mut file, &format, &check)?; + serde_json::to_string(&metadata).map_err(|error| error.to_string().into()) +} + +/// Complete application metadata. An empty hint is equivalent to the legacy +/// ReadFileMetadata call. Native descriptor ownership matches read_audio_tags. +#[uniffi::export] +pub fn read_file_metadata( + path: String, + hint: String, + lease: Option<Arc<RequestLease>>, +) -> Result<String, AudioTagsError> { + let check = || check_lease(lease.as_deref()); + check()?; + tags::file_metadata_extension(&path, &hint)?; + let mut file = open_audio_file(&path)?; + let metadata = tags::read_file_metadata(&mut file, &path, &hint, &check)?; + serde_json::to_string(&metadata).map_err(|error| error.to_string().into()) +} + +fn check_lease(lease: Option<&RequestLease>) -> Result<(), String> { + lease.map_or(Ok(()), |lease| { + lease + .inner + .check_active() + .map_err(|error| error.to_string()) + }) +} + +fn open_audio_file(path: &str) -> Result<File, String> { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + // Opening an accidentally supplied FIFO must not block cancellation. + options.custom_flags(rustix::fs::OFlags::NONBLOCK.bits() as i32); + } + let file = options.open(path).map_err(|error| error.to_string())?; + if !file + .metadata() + .map_err(|error| error.to_string())? + .is_file() + { + return Err("audio tags require a regular file".into()); + } + Ok(file) +} diff --git a/rust_backend/crates/mobile/uniffi.toml b/rust_backend/crates/mobile/uniffi.toml new file mode 100644 index 00000000..888a0333 --- /dev/null +++ b/rust_backend/crates/mobile/uniffi.toml @@ -0,0 +1,7 @@ +[bindings.kotlin] +package_name = "com.spotiflac.backend" +cdylib_name = "spotiflac_mobile" + +[bindings.swift] +module_name = "SpotiFLACBackend" +ffi_module_name = "SpotiFLACBackendFFI" diff --git a/rust_backend/crates/network/Cargo.toml b/rust_backend/crates/network/Cargo.toml new file mode 100644 index 00000000..f7e4ed3d --- /dev/null +++ b/rust_backend/crates/network/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "spotiflac-network" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +tokio.workspace = true +hyper.workspace = true +hyper-util.workspace = true +hyper-rustls.workspace = true +rustls.workspace = true +rustls-pemfile.workspace = true +http.workspace = true +http-body-util.workspace = true +bytes.workspace = true +futures-util.workspace = true +tokio-util.workspace = true +async-compression.workspace = true +tower-service.workspace = true +hickory-proto.workspace = true +cookie.workspace = true +url.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[target.'cfg(target_vendor = "apple")'.dependencies] +rustls-platform-verifier.workspace = true + +[target.'cfg(not(target_vendor = "apple"))'.dependencies] +rustls-native-certs.workspace = true + +[lints] +workspace = true diff --git a/rust_backend/crates/network/src/client.rs b/rust_backend/crates/network/src/client.rs new file mode 100644 index 00000000..071b1a2d --- /dev/null +++ b/rust_backend/crates/network/src/client.rs @@ -0,0 +1,1065 @@ +use crate::cookies::CookieJar; +use crate::dns::{Dns, Lookup, Resolver, SystemLookup}; +use crate::policy::{NetworkPermissions, private_literal_or_local}; +use crate::url::UrlParts; +use async_compression::tokio::bufread::GzipDecoder; +use bytes::Bytes; +use futures_util::{TryStreamExt, future::BoxFuture}; +use http::{HeaderMap, HeaderName, HeaderValue, Method, Request, Uri, header}; +use http_body_util::{BodyExt, Full}; +use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; +use hyper_util::client::legacy::{Client, connect::HttpConnector}; +use hyper_util::rt::{TokioExecutor, TokioTimer}; +use serde::Deserialize; +use std::collections::{BTreeMap, HashMap}; +use std::future::Future; +use std::io; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Weak}; +use std::task::{Context, Poll, Wake, Waker}; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncRead, AsyncReadExt, BufReader}; +use tokio::runtime::{Builder, Runtime}; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore}; +use tokio_util::io::StreamReader; +use tower_service::Service; + +pub const MAX_RESPONSE_BYTES: usize = 16 << 20; +type Transport = HttpsConnector<HttpConnector<Resolver>>; +type BoxError = Box<dyn std::error::Error + Send + Sync>; + +#[derive(Clone)] +struct Connector(Transport); + +impl Service<Uri> for Connector { + type Response = <Transport as Service<Uri>>::Response; + type Error = BoxError; + type Future = BoxFuture<'static, Result<Self::Response, BoxError>>; + + fn poll_ready(&mut self, context: &mut Context<'_>) -> Poll<Result<(), Self::Error>> { + self.0.poll_ready(context) + } + + fn call(&mut self, uri: Uri) -> Self::Future { + let future = self.0.call(uri); + Box::pin(async move { + tokio::time::timeout(Duration::from_secs(10), future) + .await + .map_err(|_| { + BoxError::from(io::Error::new( + io::ErrorKind::TimedOut, + "connection timeout", + )) + })? + }) + } +} + +struct Pool { + client: Client<Connector, Full<Bytes>>, + allow_private: bool, + allow_http_fallback: bool, + permits: Mutex<HashMap<String, Weak<Semaphore>>>, +} + +impl Pool { + fn new( + tls: rustls::ClientConfig, + dns: Arc<Dns>, + allow_private: bool, + allow_http_fallback: bool, + ) -> Self { + let mut http = HttpConnector::new_with_resolver(Resolver { dns, allow_private }); + http.enforce_http(false); + http.set_connect_timeout(Some(Duration::from_secs(10))); + http.set_happy_eyeballs_timeout(Some(Duration::from_millis(300))); + http.set_keepalive(Some(Duration::from_secs(30))); + let connector = HttpsConnectorBuilder::new() + .with_tls_config(tls) + .https_or_http() + .enable_http1() + .enable_http2() + .wrap_connector(http); + let client = Client::builder(TokioExecutor::new()) + .pool_timer(TokioTimer::new()) + .pool_idle_timeout(Duration::from_secs(60)) + .pool_max_idle_per_host(10) + .http1_max_buf_size(10 << 20) + .build(Connector(connector)); + Self { + client, + allow_private, + allow_http_fallback, + permits: Mutex::default(), + } + } + + async fn acquire(&self, uri: &Uri) -> OwnedSemaphorePermit { + let key = format!( + "{}://{}", + uri.scheme_str().unwrap_or(""), + uri.authority().map_or("", |authority| authority.as_str()) + ); + let semaphore = { + let mut permits = self.permits.lock().expect("HTTP connection limit lock"); + if permits.len() > 256 { + permits.retain(|_, semaphore| semaphore.strong_count() > 0); + } + match permits.get(&key).and_then(Weak::upgrade) { + Some(semaphore) => semaphore, + None => { + let semaphore = Arc::new(Semaphore::new(20)); + permits.insert(key, Arc::downgrade(&semaphore)); + semaphore + } + } + }; + semaphore + .acquire_owned() + .await + .expect("HTTP semaphore is never closed") + } +} + +/// These options belong to the native backend, never to extension JavaScript. +pub struct NetworkOptions { + pub extra_root_pem: Vec<u8>, + pub lookup: Arc<dyn Lookup>, + pub doh_upstreams: Vec<String>, +} + +impl Default for NetworkOptions { + fn default() -> Self { + Self { + extra_root_pem: Vec::new(), + lookup: Arc::new(SystemLookup), + doh_upstreams: vec![ + "https://1.1.1.1/dns-query".into(), + "https://8.8.8.8/dns-query".into(), + ], + } + } +} + +/// A shared pool and resolver with bounded executor threads. Sessions isolate +/// cookies and permissions while reusing TCP/TLS connections across extensions. +pub struct NetworkService { + runtime: Option<Runtime>, + tls: rustls::ClientConfig, + dns: Arc<Dns>, + pool: Mutex<Arc<Pool>>, + generation: AtomicU64, + checks: Arc<CheckWake>, +} + +#[derive(Default)] +struct CheckWake(Notify); + +impl Wake for CheckWake { + fn wake(self: Arc<Self>) { + self.wake_by_ref(); + } + + fn wake_by_ref(self: &Arc<Self>) { + self.0.notify_waiters(); + } +} + +impl NetworkService { + fn run<T>( + &self, + generation: u64, + timeout: Duration, + check: impl Fn() -> Result<(), String>, + operation: impl Future<Output = Result<T, String>>, + ) -> Result<T, String> { + check()?; + if tokio::runtime::Handle::try_current().is_ok() { + return Err("blocking HTTP host called from async executor".into()); + } + self.runtime + .as_ref() + .expect("network runtime") + .block_on(async { + tokio::pin!(operation); + let timeout = tokio::time::sleep(timeout); + tokio::pin!(timeout); + let mut heartbeat = tokio::time::interval(Duration::from_millis(10)); + loop { + // Create before checking: notify_waiters also reaches a Notified + // future that has not been polled yet, closing the lost-wake gap. + let changed = self.checks.0.notified(); + check()?; + if self.generation.load(Ordering::Acquire) != generation { + return Err("network policy changed".into()); + } + tokio::select! { + biased; + _ = changed => {} + _ = heartbeat.tick() => {} + _ = &mut timeout => return Err("HTTP request timeout exceeded".into()), + result = &mut operation => { check()?; return result; } + } + } + }) + } + + /// Recheck active HTTP calls when the owner's cancellation state changes. + /// Waking does not itself cancel anything or retain the network runtime. + pub fn cancellation_waker(&self) -> Waker { + Waker::from(Arc::clone(&self.checks)) + } + + pub fn new() -> io::Result<Arc<Self>> { + Self::with_options(NetworkOptions::default()) + } + + pub fn with_options(options: NetworkOptions) -> io::Result<Arc<Self>> { + let roots = rustls_pemfile::certs(&mut options.extra_root_pem.as_slice()) + .collect::<Result<Vec<_>, _>>()?; + if !options.extra_root_pem.is_empty() && roots.is_empty() { + return Err(io::Error::other("no certificates in extra root PEM")); + } + let tls = crate::tls::configuration(&roots)?; + let runtime = Builder::new_multi_thread() + .worker_threads(2) + .max_blocking_threads(16) + .thread_name("extension-network") + .enable_all() + .build()?; + let (dns, pool) = { + let _entered = runtime.enter(); + let dns = Arc::new(Dns::new( + tls.clone(), + options.lookup, + &options.doh_upstreams, + )?); + let pool = Arc::new(Pool::new(tls.clone(), Arc::clone(&dns), false, false)); + (dns, pool) + }; + Ok(Arc::new(Self { + runtime: Some(runtime), + tls, + dns, + pool: Mutex::new(pool), + generation: AtomicU64::new(0), + checks: Arc::default(), + })) + } + + /// Called only for the app's explicit private-network preference. Replacing + /// the pool prevents old private connections surviving a policy change. + pub fn set_allow_private_network(&self, allow: bool) { + let mut pool = self.pool.lock().expect("HTTP pool lock"); + if pool.allow_private == allow { + return; + } + let _entered = self.runtime.as_ref().expect("network runtime").enter(); + *pool = Arc::new(Pool::new( + self.tls.clone(), + Arc::clone(&self.dns), + allow, + pool.allow_http_fallback, + )); + self.generation.fetch_add(1, Ordering::AcqRel); + drop(pool); + self.checks.0.notify_waiters(); + } + + /// Native compatibility requests may retry HTTPS over HTTP. Extension + /// sessions retain their manifest policy. TLS verification is never disabled. + pub fn set_network_compatibility_options(&self, allow_http: bool, _insecure_tls: bool) { + let mut pool = self.pool.lock().expect("HTTP pool lock"); + if pool.allow_http_fallback == allow_http { + return; + } + let _entered = self.runtime.as_ref().expect("network runtime").enter(); + *pool = Arc::new(Pool::new( + self.tls.clone(), + Arc::clone(&self.dns), + pool.allow_private, + allow_http, + )); + // As with Go's transport update, active requests retain their options. + // In particular this must not cancel unrelated extension downloads. + } + + pub fn reset_connections(&self) { + let mut pool = self.pool.lock().expect("HTTP pool lock"); + let _entered = self.runtime.as_ref().expect("network runtime").enter(); + *pool = Arc::new(Pool::new( + self.tls.clone(), + Arc::clone(&self.dns), + pool.allow_private, + pool.allow_http_fallback, + )); + // Existing requests retain their pool; a reconnect is not a policy + // change and must not cancel unrelated in-flight downloads. + } + + pub fn session( + self: &Arc<Self>, + permissions: NetworkPermissions, + timeout: Duration, + ) -> Arc<NetworkSession> { + Arc::new(NetworkSession { + service: Arc::clone(self), + permissions: Some(permissions), + native_media: false, + timeout, + cookies: Mutex::default(), + }) + } + + /// App-owned HTTPS requests, such as the extension registry. This is never + /// exposed to JavaScript; extension sessions always retain their allowlist. + pub fn native_session(self: &Arc<Self>, timeout: Duration) -> Arc<NetworkSession> { + Arc::new(NetworkSession { + service: Arc::clone(self), + permissions: None, + native_media: false, + timeout, + cookies: Mutex::default(), + }) + } + + /// Provider artwork uses explicit HTTP or HTTPS URLs and no cookie jar, + /// matching the native media client. DNS/private-network policy and the + /// connection pool are still shared. Never exposed to extension JavaScript. + pub fn native_media_session(self: &Arc<Self>, timeout: Duration) -> Arc<NetworkSession> { + Arc::new(NetworkSession { + service: Arc::clone(self), + permissions: None, + native_media: true, + timeout, + cookies: Mutex::default(), + }) + } +} + +impl Drop for NetworkService { + fn drop(&mut self) { + // libc DNS calls cannot be interrupted. Futures are cancelled promptly; + // do not make native teardown wait for a stalled OS resolver thread. + if let Some(runtime) = self.runtime.take() { + runtime.shutdown_background(); + } + } +} + +#[derive(Debug, Deserialize)] +pub struct HttpRequest { + pub url: String, + #[serde(default)] + pub method: String, + #[serde(default)] + pub body: String, + #[serde(default)] + pub headers: BTreeMap<String, String>, + #[serde(default)] + pub default_json: bool, + #[serde(default = "default_user_agent")] + pub user_agent: String, +} + +fn default_user_agent() -> String { + "Spotiflac-Extension/1.0".to_owned() +} + +#[derive(Debug)] +pub struct HttpResponse { + pub status: u16, + pub status_text: String, + pub url: String, + pub headers: BTreeMap<String, Vec<String>>, + pub body: Vec<u8>, +} + +struct ResponseBody { + reader: Box<dyn AsyncRead + Unpin + Send>, + _permit: OwnedSemaphorePermit, +} + +/// A pull-based media response. Read and progress callbacks run on the caller's +/// worker; JavaScript never executes inside the Tokio executor. Dropping this +/// response immediately releases its body and per-origin permit. +pub struct HttpStream { + /// Response metadata. The body is delivered exclusively through `read`. + pub response: HttpResponse, + body: Option<ResponseBody>, + service: Arc<NetworkService>, + _pool: Arc<Pool>, + generation: u64, + started: Instant, + last_progress: Instant, + timeout: Duration, + stall_timeout: Duration, + failure: Option<String>, +} + +impl HttpStream { + /// Read at most 64 KiB without collecting the response in memory. Errors + /// close the stream so callers cannot accidentally resume a failed reader. + pub fn read( + &mut self, + buffer: &mut [u8], + check: impl Fn() -> Result<(), String>, + ) -> Result<usize, String> { + if let Some(error) = &self.failure { + return Err(error.clone()); + } + let result = (|| { + check()?; + if self.service.generation.load(Ordering::Acquire) != self.generation { + return Err("network policy changed".into()); + } + let Some(body) = self.body.as_mut() else { + return Ok(0); + }; + if buffer.is_empty() { + return Ok(0); + } + let total = self.timeout.saturating_sub(self.started.elapsed()); + let idle = self + .stall_timeout + .saturating_sub(self.last_progress.elapsed()); + if total.is_zero() { + return Err("HTTP request timeout exceeded".into()); + } + if idle.is_zero() { + return Err(self.stall_message()); + } + let size = buffer.len().min(64 << 10); + let count = self + .service + .run(self.generation, total.min(idle), &check, async { + body.reader + .read(&mut buffer[..size]) + .await + .map_err(|error| error.to_string()) + }) + .map_err(|error| { + if error == "HTTP request timeout exceeded" && idle <= total { + self.stall_message() + } else { + error + } + })?; + if count > 0 { + self.last_progress = Instant::now(); + } + Ok(count) + })(); + match &result { + Err(error) => { + self.body = None; + self.failure = Some(error.clone()); + } + Ok(0) if !buffer.is_empty() => self.body = None, + _ => {} + } + result + } + + fn stall_message(&self) -> String { + format!( + "download stalled: no data received for {}s (network timeout)", + self.stall_timeout.as_secs() + ) + } +} + +pub struct NetworkSession { + service: Arc<NetworkService>, + permissions: Option<NetworkPermissions>, + native_media: bool, + timeout: Duration, + cookies: Mutex<CookieJar>, +} + +impl NetworkSession { + pub fn reset_connections(&self) { + self.service.reset_connections(); + } + + /// Browser authorization endpoints need HTTPS and private-address checks, + /// but may use an identity provider outside the extension API allowlist. + pub fn validate_auth_url( + &self, + input: &str, + check: impl Fn() -> Result<(), String>, + ) -> Result<UrlParts, String> { + check()?; + let url = UrlParts::parse(input).ok_or_else(|| "invalid auth URL".to_owned())?; + if url.scheme != "https" { + return Err("invalid auth URL: only https is allowed".into()); + } + if url.hostname.is_empty() { + return Err("invalid auth URL: hostname is required".into()); + } + if url.has_credentials { + return Err("invalid auth URL: embedded credentials are not allowed".into()); + } + let (allow, generation) = { + let pool = self.service.pool.lock().expect("HTTP pool lock"); + ( + pool.allow_private, + self.service.generation.load(Ordering::Acquire), + ) + }; + if !allow { + let uri = url.request_uri()?; + let private = private_literal_or_local(&url.hostname) + || uri.host().is_some_and(private_literal_or_local); + let private = private + || self.service.run(generation, self.timeout, check, async { + Ok(self.service.dns.has_private_address(&url.hostname).await) + })?; + if private { + return Err("invalid auth URL: private/local network is not allowed".into()); + } + } + Ok(url) + } + + pub fn clear_cookies(&self) { + self.cookies.lock().expect("cookie jar lock").clear(); + } + + pub fn validate_url(&self, url: &str) -> Result<(), String> { + let allow = self + .service + .pool + .lock() + .expect("HTTP pool lock") + .allow_private; + self.validate_target(url, allow, false).map(|_| ()) + } + + fn validate_target( + &self, + input: &str, + allow_private: bool, + redirect: bool, + ) -> Result<UrlParts, String> { + if let Some(permissions) = &self.permissions { + return permissions.validate(input, allow_private, redirect); + } + let url = UrlParts::parse(input).ok_or_else(|| "invalid URL".to_owned())?; + NetworkPermissions { + domains: vec![url.hostname], + allow_http: self.native_media, + } + .validate(input, allow_private, redirect) + } + + /// Blocking host call, run from the dedicated JS worker or a native worker. + /// The heartbeat observes cancellation while DNS, TLS, headers or body wait. + pub fn request( + &self, + request: HttpRequest, + check: impl Fn() -> Result<(), String>, + ) -> Result<HttpResponse, String> { + check()?; + if tokio::runtime::Handle::try_current().is_ok() { + return Err("blocking HTTP host called from async executor".into()); + } + let (pool, generation) = { + let pool = self.service.pool.lock().expect("HTTP pool lock"); + ( + Arc::clone(&pool), + self.service.generation.load(Ordering::Acquire), + ) + }; + self.service.run( + generation, + self.timeout, + check, + self.execute(request, &pool), + ) + } + + /// Streaming downloads share DNS, TLS, cookies, redirects and connection + /// limits with API requests, but preserve identity content encoding and use + /// the native transfer's wall-clock/stall limits rather than API timeout. + pub fn open_stream( + &self, + request: HttpRequest, + timeout: Duration, + stall_timeout: Duration, + check: impl Fn() -> Result<(), String>, + ) -> Result<HttpStream, String> { + self.open_stream_with_encoding(request, timeout, stall_timeout, false, &check) + } + + /// API-style gzip decoding with pull-based reads. The caller enforces its + /// own decoded-body size limit without raising the buffered API limit. + pub fn open_response_stream( + &self, + request: HttpRequest, + check: impl Fn() -> Result<(), String>, + ) -> Result<HttpStream, String> { + self.open_stream_with_encoding(request, self.timeout, self.timeout, true, &check) + } + + fn open_stream_with_encoding( + &self, + request: HttpRequest, + timeout: Duration, + stall_timeout: Duration, + compression: bool, + check: &dyn Fn() -> Result<(), String>, + ) -> Result<HttpStream, String> { + let started = Instant::now(); + let (pool, generation) = { + let pool = self.service.pool.lock().expect("HTTP pool lock"); + ( + Arc::clone(&pool), + self.service.generation.load(Ordering::Acquire), + ) + }; + let (response, body) = self.service.run( + generation, + timeout.min(stall_timeout), + check, + self.execute_stream(request, &pool, compression), + )?; + Ok(HttpStream { + response, + body: Some(body), + service: Arc::clone(&self.service), + _pool: pool, + generation, + started, + last_progress: started, + timeout, + stall_timeout, + failure: None, + }) + } + + async fn execute(&self, request: HttpRequest, pool: &Pool) -> Result<HttpResponse, String> { + let (mut response, body) = self.execute_stream(request, pool, true).await?; + body.reader + .take((MAX_RESPONSE_BYTES + 1) as u64) + .read_to_end(&mut response.body) + .await + .map_err(|error| error.to_string())?; + if response.body.len() > MAX_RESPONSE_BYTES { + return Err(format!( + "response body exceeds {MAX_RESPONSE_BYTES} byte limit; use file.download for large media" + )); + } + Ok(response) + } + + async fn execute_stream( + &self, + request: HttpRequest, + pool: &Pool, + compression: bool, + ) -> Result<(HttpResponse, ResponseBody), String> { + let mut url = self.validate_target(&request.url, pool.allow_private, false)?; + let original = url.clone(); + let mut method = Method::from_bytes(if request.method.is_empty() { + b"GET" + } else { + request.method.as_bytes() + }) + .map_err(|_| format!("net/http: invalid method {:?}", request.method))?; + let mut body = Bytes::from(request.body); + let mut headers = HeaderMap::new(); + for (name, value) in request.headers { + let name = + HeaderName::from_bytes(name.as_bytes()).map_err(|error| error.to_string())?; + if name == header::HOST + || name == header::CONTENT_LENGTH + || name == header::TRANSFER_ENCODING + { + continue; + } + headers.insert( + name, + HeaderValue::from_str(&value).map_err(|error| error.to_string())?, + ); + } + if headers + .get(header::USER_AGENT) + .is_none_or(|value| value.is_empty()) + { + headers.insert( + header::USER_AGENT, + request + .user_agent + .parse() + .map_err(|error: http::header::InvalidHeaderValue| error.to_string())?, + ); + } + if request.default_json + && headers + .get(header::CONTENT_TYPE) + .is_none_or(|value| value.is_empty()) + { + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + } + let explicit_referer = headers + .get(header::REFERER) + .filter(|value| !value.is_empty()) + .cloned(); + for redirects in 0..10 { + let uri = url.request_uri()?; + let mut permit = pool.acquire(&uri).await; + let mut outgoing = headers.clone(); + let cookies = if self.native_media { + String::new() + } else { + self.cookies.lock().expect("cookie jar lock").header(&url) + }; + if !cookies.is_empty() { + let value = outgoing + .get(header::COOKIE) + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.is_empty()) + .map_or_else(|| cookies.clone(), |value| format!("{value}; {cookies}")); + outgoing.insert( + header::COOKIE, + value + .parse() + .map_err(|error: http::header::InvalidHeaderValue| error.to_string())?, + ); + } + let gzip = compression + && outgoing + .get(header::ACCEPT_ENCODING) + .is_none_or(|value| value.is_empty()) + && outgoing + .get(header::RANGE) + .is_none_or(|value| value.is_empty()) + && method != Method::HEAD; + if gzip { + outgoing.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + } + let mut req = Request::builder() + .method(method.clone()) + .uri(uri) + .body(Full::new(body.clone())) + .map_err(|error| error.to_string())?; + *req.headers_mut() = outgoing.clone(); + let response = tokio::time::timeout(Duration::from_secs(45), pool.client.request(req)) + .await + .map_err(|_| "HTTP response header timeout exceeded".to_owned()) + .and_then(|result| result.map_err(|error| error_chain(&error))); + let mut response_url = url.clone(); + // Nonempty bodies are buffered and can be replayed, as Go's GetBody + // permits. Treat an empty body like a nil Go request body. + let can_replay = matches!( + method, + Method::GET | Method::HEAD | Method::OPTIONS | Method::DELETE + ) || !body.is_empty(); + let response = match response { + Err(_) + if pool.allow_http_fallback + && self.permissions.is_none() + && url.scheme == "https" + && can_replay => + { + response_url.scheme = "http".into(); + let uri = response_url.request_uri()?; + // The hostname is unchanged; the shared connector still + // resolves and checks every destination against private-IP policy. + drop(permit); + permit = pool.acquire(&uri).await; + let mut retry = Request::builder() + .method(method.clone()) + .uri(uri) + .body(Full::new(body.clone())) + .map_err(|error| error.to_string())?; + *retry.headers_mut() = outgoing; + tokio::time::timeout(Duration::from_secs(45), pool.client.request(retry)) + .await + .map_err(|_| "HTTP response header timeout exceeded".to_owned())? + .map_err(|error| error_chain(&error))? + } + result => result?, + }; + if !self.native_media { + self.cookies + .lock() + .expect("cookie jar lock") + .store(&url, response.headers()); + } + let status = response.status(); + if matches!(status.as_u16(), 301 | 302 | 303 | 307 | 308) + && let Some(location) = response + .headers() + .get(header::LOCATION) + .filter(|value| !value.is_empty()) + { + let location = location.to_str().map_err(|error| error.to_string())?; + let target = url + .resolve(location) + .ok_or_else(|| "invalid redirect URL".to_owned())?; + if target.has_credentials { + return Err("invalid URL: embedded credentials are not allowed".into()); + } + self.validate_target(&target.display_url(), pool.allow_private, true)?; + if redirects < 9 { + update_explicit_cookies(&mut headers, response.headers()); + if matches!(status.as_u16(), 301..=303) { + if method != Method::GET && method != Method::HEAD { + method = Method::GET; + } + body = Bytes::new(); + for name in [ + header::CONTENT_ENCODING, + header::CONTENT_LANGUAGE, + header::CONTENT_LOCATION, + header::CONTENT_TYPE, + ] { + headers.remove(name); + } + } + if !forward_sensitive_headers(&original.hostname, &target.hostname) { + for name in [ + header::AUTHORIZATION, + header::WWW_AUTHENTICATE, + header::COOKIE, + header::PROXY_AUTHORIZATION, + header::PROXY_AUTHENTICATE, + ] { + headers.remove(name); + } + headers.remove("cookie2"); + } + if url.scheme == "https" && target.scheme == "http" { + headers.remove(header::REFERER); + } else if let Some(value) = &explicit_referer { + headers.insert(header::REFERER, value.clone()); + } else if let Ok(value) = url.display_url().parse() { + headers.insert(header::REFERER, value); + } + url = target; + // A small HTTP/1.1 redirect body can return this connection + // to the pool. Never wait for an unbounded/slow body or drain + // HTTP/2 streams, whose connections are already multiplexed. + if response.version() == http::Version::HTTP_11 + && response + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::<usize>().ok()) + .is_some_and(|length| length <= 2048) + { + let mut body = response.into_body(); + let _ = tokio::time::timeout(Duration::from_millis(20), async { + let mut received = 0; + while let Some(Ok(frame)) = body.frame().await { + received += frame.data_ref().map_or(0, Bytes::len); + if received > 2048 { + break; + } + } + }) + .await; + } + continue; + } + } + let mut response_headers = response.headers().clone(); + let decompress = gzip + && response_headers + .get(header::CONTENT_ENCODING) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"gzip")); + let stream = response + .into_body() + .into_data_stream() + .map_err(io::Error::other); + let reader = StreamReader::new(stream); + let reader: Box<dyn AsyncRead + Unpin + Send> = if decompress { + response_headers.remove(header::CONTENT_ENCODING); + response_headers.remove(header::CONTENT_LENGTH); + let mut decoder = GzipDecoder::new(BufReader::new(reader)); + decoder.multiple_members(true); + Box::new(decoder) + } else { + Box::new(reader) + }; + let mut headers = BTreeMap::new(); + for name in response_headers.keys() { + headers.insert( + canonical_header(name.as_str()), + response_headers + .get_all(name) + .iter() + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .collect(), + ); + } + return Ok(( + HttpResponse { + status: status.as_u16(), + status_text: status.canonical_reason().unwrap_or("").into(), + url: response_url.display_url(), + headers, + body: Vec::new(), + }, + ResponseBody { + reader, + _permit: permit, + }, + )); + } + unreachable!("last redirect returns its response") + } +} + +fn update_explicit_cookies(headers: &mut HeaderMap, response: &HeaderMap) { + let Some(original) = headers + .get(header::COOKIE) + .and_then(|value| value.to_str().ok()) + else { + return; + }; + let names: Vec<_> = response + .get_all(header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .filter_map(|value| cookie::Cookie::parse(value).ok()) + .map(|cookie| cookie.name().to_owned()) + .collect(); + let mut changed = false; + let mut remaining: Vec<_> = original + .split(';') + .map(str::trim) + .filter(|pair| { + let remove = pair + .split_once('=') + .is_some_and(|(name, _)| names.iter().any(|replaced| replaced == name)); + changed |= remove; + !remove + }) + .collect(); + if changed { + remaining.sort_unstable(); + let value = remaining + .join("; ") + .parse() + .expect("existing HTTP cookie header"); + headers.insert(header::COOKIE, value); + } +} + +fn forward_sensitive_headers(original: &str, target: &str) -> bool { + let original = original.to_lowercase(); + let target = target.to_lowercase(); + original == target + || (!target.contains([':', '%']) && target.ends_with(&format!(".{original}"))) +} + +fn canonical_header(name: &str) -> String { + name.split('-') + .map(|word| { + let mut bytes = word.as_bytes().to_vec(); + if let Some(first) = bytes.first_mut() { + first.make_ascii_uppercase(); + } + String::from_utf8(bytes).expect("ASCII HTTP header name") + }) + .collect::<Vec<_>>() + .join("-") +} + +fn error_chain(error: &dyn std::error::Error) -> String { + let mut message = error.to_string(); + if let Some(source) = error.source() { + message.push_str(": "); + message.push_str(&error_chain(source)); + } + message +} + +#[cfg(test)] +mod tests { + use super::*; + use std::future::poll_fn; + use std::sync::{atomic::AtomicBool, mpsc}; + + #[test] + fn cancellation_after_operation_is_pending_drops_it_before_return() { + struct Dropped<'a>(&'a AtomicBool); + impl Drop for Dropped<'_> { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } + } + + let service = NetworkService::new().unwrap(); + // Sample two phases; timings are diagnostics, not shared-host limits. + for (waiting_for, wake) in [ + (Duration::ZERO, false), + (Duration::from_millis(15), false), + (Duration::from_millis(15), true), + ] { + let cancelled = AtomicBool::new(false); + let dropped = AtomicBool::new(false); + let (waiting_tx, waiting_rx) = mpsc::sync_channel(1); + let (finished_tx, finished_rx) = mpsc::sync_channel(1); + std::thread::scope(|scope| { + let worker = scope.spawn(|| { + let mut waiting = Some(waiting_tx); + let result = service.run( + 0, + Duration::from_secs(2), + || { + if cancelled.load(Ordering::Acquire) { + Err("cancelled".into()) + } else { + Ok(()) + } + }, + async { + let _dropped = Dropped(&dropped); + if !waiting_for.is_zero() { + tokio::time::sleep(waiting_for).await; + } + poll_fn(|_| { + if let Some(waiting) = waiting.take() { + waiting.send(()).unwrap(); + } + Poll::<Result<(), String>>::Pending + }) + .await + }, + ); + finished_tx + .send((result, Instant::now(), dropped.load(Ordering::Acquire))) + .unwrap(); + }); + waiting_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + let began = Instant::now(); + cancelled.store(true, Ordering::Release); + if wake { + service.cancellation_waker().wake(); + } + let (result, returned, dropped_before_return) = + finished_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + worker.join().unwrap(); + assert_eq!(result, Err("cancelled".into())); + assert!(dropped_before_return); + println!( + "pending operation cancellation (wait={}ms, wake={wake}): return={}us join={}us", + waiting_for.as_millis(), + returned.duration_since(began).as_micros(), + began.elapsed().as_micros(), + ); + }); + } + assert_eq!( + service.run(0, Duration::from_secs(1), || Ok(()), async { Ok(7) }), + Ok(7) + ); + } +} diff --git a/rust_backend/crates/network/src/cookies.rs b/rust_backend/crates/network/src/cookies.rs new file mode 100644 index 00000000..cb8afdc0 --- /dev/null +++ b/rust_backend/crates/network/src/cookies.rs @@ -0,0 +1,238 @@ +use crate::url::UrlParts; +use cookie::Cookie; +use cookie::time::OffsetDateTime; +use std::collections::BTreeMap; +use std::net::IpAddr; + +#[derive(Default)] +pub(crate) struct CookieJar { + entries: BTreeMap<(String, String, Vec<u8>, String), Entry>, + sequence: u64, +} + +struct Entry { + value: String, + host_only: bool, + secure: bool, + expires: Option<OffsetDateTime>, + sequence: u64, +} + +impl CookieJar { + pub(crate) fn store(&mut self, url: &UrlParts, headers: &http::HeaderMap) { + let host = canonical_host(url); + let partition = jar_key(&host); + let now = OffsetDateTime::now_utc(); + // Go's default cookie parser rejects the entire batch over this limit. + if headers.get_all(http::header::SET_COOKIE).iter().count() > 3000 { + return; + } + for header in headers.get_all(http::header::SET_COOKIE) { + let Ok(raw) = header.to_str() else { continue }; + let Ok(cookie) = Cookie::parse(raw) else { + continue; + }; + if http::HeaderName::from_bytes(cookie.name().as_bytes()).is_err() { + continue; + } + let Some((_, raw_value)) = raw.split(';').next().and_then(|pair| pair.split_once('=')) + else { + continue; + }; + let raw_value = raw_value.trim(); + let quoted = + raw_value.starts_with('"') && raw_value.ends_with('"') && raw_value.len() >= 2; + let value = if quoted { + &raw_value[1..raw_value.len() - 1] + } else { + raw_value + }; + if !value + .bytes() + .all(|byte| (32..127).contains(&byte) && !b"\";\\".contains(&byte)) + { + continue; + } + // Keep the unstripped Domain attribute: Go rejects .IP and ..host. + let raw_domain = raw + .split(';') + .skip(1) + .filter_map(|attribute| attribute.trim().split_once('=')) + .filter(|(name, _)| name.eq_ignore_ascii_case("domain")) + .map(|(_, value)| value.trim()) + .last() + .unwrap_or(""); + let (domain, host_only) = if raw_domain.is_empty() { + (host.clone(), true) + } else if host.parse::<IpAddr>().is_ok() || host.contains(':') { + if raw_domain != host { + continue; + } + (host.clone(), true) + } else { + let domain = raw_domain + .strip_prefix('.') + .unwrap_or(raw_domain) + .to_ascii_lowercase(); + if domain.is_empty() + || !domain.is_ascii() + || domain.starts_with('.') + || domain.ends_with('.') + || (host != domain && !host.ends_with(&format!(".{domain}"))) + { + continue; + } + (domain, false) + }; + let path = cookie + .path() + .filter(|path| path.starts_with('/')) + .map(|path| path.as_bytes().to_vec()) + .unwrap_or_else(|| default_path(&url.path)); + let key = (partition.clone(), domain, path, cookie.name().to_owned()); + let max_age = raw + .split(';') + .skip(1) + .filter_map(|attribute| attribute.trim().split_once('=')) + .filter(|(name, _)| name.eq_ignore_ascii_case("max-age")) + .filter_map(|(_, value)| { + let value = value.trim(); + let seconds = value.parse::<isize>().ok()?; + if seconds != 0 && value.starts_with('0') { + None + } else { + Some(seconds) + } + }) + .last(); + let expires = match max_age { + Some(seconds) if seconds <= 0 => Some(now), + Some(seconds) => now.checked_add(cookie::time::Duration::seconds(seconds as i64)), + None => cookie.expires_datetime(), + }; + if expires.is_some_and(|expires| expires <= now) { + self.entries.remove(&key); + continue; + } + let sequence = self.entries.get(&key).map_or_else( + || { + let sequence = self.sequence; + self.sequence += 1; + sequence + }, + |entry| entry.sequence, + ); + let value = if quoted || value.contains([' ', ',']) { + format!("\"{value}\"") + } else { + value.to_owned() + }; + self.entries.insert( + key, + Entry { + value, + host_only, + secure: cookie.secure().unwrap_or(false), + expires, + sequence, + }, + ); + } + } + + pub(crate) fn header(&mut self, url: &UrlParts) -> String { + let now = OffsetDateTime::now_utc(); + self.entries + .retain(|_, entry| entry.expires.is_none_or(|expires| expires > now)); + let host = canonical_host(url); + let partition = jar_key(&host); + let mut selected: Vec<_> = self + .entries + .iter() + .filter(|((jar, domain, path, _), entry)| { + *jar == partition + && (!entry.secure || url.scheme == "https") + && (host == *domain + || (!entry.host_only && host.ends_with(&format!(".{domain}")))) + && (url.path == *path + || (url.path.starts_with(path) + && (path.ends_with(b"/") || url.path.get(path.len()) == Some(&b'/')))) + }) + .collect(); + selected.sort_by(|((_, _, left, _), a), ((_, _, right, _), b)| { + right.cmp(left).then(a.sequence.cmp(&b.sequence)) + }); + selected + .iter() + .map(|((_, _, _, name), entry)| format!("{name}={}", entry.value)) + .collect::<Vec<_>>() + .join("; ") + } + + pub(crate) fn clear(&mut self) { + self.entries.clear(); + } +} + +fn canonical_host(url: &UrlParts) -> String { + let host = url.hostname.strip_suffix('.').unwrap_or(&url.hostname); + ::url::Host::parse(host).map_or_else(|_| host.to_ascii_lowercase(), |host| host.to_string()) +} + +// Go uses cookiejar.New(nil): buckets use the final two labels, with no PSL. +fn jar_key(host: &str) -> String { + if host.parse::<IpAddr>().is_ok() || host.contains(':') { + return host.to_owned(); + } + let mut dots = host.rmatch_indices('.'); + dots.next(); + host[dots.next().map_or(0, |(index, _)| index + 1)..].to_owned() +} + +fn default_path(path: &[u8]) -> Vec<u8> { + let end = path.iter().rposition(|byte| *byte == b'/').unwrap_or(0); + if !path.starts_with(b"/") || end == 0 { + b"/".to_vec() + } else { + path[..end].to_vec() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn paths_domains_expiry_quoting_and_clear_are_session_local() { + let mut jar = CookieJar::default(); + let url = UrlParts::parse("https://api.example.test/a/b").unwrap(); + let mut headers = http::HeaderMap::new(); + for value in [ + "root=1; Path=/", + "deep=2", + "wide=3; Domain=.example.test; Secure", + "quoted=\"a b\"; Path=/", + "bad=4; Domain=other.test", + "old=5; Max-Age=0", + "public=6; Domain=test; Path=/", + ] { + headers.append(http::header::SET_COOKIE, value.parse().unwrap()); + } + jar.store(&url, &headers); + assert_eq!( + jar.header(&url), + "deep=2; wide=3; root=1; quoted=\"a b\"; public=6" + ); + assert_eq!( + jar.header(&UrlParts::parse("http://sub.example.test/a/b").unwrap()), + "public=6" + ); + assert_eq!( + jar.header(&UrlParts::parse("https://other.test/a/b").unwrap()), + "" + ); + assert_eq!(CookieJar::default().header(&url), ""); + jar.clear(); + assert_eq!(jar.header(&url), ""); + } +} diff --git a/rust_backend/crates/network/src/dns.rs b/rust_backend/crates/network/src/dns.rs new file mode 100644 index 00000000..23502c62 --- /dev/null +++ b/rust_backend/crates/network/src/dns.rs @@ -0,0 +1,266 @@ +use crate::policy::is_private_ip; +use bytes::Bytes; +use futures_util::future::BoxFuture; +use hickory_proto::op::{Message, MessageType, OpCode, Query, ResponseCode}; +use hickory_proto::rr::{Name, RData, RecordType}; +use http_body_util::{BodyExt, Full}; +use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; +use hyper_util::client::legacy::Client; +use hyper_util::client::legacy::connect::{HttpConnector, dns}; +use hyper_util::rt::{TokioExecutor, TokioTimer}; +use std::collections::HashMap; +use std::io; +use std::net::{IpAddr, SocketAddr}; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; +use tower_service::Service; + +pub type LookupFuture = BoxFuture<'static, io::Result<Vec<IpAddr>>>; + +/// Trusted platform resolver injection. JavaScript cannot supply DNS answers. +pub trait Lookup: Send + Sync { + fn lookup(&self, hostname: &str) -> LookupFuture; +} + +pub struct SystemLookup; + +impl Lookup for SystemLookup { + fn lookup(&self, hostname: &str) -> LookupFuture { + let hostname = hostname.to_owned(); + Box::pin(async move { + Ok(tokio::net::lookup_host((hostname, 0)) + .await? + .map(|address| address.ip()) + .collect()) + }) + } +} + +type DohClient = Client<HttpsConnector<HttpConnector>, Full<Bytes>>; + +struct Cached { + addresses: Vec<IpAddr>, + expires: Instant, +} + +pub(crate) struct Dns { + lookup: Arc<dyn Lookup>, + doh: DohClient, + upstreams: Vec<http::Uri>, + cache: Mutex<HashMap<String, Cached>>, +} + +impl Dns { + pub(crate) async fn has_private_address(&self, hostname: &str) -> bool { + self.lookup + .lookup(hostname) + .await + .is_ok_and(|addresses| addresses.into_iter().any(is_private_ip)) + } + + pub(crate) fn new( + tls: rustls::ClientConfig, + lookup: Arc<dyn Lookup>, + upstreams: &[String], + ) -> io::Result<Self> { + let upstreams = upstreams + .iter() + .map(|url| { + let uri = url.parse::<http::Uri>().map_err(io::Error::other)?; + // Literal HTTPS endpoints avoid recursively resolving the resolver. + if uri.scheme_str() != Some("https") + || uri + .host() + .and_then(|host| host.trim_matches(['[', ']']).parse::<IpAddr>().ok()) + .is_none() + { + return Err(io::Error::other( + "DoH upstream must use HTTPS and an IP literal", + )); + } + Ok(uri) + }) + .collect::<io::Result<Vec<_>>>()?; + let mut http = HttpConnector::new(); + http.enforce_http(false); + http.set_connect_timeout(Some(Duration::from_secs(5))); + let connector = HttpsConnectorBuilder::new() + .with_tls_config(tls) + .https_only() + .enable_http1() + .enable_http2() + .wrap_connector(http); + let doh = Client::builder(TokioExecutor::new()) + .pool_timer(TokioTimer::new()) + .pool_idle_timeout(Duration::from_secs(60)) + .pool_max_idle_per_host(2) + .build(connector); + Ok(Self { + lookup, + doh, + upstreams, + cache: Mutex::default(), + }) + } + + pub(crate) async fn resolve( + &self, + host: &str, + allow_private: bool, + ) -> io::Result<Vec<SocketAddr>> { + let addresses = match self.lookup.lookup(host).await { + Ok(addresses) => addresses, + Err(original) => { + if self.upstreams.is_empty() { + return Err(original); + } + self.fallback(host).await? + } + }; + let addresses: Vec<_> = addresses + .into_iter() + .filter(|address| allow_private || !is_private_ip(*address)) + .map(|address| SocketAddr::new(address, 0)) + .collect(); + if addresses.is_empty() { + return Err(io::Error::other(format!( + "network access denied: no permitted DNS addresses for '{host}'" + ))); + } + Ok(addresses) + } + + async fn fallback(&self, host: &str) -> io::Result<Vec<IpAddr>> { + let key = host.to_lowercase(); + if let Some(entry) = self + .cache + .lock() + .expect("DNS cache lock") + .get(&key) + .filter(|entry| Instant::now() < entry.expires) + { + return if entry.addresses.is_empty() { + Err(io::Error::other("cached DNS lookup failure")) + } else { + Ok(entry.addresses.clone()) + }; + } + let mut result = Err(io::Error::other("DNS-over-HTTPS lookup failed")); + for upstream in &self.upstreams { + result = self.query(upstream, host, RecordType::A).await; + if matches!(&result, Ok((addresses, _)) if addresses.is_empty()) { + result = self.query(upstream, host, RecordType::AAAA).await; + } + if matches!(&result, Ok((addresses, _)) if !addresses.is_empty()) { + break; + } + } + let (addresses, seconds) = match &result { + Ok((addresses, ttl)) if !addresses.is_empty() => { + (addresses.clone(), u64::from((*ttl).clamp(60, 1800))) + } + _ => (Vec::new(), 30), + }; + let mut cache = self.cache.lock().expect("DNS cache lock"); + if cache.len() >= 256 { + cache.retain(|_, entry| Instant::now() < entry.expires); + if cache.len() >= 256 { + cache.clear(); + } + } + cache.insert( + key, + Cached { + addresses: addresses.clone(), + expires: Instant::now() + Duration::from_secs(seconds), + }, + ); + if addresses.is_empty() { + Err(io::Error::other("DNS-over-HTTPS returned no addresses")) + } else { + Ok(addresses) + } + } + + async fn query( + &self, + upstream: &http::Uri, + host: &str, + kind: RecordType, + ) -> io::Result<(Vec<IpAddr>, u32)> { + let mut message = Message::new(0, MessageType::Query, OpCode::Query); + message.metadata.recursion_desired = true; + message.queries.push(Query::query( + Name::from_ascii(host).map_err(io::Error::other)?, + kind, + )); + let request = http::Request::post(upstream.clone()) + .header("Content-Type", "application/dns-message") + .header("Accept", "application/dns-message") + .body(Full::new(Bytes::from( + message.to_vec().map_err(io::Error::other)?, + ))) + .map_err(io::Error::other)?; + tokio::time::timeout(Duration::from_secs(10), async { + let response = self.doh.request(request).await.map_err(io::Error::other)?; + if !response.status().is_success() { + return Err(io::Error::other("DoH HTTP failure")); + } + let mut body = http_body_util::Limited::new(response.into_body(), 65536); + let mut bytes = Vec::new(); + while let Some(frame) = body.frame().await { + if let Ok(data) = frame.map_err(io::Error::other)?.into_data() { + bytes.extend_from_slice(&data); + } + } + let response = Message::from_vec(&bytes).map_err(io::Error::other)?; + if response.metadata.response_code != ResponseCode::NoError + || response.metadata.message_type != MessageType::Response + { + return Err(io::Error::other("DoH DNS failure")); + } + let mut addresses = Vec::new(); + let mut ttl = u32::MAX; + for record in response.answers { + let address = match record.data { + RData::A(address) => IpAddr::V4(address.0), + RData::AAAA(address) => IpAddr::V6(address.0), + _ => continue, + }; + addresses.push(address); + ttl = ttl.min(record.ttl); + } + Ok((addresses, ttl)) + }) + .await + .map_err(io::Error::other)? + } +} + +#[derive(Clone)] +pub(crate) struct Resolver { + pub dns: Arc<Dns>, + pub allow_private: bool, +} + +impl Service<dns::Name> for Resolver { + type Response = std::vec::IntoIter<SocketAddr>; + type Error = io::Error; + type Future = BoxFuture<'static, io::Result<Self::Response>>; + + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll<io::Result<()>> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, name: dns::Name) -> Self::Future { + let resolver = self.clone(); + Box::pin(async move { + Ok(resolver + .dns + .resolve(name.as_str(), resolver.allow_private) + .await? + .into_iter()) + }) + } +} diff --git a/rust_backend/crates/network/src/lib.rs b/rust_backend/crates/network/src/lib.rs new file mode 100644 index 00000000..e5d939ac --- /dev/null +++ b/rust_backend/crates/network/src/lib.rs @@ -0,0 +1,30 @@ +//! Shared, cancellable HTTP transport for the embedded backend. + +mod client; +mod cookies; +mod dns; +pub mod policy; +pub mod query; +mod tls; +pub mod url; + +pub use client::{ + HttpRequest, HttpResponse, HttpStream, MAX_RESPONSE_BYTES, NetworkOptions, NetworkService, + NetworkSession, +}; +pub use dns::{Lookup, LookupFuture, SystemLookup}; + +/// Shared browser identity for native clients and `utils.randomUserAgent()`. +pub fn random_user_agent() -> String { + use std::collections::hash_map::RandomState; + use std::hash::BuildHasher; + + // Keep the existing 26-major-version window, now ending at Chrome 152. + let random = RandomState::new().hash_one(()); + format!( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{}.0.{}.{} Safari/537.36", + 127 + random % 26, + 6000 + (random >> 8) % 1500, + 100 + (random >> 24) % 200 + ) +} diff --git a/rust_backend/crates/network/src/policy.rs b/rust_backend/crates/network/src/policy.rs new file mode 100644 index 00000000..7f3e89a0 --- /dev/null +++ b/rust_backend/crates/network/src/policy.rs @@ -0,0 +1,205 @@ +//! Permission checks happen before dispatch and again on every resolved address. + +use crate::url::UrlParts; +use std::net::IpAddr; + +#[derive(Clone, Debug, Default)] +pub struct NetworkPermissions { + pub domains: Vec<String>, + pub allow_http: bool, +} + +impl NetworkPermissions { + pub fn allows_domain(&self, domain: &str) -> bool { + let domain = domain.trim().to_lowercase(); + self.domains.iter().any(|allowed| { + let allowed = allowed.trim().to_lowercase(); + allowed == domain + || allowed.strip_prefix("*.").is_some_and(|suffix| { + domain.len() > suffix.len() + 1 && domain.ends_with(&format!(".{suffix}")) + }) + }) + } + + pub fn validate( + &self, + input: &str, + allow_private: bool, + redirect: bool, + ) -> Result<UrlParts, String> { + let url = UrlParts::parse(input).ok_or_else(|| "invalid URL".to_owned())?; + if url.scheme.is_empty() && !redirect { + return Err("invalid URL: scheme is required".to_owned()); + } + if url.scheme != "https" && !(self.allow_http && url.scheme == "http") { + return Err(if redirect { + "redirect blocked: only https is allowed" + } else { + "network access denied: only https is allowed" + } + .to_owned()); + } + if url.has_credentials { + return Err("invalid URL: embedded credentials are not allowed".to_owned()); + } + if url.hostname.is_empty() { + return Err(if redirect { + "redirect blocked: hostname is required" + } else { + "invalid URL: hostname is required" + } + .to_owned()); + } + if !redirect && !allow_private && private_literal_or_local(&url.hostname) { + return Err(format!( + "network access denied: private/local network '{}' not allowed", + url.hostname + )); + } + if !self.allows_domain(&url.hostname) { + return Err(format!( + "{}: domain '{}' not in allowed list", + if redirect { + "redirect blocked" + } else { + "network access denied" + }, + url.hostname + )); + } + if redirect && !allow_private && private_literal_or_local(&url.hostname) { + return Err("redirect blocked: private/local network access denied".to_owned()); + } + // Hyper skips the resolver for literal addresses. Check the actual URI + // too, including any host canonicalization performed by IDNA parsing. + let uri = url.request_uri()?; + if !allow_private && uri.host().is_some_and(private_literal_or_local) { + return Err(format!( + "network access denied: private/local network '{}' not allowed", + url.hostname + )); + } + Ok(url) + } +} + +pub fn private_literal_or_local(host: &str) -> bool { + let host = host + .trim() + .trim_start_matches('[') + .trim_end_matches(']') + .to_lowercase(); + host == "localhost" + || host.ends_with(".local") + || host + .split('%') + .next() + .and_then(|address| address.parse().ok()) + .is_some_and(is_private_ip) +} + +/// Matches Go net.IP's private/global-unicast checks, including mapped IPv4. +/// TEST-NET, CGNAT and reserved unicast ranges are not silently reclassified. +pub fn is_private_ip(address: IpAddr) -> bool { + match address { + IpAddr::V4(ip) => { + ip.is_loopback() + || ip.is_private() + || ip.is_link_local() + || ip.is_multicast() + || ip.is_unspecified() + || ip.is_broadcast() + } + IpAddr::V6(ip) => { + if let Some(ip) = ip.to_ipv4_mapped() { + return is_private_ip(IpAddr::V4(ip)); + } + ip.is_loopback() + || ip.is_unique_local() + || ip.is_unicast_link_local() + || ip.is_multicast() + || ip.is_unspecified() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn private_ranges_match_go_and_mapped_addresses() { + for address in [ + "127.0.0.1", + "10.2.3.4", + "169.254.1.1", + "172.31.0.1", + "192.168.1.1", + "224.0.0.1", + "255.255.255.255", + "0.0.0.0", + "::", + "::1", + "fc00::1", + "fe80::1", + "ff02::1", + "::ffff:127.0.0.1", + ] { + assert!(is_private_ip(address.parse().unwrap()), "{address}"); + } + for address in [ + "1.1.1.1", + "100.64.0.1", + "192.0.2.1", + "198.51.100.1", + "203.0.113.1", + "240.0.0.1", + "0.0.0.1", + "2001:db8::1", + "::127.0.0.1", + "::ffff:192.0.2.1", + ] { + assert!(!is_private_ip(address.parse().unwrap()), "{address}"); + } + } + + #[test] + fn deny_by_default_and_validate_redirects_and_actual_literal() { + let mut permissions = NetworkPermissions::default(); + assert!( + permissions + .validate("https://api.example.test/x", false, false) + .is_err() + ); + permissions.domains = vec!["*.example.test".into(), "127.1".into(), "localhost".into()]; + assert!( + permissions + .validate("https://api.example.test/x/../y", false, false) + .is_ok() + ); + assert!(!permissions.allows_domain("example.test")); + assert!(!permissions.allows_domain("badexample.test")); + assert!( + permissions + .validate("https://127.1/x", false, false) + .unwrap_err() + .contains("private/local") + ); + assert!( + permissions + .validate("https://user@api.example.test", false, false) + .is_err() + ); + assert_eq!( + permissions + .validate("http://api.example.test", false, true) + .unwrap_err(), + "redirect blocked: only https is allowed" + ); + assert!( + permissions + .validate("https://localhost", true, false) + .is_ok() + ); + } +} diff --git a/rust_backend/crates/network/src/query.rs b/rust_backend/crates/network/src/query.rs new file mode 100644 index 00000000..36ca9dbd --- /dev/null +++ b/rust_backend/crates/network/src/query.rs @@ -0,0 +1,79 @@ +//! Go net/url query encoding, including invalid-pair and semicolon handling. + +use std::collections::BTreeMap; + +pub type Query = BTreeMap<Vec<u8>, Vec<Vec<u8>>>; + +pub fn parse(query: &str) -> Query { + let mut values = Query::new(); + for pair in query + .split('&') + .filter(|pair| !pair.is_empty() && !pair.contains(';')) + { + let (key, value) = pair.split_once('=').unwrap_or((pair, "")); + if let (Some(key), Some(value)) = (decode(key), decode(value)) { + values.entry(key).or_default().push(value); + } + } + values +} + +pub fn set(values: &mut Query, key: &str, value: &str) { + values.insert(key.as_bytes().to_vec(), vec![value.as_bytes().to_vec()]); +} + +pub fn encode(values: &Query) -> String { + values + .iter() + .flat_map(|(key, values)| { + values + .iter() + .map(move |value| format!("{}={}", escape(key), escape(value))) + }) + .collect::<Vec<_>>() + .join("&") +} + +fn escape(value: &[u8]) -> String { + let mut result = String::new(); + for byte in value { + if byte.is_ascii_alphanumeric() || b"-_.~".contains(byte) { + result.push(char::from(*byte)); + } else if *byte == b' ' { + result.push('+'); + } else { + use std::fmt::Write; + let _ = write!(result, "%{byte:02X}"); + } + } + result +} + +fn decode(value: &str) -> Option<Vec<u8>> { + let mut result = Vec::new(); + let mut bytes = value.bytes(); + while let Some(byte) = bytes.next() { + result.push(match byte { + b'+' => b' ', + b'%' => { + (char::from(bytes.next()?).to_digit(16)? * 16 + + char::from(bytes.next()?).to_digit(16)?) as u8 + } + _ => byte, + }); + } + Some(result) +} + +#[cfg(test)] +mod tests { + #[test] + fn query_keeps_bytes_duplicates_and_go_escaping() { + assert_eq!( + super::encode(&super::parse( + "z=one+two&x=%FF&x=%2f&bad=%Q0&semi=a;b&empty&~!=*" + )), + "empty=&x=%FF&x=%2F&z=one+two&~%21=%2A" + ); + } +} diff --git a/go_backend/tls_roots.go b/rust_backend/crates/network/src/roots.pem similarity index 65% rename from go_backend/tls_roots.go rename to rust_backend/crates/network/src/roots.pem index 85b5edd4..7bcfa88d 100644 --- a/go_backend/tls_roots.go +++ b/rust_backend/crates/network/src/roots.pem @@ -1,12 +1,4 @@ -package gobackend - -import ( - "crypto/tls" - "crypto/x509" - "sync" -) - -const isrgRootX1PEM = `-----BEGIN CERTIFICATE----- +-----BEGIN CERTIFICATE----- MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 @@ -36,9 +28,8 @@ oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq 4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= ------END CERTIFICATE-----` - -const isrgRootX2PEM = `-----BEGIN CERTIFICATE----- +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00 @@ -51,42 +42,4 @@ AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1 /q4AaOeMSQ+2b1tbFfLn ------END CERTIFICATE-----` - -var ( - supplementalRootCAsOnce sync.Once - supplementalRootCAsPool *x509.CertPool -) - -func supplementalRootCAs() *x509.CertPool { - supplementalRootCAsOnce.Do(func() { - pool, err := x509.SystemCertPool() - if err != nil || pool == nil { - pool = x509.NewCertPool() - } - - for _, pem := range []string{isrgRootX1PEM, isrgRootX2PEM} { - pool.AppendCertsFromPEM([]byte(pem)) - } - supplementalRootCAsPool = pool - }) - - return supplementalRootCAsPool -} - -// stdTLSSessionCache is shared by every stdlib transport so TLS session -// tickets enable resumption (fewer handshake round-trips) after idle-pool -// evictions and network switches. Package-level so toggling compatibility -// options does not drop accumulated sessions. -var stdTLSSessionCache = tls.NewLRUClientSessionCache(64) - -func newTLSCompatibilityConfig(_ bool) *tls.Config { - return &tls.Config{ - RootCAs: supplementalRootCAs(), - // Hostname and certificate verification are never disabled globally. - // Compatibility fallbacks must not turn every API/download connection - // into an unauthenticated channel. - InsecureSkipVerify: false, - ClientSessionCache: stdTLSSessionCache, - } -} +-----END CERTIFICATE----- diff --git a/rust_backend/crates/network/src/tls.rs b/rust_backend/crates/network/src/tls.rs new file mode 100644 index 00000000..ff7cd647 --- /dev/null +++ b/rust_backend/crates/network/src/tls.rs @@ -0,0 +1,73 @@ +use rustls::ClientConfig; +use rustls::pki_types::CertificateDer; +use std::io; +use std::sync::Arc; + +pub(crate) fn configuration(extra_roots: &[CertificateDer<'static>]) -> io::Result<ClientConfig> { + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let builder = ClientConfig::builder_with_provider(Arc::clone(&provider)) + .with_safe_default_protocol_versions() + .map_err(io::Error::other)?; + let supplemental = rustls_pemfile::certs(&mut include_bytes!("roots.pem").as_slice()) + .collect::<Result<Vec<_>, _>>()?; + + #[cfg(target_vendor = "apple")] + let builder = { + // This is the OS certificate and hostname verifier, including iOS's + // system trust store. It never skips certificate verification. + let mut supplemental = supplemental; + supplemental.extend_from_slice(extra_roots); + let verifier = + rustls_platform_verifier::Verifier::new_with_extra_roots(supplemental, provider) + .map_err(io::Error::other)?; + builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(verifier)) + }; + #[cfg(not(target_vendor = "apple"))] + let builder = { + let load = || { + let mut roots = rustls::RootCertStore::empty(); + roots.add_parsable_certificates(rustls_native_certs::load_native_certs().certs); + #[cfg(target_os = "android")] + for directory in [ + "/system/etc/security/cacerts", + "/data/misc/keychain/certs-added", + "/apex/com.android.conscrypt/cacerts", + ] { + if let Ok(entries) = std::fs::read_dir(directory) { + for entry in entries.flatten() { + if let Ok(pem) = std::fs::read(entry.path()) { + roots.add_parsable_certificates( + rustls_pemfile::certs(&mut pem.as_slice()).flatten(), + ); + } + } + } + } + roots.add_parsable_certificates(supplemental); + Arc::new(roots) + }; + // Match Go's process-wide system CA snapshot on Android. Only immutable + // trust anchors are shared; caller-supplied roots never enter this cache. + #[cfg(target_os = "android")] + let roots = { + static ROOTS: std::sync::OnceLock<Arc<rustls::RootCertStore>> = + std::sync::OnceLock::new(); + Arc::clone(ROOTS.get_or_init(load)) + }; + #[cfg(not(target_os = "android"))] + let roots = load(); + let roots = if extra_roots.is_empty() { + roots + } else { + let mut scoped = (*roots).clone(); + scoped.add_parsable_certificates(extra_roots.iter().cloned()); + Arc::new(scoped) + }; + builder.with_root_certificates(roots) + }; + let mut config = builder.with_no_client_auth(); + config.resumption = rustls::client::Resumption::in_memory_sessions(64); + Ok(config) +} diff --git a/rust_backend/crates/network/src/url.rs b/rust_backend/crates/network/src/url.rs new file mode 100644 index 00000000..08c1648b --- /dev/null +++ b/rust_backend/crates/network/src/url.rs @@ -0,0 +1,410 @@ +//! net/url-compatible parsing for permissions and HTTP paths. The HTTP request +//! keeps escaped path bytes; browser-style normalization would change signatures. + +use std::net::Ipv6Addr; + +#[derive(Clone, Debug)] +pub struct UrlParts { + pub scheme: String, + pub hostname: String, + pub path: Vec<u8>, + pub raw_path: String, + pub raw_query: String, + pub force_query: bool, + pub fragment: String, + pub port: Option<String>, + pub has_credentials: bool, +} + +impl UrlParts { + pub fn parse(input: &str) -> Option<Self> { + let (input, fragment) = input.split_once('#').unwrap_or((input, "")); + decode(fragment, Escape::Path)?; + if input.bytes().any(|byte| byte < 32 || byte == 127) { + return None; + } + let mut scheme = ""; + let mut rest = input; + for (index, byte) in input.bytes().enumerate() { + if byte == b':' { + if index == 0 { + return None; + } + scheme = &input[..index]; + rest = &input[index + 1..]; + break; + } + if !byte.is_ascii_alphabetic() + && !(index > 0 && (byte.is_ascii_digit() || b"+-.".contains(&byte))) + { + break; + } + } + let (path, query) = rest.split_once('?').unwrap_or((rest, "")); + let force_query = rest.ends_with('?') && query.is_empty(); + rest = path; + let mut hostname = String::new(); + let mut port = None; + let mut has_credentials = false; + if !rest.starts_with('/') { + if !scheme.is_empty() { + return Some(Self { + scheme: scheme.to_lowercase(), + hostname, + path: vec![], + raw_path: rest.to_owned(), + raw_query: query.to_owned(), + force_query, + fragment: fragment.to_owned(), + port, + has_credentials, + }); + } + if rest.split('/').next()?.contains(':') { + return None; + } + } + if rest.starts_with("//") && (!scheme.is_empty() || !rest.starts_with("///")) { + let authority_and_path = &rest[2..]; + let end = authority_and_path + .find('/') + .unwrap_or(authority_and_path.len()); + let authority = &authority_and_path[..end]; + rest = &authority_and_path[end..]; + let host = if let Some((user, host)) = authority.rsplit_once('@') { + has_credentials = true; + if !user.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || b"-._~!$&'()*+,;=:%@".contains(&byte) + }) { + return None; + } + decode(user, Escape::Path)?; + host + } else { + authority + }; + hostname = parse_host(host, scheme)?; + let port_part = if host.starts_with('[') { + &host[host.rfind(']')? + 1..] + } else { + host.rfind(':').map_or("", |index| &host[index..]) + }; + port = port_part.strip_prefix(':').map(str::to_owned); + } + let path = if rest.is_empty() { + b"/".to_vec() + } else { + decode(rest, Escape::Path)? + }; + Some(Self { + scheme: scheme.to_lowercase(), + hostname, + path, + raw_path: rest.to_owned(), + raw_query: query.to_owned(), + force_query, + fragment: fragment.to_owned(), + port, + has_credentials, + }) + } + + pub fn authority(&self) -> String { + let host = if self.hostname.contains(':') { + format!("[{}]", self.hostname) + } else { + self.hostname.clone() + }; + match &self.port { + Some(port) if !port.is_empty() => format!("{host}:{port}"), + _ => host, + } + } + + pub fn display_url(&self) -> String { + let mut result = format!( + "{}://{}{}", + self.scheme, + escape(&self.authority(), true), + self.escaped_path() + ); + if self.force_query || !self.raw_query.is_empty() { + result.push('?'); + result.push_str(&self.raw_query); + } + if !self.fragment.is_empty() { + result.push('#'); + result.push_str(&escape(&self.fragment, false)); + } + result + } + + pub fn escaped_path(&self) -> String { + if self + .raw_path + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-._~!$&'()*+,;=:[]/%@".contains(&byte)) + { + return self.raw_path.clone(); + } + // net/url ignores RawPath if it contains unescaped Unicode/spaces. + // Re-encode the decoded path as a whole, including percent escapes. + let mut result = String::new(); + for byte in decode(&self.raw_path, Escape::Path).unwrap_or_default() { + if byte.is_ascii_alphanumeric() || b"-._~$&+,/:;=@".contains(&byte) { + result.push(char::from(byte)); + } else { + use std::fmt::Write; + let _ = write!(result, "%{byte:02X}"); + } + } + result + } + + pub fn request_uri(&self) -> Result<http::Uri, String> { + if let Some(port) = self.port.as_deref().filter(|port| !port.is_empty()) { + port.parse::<u16>() + .map_err(|_| "invalid URL port".to_owned())?; + } + let hostname = if self.hostname.contains(':') { + format!("[{}]", self.hostname) + } else { + // URL's domain parser supplies IDNA; the path is deliberately not + // passed through its browser-style path parser. + ::url::Host::parse(&self.hostname) + .map_err(|error| error.to_string())? + .to_string() + }; + let authority = match self.port.as_deref().filter(|port| !port.is_empty()) { + Some(port) => format!("{hostname}:{port}"), + None => hostname, + }; + let mut path = if self.raw_path.is_empty() { + "/".to_owned() + } else { + self.escaped_path() + }; + if self.force_query || !self.raw_query.is_empty() { + path.push('?'); + path.push_str(&self.raw_query); + } + http::Uri::builder() + .scheme(self.scheme.as_str()) + .authority(authority) + .path_and_query(path) + .build() + .map_err(|error| error.to_string()) + } + + pub fn resolve(&self, location: &str) -> Option<Self> { + let mut target = Self::parse(location)?; + let absolute = + !target.scheme.is_empty() || !target.hostname.is_empty() || target.has_credentials; + if target.scheme.is_empty() { + target.scheme.clone_from(&self.scheme); + } + if absolute { + target.raw_path = resolve_path(&target.escaped_path(), ""); + } else { + if target.raw_path.is_empty() && !target.force_query && target.raw_query.is_empty() { + target.raw_query.clone_from(&self.raw_query); + if target.fragment.is_empty() { + target.fragment.clone_from(&self.fragment); + } + } + target.hostname.clone_from(&self.hostname); + target.port.clone_from(&self.port); + target.has_credentials = self.has_credentials; + target.raw_path = resolve_path(&self.escaped_path(), &target.escaped_path()); + } + target.path = decode(&target.raw_path, Escape::Path)?; + Some(target) + } +} + +// RFC 3986 dot segments apply to escaped paths. In particular, %2e%2e and +// %2f remain escaped rather than becoming browser-style traversal segments. +fn resolve_path(base: &str, reference: &str) -> String { + let full = if reference.is_empty() { + base.to_owned() + } else if reference.starts_with('/') { + reference.to_owned() + } else { + format!( + "{}{reference}", + &base[..base.rfind('/').map_or(0, |index| index + 1)] + ) + }; + if full.is_empty() { + return full; + } + let mut result = String::from("/"); + let mut first = true; + let mut last = ""; + for part in full.split('/') { + last = part; + match part { + "." => first = false, + ".." => { + result.truncate(result[1..].rfind('/').map_or(1, |index| index + 1)); + first = result.len() == 1; + } + _ => { + if !first { + result.push('/'); + } + result.push_str(part); + first = false; + } + } + } + if last == "." || last == ".." { + result.push('/'); + } + if result.starts_with("//") { + result.remove(0); + } + result +} + +fn escape(value: &str, host: bool) -> String { + let mut result = String::with_capacity(value.len()); + for byte in value.bytes() { + let allowed = byte.is_ascii_alphanumeric() + || b"-._~!$&'()*+,;=:[]".contains(&byte) + || (!host && b"/%@?".contains(&byte)); + if allowed { + result.push(char::from(byte)); + } else { + use std::fmt::Write; + let _ = write!(result, "%{byte:02X}"); + } + } + result +} + +fn valid_port(port: &str) -> bool { + port.is_empty() + || port + .strip_prefix(':') + .is_some_and(|port| port.bytes().all(|byte| byte.is_ascii_digit())) +} + +fn parse_host(host: &str, scheme: &str) -> Option<String> { + if let Some(open) = host.rfind('[') { + if open != 0 { + return None; + } + let close = host.rfind(']')?; + if !valid_port(&host[close + 1..]) { + return None; + } + let raw = &host[1..close]; + let decoded = if let Some((address, zone)) = raw.split_once("%25") { + let mut bytes = decode(address, Escape::Host)?; + bytes.extend(decode(&format!("%25{zone}"), Escape::Zone)?); + bytes + } else { + decode(raw, Escape::Host)? + }; + let hostname = String::from_utf8_lossy(&decoded).into_owned(); + let address = if let Some((address, zone)) = hostname.split_once('%') { + if zone.is_empty() { + return None; + } + address + } else { + hostname.as_str() + }; + address.parse::<Ipv6Addr>().ok()?; + return Some(hostname); + } + let mut hostname = host; + if let Some(first) = host.find(':') { + let index = if scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https") { + first + } else { + host.rfind(':')? + }; + if !valid_port(&host[index..]) { + return None; + } + hostname = &host[..index]; + } + Some(String::from_utf8_lossy(&decode(hostname, Escape::Host)?).into_owned()) +} + +#[derive(Clone, Copy)] +enum Escape { + Path, + Host, + Zone, +} + +fn valid_host_byte(byte: u8) -> bool { + byte >= 128 || byte.is_ascii_alphanumeric() || b"-._~!$&'()*+,;=:[]<>\"".contains(&byte) +} + +fn decode(input: &str, mode: Escape) -> Option<Vec<u8>> { + let mut result = Vec::with_capacity(input.len()); + let mut bytes = input.bytes(); + while let Some(byte) = bytes.next() { + if byte == b'%' { + let high = char::from(bytes.next()?).to_digit(16)?; + let low = char::from(bytes.next()?).to_digit(16)?; + let value = (high * 16 + low) as u8; + match mode { + Escape::Host if value < 128 && value != b'%' => return None, + Escape::Zone if value != b'%' && value != b' ' && !valid_host_byte(value) => { + return None; + } + _ => result.push(value), + } + } else { + if !matches!(mode, Escape::Path) && !valid_host_byte(byte) { + return None; + } + result.push(byte); + } + } + Some(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn raw_requests_and_relative_redirect_paths_keep_go_semantics() { + let base = UrlParts::parse("https://example.test/a/../b/%2e%2e/x?old#fragment?").unwrap(); + assert_eq!( + base.request_uri() + .unwrap() + .path_and_query() + .unwrap() + .as_str(), + "/a/../b/%2e%2e/x?old" + ); + for (reference, expected) in [ + ("../z", "https://example.test/b/z"), + ("%2e%2e/z", "https://example.test/b/%2e%2e/%2e%2e/z"), + ("/a//b/../c", "https://example.test/a//c"), + ("?", "https://example.test/b/%2e%2e/x?"), + ("#new?", "https://example.test/b/%2e%2e/x?old#new?"), + ] { + assert_eq!( + base.resolve(reference).unwrap().display_url(), + expected, + "{reference}" + ); + } + assert_eq!( + UrlParts::parse("https://example.test?") + .unwrap() + .request_uri() + .unwrap() + .to_string(), + "https://example.test/?" + ); + } +} diff --git a/rust_backend/crates/providers/Cargo.toml b/rust_backend/crates/providers/Cargo.toml new file mode 100644 index 00000000..cb13e2cc --- /dev/null +++ b/rust_backend/crates/providers/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "spotiflac-providers" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +spotiflac-core.workspace = true +spotiflac-network.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +url.workspace = true +tempfile.workspace = true +rustix.workspace = true +base64.workspace = true +regex.workspace = true +scraper.workspace = true +html5ever.workspace = true +httpdate.workspace = true + +[lints] +workspace = true diff --git a/rust_backend/crates/providers/src/deezer.rs b/rust_backend/crates/providers/src/deezer.rs new file mode 100644 index 00000000..ab289eba --- /dev/null +++ b/rust_backend/crates/providers/src/deezer.rs @@ -0,0 +1,222 @@ +//! Deezer metadata used by native lookups, browsing and platform resolution. + +mod browse; +mod cache; +mod extended; +mod search; +pub use extended::parse_url; + +use crate::resolver::{Check, ResolverError, http::ResolverHttp}; +use serde::de::DeserializeOwned; +use spotiflac_core::{ + lyrics, + metadata::{ + TrackMetadata, + deezer::{Track, Tracks}, + }, +}; +use spotiflac_network::NetworkService; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime}; + +pub trait MetadataLookup: Send + Sync + 'static { + /// Observe `check` during HTTP calls and retry waits so native shutdown can + /// wait for outstanding metadata requests to release their resources. + fn search_by_isrc(&self, isrc: &str, check: &Check<'_>) + -> Result<TrackMetadata, ResolverError>; +} + +pub struct DeezerClient { + http: ResolverHttp, + language: Mutex<String>, + cache: Mutex<cache::Cache>, + flights: Mutex<BTreeMap<String, Arc<cache::Flight>>>, +} + +impl DeezerClient { + pub fn new(network: &Arc<NetworkService>) -> Self { + Self::with_endpoint(network, "https://api.deezer.com").expect("default Deezer endpoint") + } + + pub fn with_endpoint( + network: &Arc<NetworkService>, + endpoint: &str, + ) -> Result<Self, ResolverError> { + Ok(Self { + http: ResolverHttp::new( + network, + BTreeMap::from([("https://api.deezer.com".into(), endpoint.into())]), + Duration::from_secs(25), + )?, + language: Mutex::new(String::new()), + cache: Mutex::new(cache::Cache::default()), + flights: Mutex::new(BTreeMap::new()), + }) + } + + pub fn set_language(&self, tag: &str) { + *self.language.lock().unwrap() = tag.trim().into(); + } + + pub fn get_track(&self, id: &str, check: &Check<'_>) -> Result<TrackMetadata, ResolverError> { + let track: Track = + self.get_json(&format!("https://api.deezer.com/2.0/track/{id}"), check)?; + Ok(track.metadata()) + } + + fn get_json<T: DeserializeOwned>( + &self, + endpoint: &str, + check: &Check<'_>, + ) -> Result<T, ResolverError> { + if endpoint.len() > 64 << 10 { + return Err(ResolverError::Failed("metadata URL exceeds limit".into())); + } + let mut retry_after = Duration::ZERO; + for attempt in 0..3 { + if attempt > 0 { + let delay = if retry_after.is_zero() { + Duration::from_millis(500 << (attempt - 1)) + } else { + retry_after + }; + let started = Instant::now(); + while started.elapsed() < delay { + check().map_err(ResolverError::Cancelled)?; + std::thread::sleep( + delay + .saturating_sub(started.elapsed()) + .min(Duration::from_millis(25)), + ); + } + } + check().map_err(ResolverError::Cancelled)?; + let language = self.language.lock().unwrap().clone(); + let language = if language.is_empty() || language.to_ascii_lowercase().starts_with("en") + { + "en-US,en;q=0.9".into() + } else { + format!("{language},en;q=0.8") + }; + let response = match self.http.metadata(endpoint, &language, check) { + Ok(response) => response, + Err(ResolverError::Transport(_)) if attempt < 2 => { + retry_after = Duration::ZERO; + continue; + } + Err(error) => return Err(error), + }; + if response.status == 200 { + let result = lyrics::decode_document(&response.body); + check().map_err(ResolverError::Cancelled)?; + return result.map_err(|error| ResolverError::Failed(error.to_string())); + } + if attempt < 2 && (response.status == 429 || response.status >= 500) { + retry_after = response + .headers + .get("retry-after") + .and_then(|values| values.first()) + .map(|value| retry_delay(value, SystemTime::now())) + .unwrap_or_default(); + continue; + } + return Err(ResolverError::Failed(format!( + "deezer API returned status {}: {}", + response.status, + lyrics::text_from_bytes(&response.body) + ))); + } + unreachable!("last metadata attempt returns its result") + } +} + +impl MetadataLookup for DeezerClient { + fn search_by_isrc( + &self, + isrc: &str, + check: &Check<'_>, + ) -> Result<TrackMetadata, ResolverError> { + let track = self.get_json::<Track>( + &format!("https://api.deezer.com/2.0/track/isrc:{isrc}"), + check, + ); + match track { + Ok(track) if track.id != 0 => Ok(track.metadata()), + Ok(_) => Err(ResolverError::Failed(format!( + "no track found for ISRC: {isrc}" + ))), + Err(error @ ResolverError::Cancelled(_)) => Err(error), + Err(_) => { + let tracks: Tracks = self.get_json( + &format!("https://api.deezer.com/2.0/search/track?q=isrc:{isrc}&limit=1"), + check, + )?; + tracks + .data + .as_deref() + .unwrap_or_default() + .first() + .map(Track::metadata) + .ok_or_else(|| { + ResolverError::Failed(format!("no track found for ISRC: {isrc}")) + }) + } + } + } +} + +pub fn retry_delay(value: &str, now: SystemTime) -> Duration { + if let Ok(seconds) = value.parse::<isize>() { + let nanos = (seconds as i64) + .wrapping_mul(1_000_000_000) + .min(120_000_000_000); + return Duration::from_nanos(nanos.max(0) as u64); + } + httpdate::parse_http_date(value) + .ok() + .and_then(|date| date.duration_since(now).ok()) + .unwrap_or_default() + .min(Duration::from_secs(120)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retry_after_preserves_integer_overflow_dates_and_two_minute_cap() { + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); + for (input, expected) in [ + ("", 0), + ("0", 0), + ("1", 1), + ("+2", 2), + ("-1", 0), + ("121", 120), + (" 1", 0), + ("9223372036854775807", 0), + ] { + assert_eq!( + retry_delay(input, now), + Duration::from_secs(expected), + "{input}" + ); + } + assert_eq!( + retry_delay(&httpdate::fmt_http_date(now + Duration::from_secs(30)), now), + Duration::from_secs(30) + ); + assert_eq!( + retry_delay( + &httpdate::fmt_http_date(now + Duration::from_secs(300)), + now + ), + Duration::from_secs(120) + ); + assert_eq!( + retry_delay(&httpdate::fmt_http_date(now - Duration::from_secs(30)), now), + Duration::ZERO + ); + } +} diff --git a/rust_backend/crates/providers/src/deezer/browse.rs b/rust_backend/crates/providers/src/deezer/browse.rs new file mode 100644 index 00000000..de1e0c93 --- /dev/null +++ b/rust_backend/crates/providers/src/deezer/browse.rs @@ -0,0 +1,359 @@ +use super::{ + Check, DeezerClient, ResolverError, + cache::{Bucket, Value}, +}; +use spotiflac_core::metadata::{ + AlbumInfoMetadata, AlbumResponsePayload, AlbumTrackMetadata, ArtistAlbumMetadata, + ArtistInfoMetadata, ArtistResponsePayload, PlaylistInfoMetadata, PlaylistOwner, + PlaylistResponsePayload, PlaylistTrackCount, + deezer::{ + self, AlbumTrackCount, ArtistAlbums, FullAlbum, FullArtist, FullPlaylist, Track, TrackPage, + }, +}; +use std::collections::BTreeMap; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, +}; +use std::time::Instant; + +impl DeezerClient { + pub fn get_album( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<Arc<AlbumResponsePayload>, ResolverError> { + check().map_err(ResolverError::Cancelled)?; + if let Some(Value::Album(value)) = self.cached(Bucket::Album, id) { + return Ok(value); + } + let album: FullAlbum = + self.get_json(&format!("https://api.deezer.com/2.0/album/{id}"), check)?; + let image = album.image(); + let artists = match album.contributors.as_deref() { + Some(values) if !values.is_empty() => values + .iter() + .map(|artist| artist.name.as_str()) + .collect::<Vec<_>>() + .join(", "), + _ => album.artist.name.clone(), + }; + let info = AlbumInfoMetadata { + total_tracks: album.nb_tracks, + name: album.title.clone(), + release_date: album.release_date.clone(), + artists: artists.clone(), + artist_id: format!("deezer:{}", album.artist.id), + images: image.clone(), + genre: album.genres.display(), + label: album.label.clone(), + ..Default::default() + }; + let mut tracks = album.tracks.data.unwrap_or_default(); + self.remaining_tracks("album", id, album.nb_tracks, &mut tracks, check)?; + let isrcs = self.track_isrcs(&tracks, check)?; + let discs = tracks + .iter() + .map(|track| track.disk_number) + .max() + .unwrap_or_default() + .max(0); + let mut result = AlbumResponsePayload { + album_info: info, + track_list: Vec::with_capacity(tracks.len()), + }; + for (index, track) in tracks.iter().enumerate() { + check().map_err(ResolverError::Cancelled)?; + result.track_list.push(AlbumTrackMetadata { + spotify_id: format!("deezer:{}", track.id), + artists: track.artist_display(), + name: track.title.clone(), + album_name: album.title.clone(), + album_artist: artists.clone(), + duration_ms: track.duration.wrapping_mul(1000), + images: image.clone(), + release_date: album.release_date.clone(), + track_number: if track.track_position == 0 { + (index + 1) as isize + } else { + track.track_position + }, + total_tracks: album.nb_tracks, + disc_number: track.disk_number, + total_discs: discs, + external_urls: track.link.clone(), + isrc: isrcs + .get(&track.id.to_string()) + .cloned() + .unwrap_or_default(), + album_id: format!("deezer:{}", album.id), + album_type: deezer::album_type(&album.record_type), + explicit: track.is_explicit(), + ..Default::default() + }); + } + check().map_err(ResolverError::Cancelled)?; + let result = Arc::new(result); + self.store(Bucket::Album, id.into(), Value::Album(result.clone())); + Ok(result) + } + + pub fn get_artist( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<Arc<ArtistResponsePayload>, ResolverError> { + check().map_err(ResolverError::Cancelled)?; + if let Some(Value::Artist(value)) = self.cached(Bucket::Artist, id) { + return Ok(value); + } + let artist: FullArtist = + self.get_json(&format!("https://api.deezer.com/2.0/artist/{id}"), check)?; + let mut result = ArtistResponsePayload { + artist_info: ArtistInfoMetadata { + id: format!("deezer:{}", artist.id), + name: artist.name.clone(), + images: artist.image(), + followers: artist.nb_fan, + popularity: 0, + }, + albums: Vec::new(), + }; + match self.get_json::<ArtistAlbums>( + &format!("https://api.deezer.com/2.0/artist/{id}/albums?limit=100"), + check, + ) { + Ok(response) => { + for album in response.data.unwrap_or_default() { + check().map_err(ResolverError::Cancelled)?; + result.albums.push(ArtistAlbumMetadata { + id: format!("deezer:{}", album.id), + name: album.title.clone(), + release_date: album.release_date.clone(), + total_tracks: album.nb_tracks, + images: album.image(), + album_type: deezer::album_type(&album.record_type), + artists: artist.name.clone(), + }); + } + let missing: Vec<_> = result + .albums + .iter() + .enumerate() + .filter(|(_, album)| album.total_tracks == 0) + .map(|(index, album)| { + (index, album.id.trim_start_matches("deezer:").to_owned()) + }) + .collect(); + for (index, count) in parallel(&missing, |(index, id)| { + let count = self + .get_json::<AlbumTrackCount>( + &format!("https://api.deezer.com/2.0/album/{id}"), + check, + ) + .ok() + .map(|album| album.nb_tracks); + (*index, count) + }) { + if let Some(count) = count { + result.albums[index].total_tracks = count; + } + } + } + Err(error @ (ResolverError::Cancelled(_) | ResolverError::Closed)) => { + return Err(error); + } + Err(_) => {} + } + check().map_err(ResolverError::Cancelled)?; + let result = Arc::new(result); + self.store(Bucket::Artist, id.into(), Value::Artist(result.clone())); + Ok(result) + } + + pub fn get_playlist( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<PlaylistResponsePayload, ResolverError> { + let playlist: FullPlaylist = + self.get_json(&format!("https://api.deezer.com/2.0/playlist/{id}"), check)?; + // Unlike album/search, Go does not fall back to the smallest image here. + let image = deezer::best_image([ + playlist.picture_xl.as_str(), + &playlist.picture_big, + &playlist.picture_medium, + ]); + let info = PlaylistInfoMetadata { + tracks: PlaylistTrackCount { + total: playlist.nb_tracks, + }, + owner: PlaylistOwner { + display_name: playlist.creator.name, + name: playlist.title, + images: image, + }, + ..Default::default() + }; + let mut tracks = playlist.tracks.data.unwrap_or_default(); + self.remaining_tracks("playlist", id, playlist.nb_tracks, &mut tracks, check)?; + let isrcs = self.track_isrcs(&tracks, check)?; + let mut result = PlaylistResponsePayload { + playlist_info: info, + track_list: Vec::with_capacity(tracks.len()), + }; + for track in tracks { + check().map_err(ResolverError::Cancelled)?; + result.track_list.push(AlbumTrackMetadata { + spotify_id: format!("deezer:{}", track.id), + artists: track.artist_display(), + name: track.title.clone(), + album_name: track.album.title.clone(), + album_artist: track.artist.name.clone(), + duration_ms: track.duration.wrapping_mul(1000), + images: deezer::best_image([ + track.album.cover_xl.as_str(), + &track.album.cover_big, + &track.album.cover_medium, + ]), + track_number: track.track_position, + disc_number: track.disk_number, + external_urls: track.link.clone(), + isrc: isrcs + .get(&track.id.to_string()) + .cloned() + .unwrap_or_default(), + album_id: format!("deezer:{}", track.album.id), + explicit: track.is_explicit(), + ..Default::default() + }); + } + Ok(result) + } + + fn remaining_tracks( + &self, + kind: &str, + id: &str, + total: isize, + tracks: &mut Vec<Track>, + check: &Check<'_>, + ) -> Result<(), ResolverError> { + let mut next = format!( + "https://api.deezer.com/2.0/{kind}/{id}/tracks?limit=100&index={}", + tracks.len() + ); + while (tracks.len() as i128) < total as i128 { + match self.get_json::<TrackPage>(&next, check) { + Ok(page) => { + let data = page.data.unwrap_or_default(); + if data.is_empty() { + break; + } + tracks.extend(data); + if page.next.is_empty() { + break; + } + next = page.next; + } + Err(error @ (ResolverError::Cancelled(_) | ResolverError::Closed)) => { + return Err(error); + } + Err(_) => break, + } + } + check().map_err(ResolverError::Cancelled) + } + + fn track_isrcs( + &self, + tracks: &[Track], + check: &Check<'_>, + ) -> Result<BTreeMap<String, String>, ResolverError> { + let mut result = BTreeMap::new(); + let mut missing = Vec::new(); + check().map_err(ResolverError::Cancelled)?; + { + let mut cache = self.cache.lock().unwrap(); + let mut direct = BTreeMap::new(); + for track in tracks { + let id = track.id.to_string(); + if !track.isrc.is_empty() { + result.insert(id.clone(), track.isrc.clone()); + if !cache.isrc.contains_key(&id) { + direct.insert(id, track.isrc.clone()); + } + } else if let Some(isrc) = cache.isrc.get(&id) { + result.insert(id, isrc.clone()); + } else { + missing.push(id); + } + } + cache.isrc.extend(direct); + cache.cleanup(Instant::now()); + } + for (id, isrc) in parallel(&missing, |id| { + let value = self + .get_json::<Track>(&format!("https://api.deezer.com/2.0/track/{id}"), check) + .ok(); + if let Some(track) = &value { + let mut cache = self.cache.lock().unwrap(); + cache.isrc.insert(id.clone(), track.isrc.clone()); + cache.cleanup(Instant::now()); + } + (id.clone(), value.map(|track| track.isrc)) + }) { + if let Some(isrc) = isrc { + result.insert(id, isrc); + } + } + check().map_err(ResolverError::Cancelled)?; + Ok(result) + } +} + +/// Keep both OS thread count and HTTP concurrency bounded, including very +/// large playlists. Output retains input order, independent of worker timing. +fn parallel<T: Sync, R: Send>(items: &[T], fetch: impl Fn(&T) -> R + Sync) -> Vec<R> { + let next = AtomicUsize::new(0); + let result = Mutex::new(Vec::with_capacity(items.len())); + std::thread::scope(|scope| { + for _ in 0..items.len().min(10) { + scope.spawn(|| { + loop { + let index = next.fetch_add(1, Ordering::Relaxed); + let Some(item) = items.get(index) else { + break; + }; + let value = fetch(item); + result.lock().unwrap().push((index, value)); + } + }); + } + }); + let mut result = result.into_inner().unwrap(); + result.sort_by_key(|(index, _)| *index); + result.into_iter().map(|(_, result)| result).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parallel_metadata_work_overlaps_without_exceeding_ten_workers_or_reordering_output() { + let active = AtomicUsize::new(0); + let peak = AtomicUsize::new(0); + let items: Vec<_> = (0..60).collect(); + let result = parallel(&items, |index| { + let count = active.fetch_add(1, Ordering::AcqRel) + 1; + peak.fetch_max(count, Ordering::AcqRel); + std::thread::sleep(std::time::Duration::from_millis(10)); + active.fetch_sub(1, Ordering::AcqRel); + *index + }); + assert_eq!(result, items); + assert!((2..=10).contains(&peak.load(Ordering::Acquire))); + assert_eq!(active.load(Ordering::Acquire), 0); + } +} diff --git a/rust_backend/crates/providers/src/deezer/cache.rs b/rust_backend/crates/providers/src/deezer/cache.rs new file mode 100644 index 00000000..8b1d9553 --- /dev/null +++ b/rust_backend/crates/providers/src/deezer/cache.rs @@ -0,0 +1,397 @@ +use super::{Check, DeezerClient, ResolverError}; +use spotiflac_core::metadata::{ + AlbumExtendedMetadata, AlbumResponsePayload, ArtistResponsePayload, SearchAllResult, +}; +use std::collections::BTreeMap; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +#[derive(Clone)] +pub(super) enum Value { + Search(Arc<SearchAllResult>), + Album(Arc<AlbumResponsePayload>), + Artist(Arc<ArtistResponsePayload>), + Extended(Arc<AlbumExtendedMetadata>), + AlbumId(String), +} + +#[derive(Clone, Copy)] +pub(super) enum Bucket { + Search, + Album, + Artist, +} + +struct Entry { + value: Value, + expires: Instant, +} + +#[derive(Default)] +pub(super) struct Cache { + search: BTreeMap<String, Entry>, + albums: BTreeMap<String, Entry>, + artists: BTreeMap<String, Entry>, + pub isrc: BTreeMap<String, String>, + last_cleanup: Option<Instant>, +} + +impl Cache { + fn bucket(&mut self, bucket: Bucket) -> &mut BTreeMap<String, Entry> { + match bucket { + Bucket::Search => &mut self.search, + Bucket::Album => &mut self.albums, + Bucket::Artist => &mut self.artists, + } + } + + pub fn get(&mut self, bucket: Bucket, key: &str, now: Instant) -> Option<Value> { + self.bucket(bucket) + .get(key) + .filter(|entry| now <= entry.expires) + .map(|entry| entry.value.clone()) + } + + pub fn put(&mut self, bucket: Bucket, key: String, value: Value, now: Instant) { + self.bucket(bucket).insert( + key, + Entry { + value, + expires: now + Duration::from_secs(600), + }, + ); + self.cleanup(now); + } + + pub fn cleanup(&mut self, now: Instant) { + let periodic = self + .last_cleanup + .is_none_or(|last| now.saturating_duration_since(last) >= Duration::from_secs(300)); + for (bucket, limit) in [ + (Bucket::Search, 300), + (Bucket::Album, 200), + (Bucket::Artist, 200), + ] { + let entries = self.bucket(bucket); + if periodic || entries.len() > limit { + entries.retain(|_, entry| now <= entry.expires); + while entries.len() > limit { + let key = entries + .iter() + .min_by_key(|(_, entry)| entry.expires) + .unwrap() + .0 + .clone(); + entries.remove(&key); + } + } + } + while self.isrc.len() > 4000 { + self.isrc.pop_first(); + } + if periodic { + self.last_cleanup = Some(now); + } + } +} + +#[derive(Default)] +pub(super) struct Flight { + result: Mutex<Option<Result<Value, ResolverError>>>, + ready: Condvar, +} + +impl DeezerClient { + pub(super) fn cached(&self, bucket: Bucket, key: &str) -> Option<Value> { + self.cache.lock().unwrap().get(bucket, key, Instant::now()) + } + + pub(super) fn store(&self, bucket: Bucket, key: String, value: Value) { + self.cache + .lock() + .unwrap() + .put(bucket, key, value, Instant::now()); + } + + /// Only Go's two extended-metadata operations coalesce. Waiting callers + /// can cancel independently; a cancelled leader lets live waiters retry. + pub(super) fn coalesced( + &self, + key: &str, + check: &Check<'_>, + fetch: impl Fn() -> Result<Value, ResolverError>, + ) -> Result<Value, ResolverError> { + loop { + check().map_err(ResolverError::Cancelled)?; + if let Some(value) = self.cached(Bucket::Search, key) { + return Ok(value); + } + let (flight, owner) = { + let mut flights = self.flights.lock().unwrap(); + match flights.get(key) { + Some(flight) => (flight.clone(), false), + None => { + if flights.len() >= 64 { + return Err(ResolverError::Busy); + } + let flight = Arc::new(Flight::default()); + flights.insert(key.into(), flight.clone()); + (flight, true) + } + } + }; + if owner { + let result = catch_unwind(AssertUnwindSafe(|| { + let value = match self.cached(Bucket::Search, key) { + Some(value) => value, + None => fetch()?, + }; + check().map_err(ResolverError::Cancelled)?; + self.store(Bucket::Search, key.into(), value.clone()); + Ok(value) + })) + .unwrap_or_else(|_| Err(ResolverError::Failed("metadata request panicked".into()))); + self.flights.lock().unwrap().remove(key); + *flight.result.lock().unwrap() = Some(result.clone()); + flight.ready.notify_all(); + return result; + } + let mut result = flight.result.lock().unwrap(); + loop { + check().map_err(ResolverError::Cancelled)?; + if let Some(value) = result.as_ref() { + if matches!(value, Err(ResolverError::Cancelled(_))) { + break; + } + return value.clone(); + } + result = flight + .ready + .wait_timeout(result, Duration::from_millis(25)) + .unwrap() + .0; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + fn until(ready: impl Fn() -> bool) { + let started = Instant::now(); + while !ready() { + assert!(started.elapsed() < Duration::from_secs(3)); + std::thread::sleep(Duration::from_millis(1)); + } + } + + fn id(value: Result<Value, ResolverError>) -> String { + match value.unwrap() { + Value::AlbumId(value) => value, + _ => panic!("wrong cache type"), + } + } + + #[test] + fn cache_budgets_expiry_and_empty_isrc_follow_the_metadata_contract() { + let now = Instant::now(); + let mut cache = Cache::default(); + for (bucket, limit) in [ + (Bucket::Search, 300), + (Bucket::Album, 200), + (Bucket::Artist, 200), + ] { + for index in 0..=limit { + cache.put( + bucket, + index.to_string(), + Value::AlbumId(index.to_string()), + now + Duration::from_millis(index), + ); + } + assert!(cache.get(bucket, "0", now).is_none()); + assert!( + cache + .get(bucket, "1", now + Duration::from_secs(600)) + .is_some() + ); + assert!( + cache + .get(bucket, "1", now + Duration::from_secs(601)) + .is_none() + ); + assert_eq!(cache.bucket(bucket).len(), limit as usize); + } + for index in 0..4005 { + cache.isrc.insert(index.to_string(), String::new()); + } + cache.cleanup(now + Duration::from_secs(601)); + assert!(cache.search.is_empty() && cache.albums.is_empty() && cache.artists.is_empty()); + assert_eq!(cache.isrc.len(), 4000); + assert!(cache.isrc.values().all(String::is_empty)); + cache.cleanup(now + Duration::from_secs(86400)); + assert_eq!(cache.isrc.len(), 4000); + } + + #[test] + fn metadata_flights_coalesce_and_a_cancelled_waiter_does_not_cancel_the_leader() { + let network = spotiflac_network::NetworkService::new().unwrap(); + let client = DeezerClient::new(&network); + let started = AtomicBool::new(false); + let release = AtomicBool::new(false); + let cancel = AtomicBool::new(false); + let checks = AtomicUsize::new(0); + let calls = AtomicUsize::new(0); + std::thread::scope(|scope| { + let leader = scope.spawn(|| { + client.coalesced("metadata", &|| Ok(()), || { + calls.fetch_add(1, Ordering::AcqRel); + started.store(true, Ordering::Release); + until(|| release.load(Ordering::Acquire)); + Ok(Value::AlbumId("shared".into())) + }) + }); + until(|| started.load(Ordering::Acquire)); + let cancelled = scope.spawn(|| { + client.coalesced( + "metadata", + &|| { + checks.fetch_add(1, Ordering::AcqRel); + if cancel.load(Ordering::Acquire) { + Err("cancel waiter".into()) + } else { + Ok(()) + } + }, + || panic!("waiter became leader"), + ) + }); + until(|| checks.load(Ordering::Acquire) >= 2); + cancel.store(true, Ordering::Release); + assert!(matches!( + cancelled.join().unwrap(), + Err(ResolverError::Cancelled(_)) + )); + let waiters: Vec<_> = (0..24) + .map(|_| { + scope.spawn(|| { + client.coalesced("metadata", &|| Ok(()), || { + panic!("duplicate metadata lookup") + }) + }) + }) + .collect(); + release.store(true, Ordering::Release); + assert_eq!(id(leader.join().unwrap()), "shared"); + for waiter in waiters { + assert_eq!(id(waiter.join().unwrap()), "shared"); + } + }); + assert_eq!(calls.load(Ordering::Acquire), 1); + assert!(client.flights.lock().unwrap().is_empty()); + assert_eq!( + id(client.coalesced("metadata", &|| Ok(()), || panic!("cache miss"))), + "shared" + ); + } + + #[test] + fn cancelled_leaders_release_live_waiters_and_panics_leave_no_stuck_flight() { + let network = spotiflac_network::NetworkService::new().unwrap(); + let client = DeezerClient::new(&network); + let started = AtomicBool::new(false); + let cancel = AtomicBool::new(false); + let checks = AtomicUsize::new(0); + let retried = AtomicUsize::new(0); + std::thread::scope(|scope| { + let leader = scope.spawn(|| { + client.coalesced("metadata", &|| Ok(()), || { + started.store(true, Ordering::Release); + until(|| cancel.load(Ordering::Acquire)); + Err(ResolverError::Cancelled("cancel leader".into())) + }) + }); + until(|| started.load(Ordering::Acquire)); + let waiter = scope.spawn(|| { + client.coalesced( + "metadata", + &|| { + checks.fetch_add(1, Ordering::AcqRel); + Ok(()) + }, + || { + retried.fetch_add(1, Ordering::AcqRel); + Ok(Value::AlbumId("retry".into())) + }, + ) + }); + until(|| checks.load(Ordering::Acquire) >= 2); + cancel.store(true, Ordering::Release); + assert!(matches!( + leader.join().unwrap(), + Err(ResolverError::Cancelled(_)) + )); + assert_eq!(id(waiter.join().unwrap()), "retry"); + }); + assert_eq!(retried.load(Ordering::Acquire), 1); + assert!( + client + .coalesced("panic", &|| Ok(()), || panic!("fixture panic")) + .is_err() + ); + assert!(client.flights.lock().unwrap().is_empty()); + assert_eq!( + id(client.coalesced("panic", &|| Ok(()), || Ok(Value::AlbumId( + "recovered".into() + )))), + "recovered" + ); + } + + #[test] + fn distinct_pending_metadata_flights_are_bounded_and_release_their_slots() { + let network = spotiflac_network::NetworkService::new().unwrap(); + let client = DeezerClient::new(&network); + let entered = AtomicUsize::new(0); + let release = AtomicBool::new(false); + std::thread::scope(|scope| { + let workers: Vec<_> = (0..64) + .map(|index| { + let key = index.to_string(); + let client = &client; + let entered = &entered; + let release = &release; + scope.spawn(move || { + client.coalesced(&key, &|| Ok(()), || { + entered.fetch_add(1, Ordering::AcqRel); + until(|| release.load(Ordering::Acquire)); + Ok(Value::AlbumId(key.clone())) + }) + }) + }) + .collect(); + until(|| entered.load(Ordering::Acquire) == 64); + assert!(matches!( + client.coalesced("overflow", &|| Ok(()), || panic!("limit bypassed")), + Err(ResolverError::Busy) + )); + release.store(true, Ordering::Release); + for worker in workers { + assert!(!id(worker.join().unwrap()).is_empty()); + } + }); + assert!(client.flights.lock().unwrap().is_empty()); + assert_eq!( + id( + client.coalesced("overflow", &|| Ok(()), || Ok(Value::AlbumId( + "available".into() + ))) + ), + "available" + ); + } +} diff --git a/rust_backend/crates/providers/src/deezer/extended.rs b/rust_backend/crates/providers/src/deezer/extended.rs new file mode 100644 index 00000000..1a2bb155 --- /dev/null +++ b/rust_backend/crates/providers/src/deezer/extended.rs @@ -0,0 +1,202 @@ +use super::{Check, DeezerClient, MetadataLookup, ResolverError, cache::Value, search::context}; +use spotiflac_core::metadata::{ + AlbumExtendedMetadata, TrackMetadata, + deezer::{FullAlbum, Track}, +}; +use spotiflac_network::url::UrlParts; +use std::sync::Arc; +use std::time::Instant; + +impl DeezerClient { + pub fn get_track_isrc(&self, id: &str, check: &Check<'_>) -> Result<String, ResolverError> { + check().map_err(ResolverError::Cancelled)?; + if let Some(value) = self.cache.lock().unwrap().isrc.get(id).cloned() { + return Ok(value); + } + let track: Track = + self.get_json(&format!("https://api.deezer.com/2.0/track/{id}"), check)?; + let mut cache = self.cache.lock().unwrap(); + cache.isrc.insert(id.into(), track.isrc.clone()); + cache.cleanup(Instant::now()); + Ok(track.isrc) + } + + pub fn get_track_album_id(&self, id: &str, check: &Check<'_>) -> Result<String, ResolverError> { + match self.coalesced(&format!("track_album:{id}"), check, || { + let track: Track = + self.get_json(&format!("https://api.deezer.com/2.0/track/{id}"), check)?; + Ok(Value::AlbumId(track.album.id.to_string())) + })? { + Value::AlbumId(value) => Ok(value), + _ => unreachable!("track album cache type"), + } + } + + pub fn get_album_extended_metadata( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<Arc<AlbumExtendedMetadata>, ResolverError> { + check().map_err(ResolverError::Cancelled)?; + if id.is_empty() { + return Err(ResolverError::Failed("empty album ID".into())); + } + match self.coalesced(&format!("album_meta:{id}"), check, || { + let album: FullAlbum = self + .get_json(&format!("https://api.deezer.com/2.0/album/{id}"), check) + .map_err(|error| context(error, "failed to fetch album"))?; + Ok(Value::Extended(Arc::new(AlbumExtendedMetadata { + genre: album.genres.display(), + label: album.label, + copyright: album.copyright, + }))) + })? { + Value::Extended(value) => Ok(value), + _ => unreachable!("album metadata cache type"), + } + } + + pub fn get_extended_metadata_by_track_id( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<Arc<AlbumExtendedMetadata>, ResolverError> { + let album = self + .get_track_album_id(id, check) + .map_err(|error| context(error, "failed to get album ID"))?; + self.get_album_extended_metadata(&album, check) + } + + pub fn get_extended_metadata_by_isrc( + &self, + isrc: &str, + check: &Check<'_>, + ) -> Result<Arc<AlbumExtendedMetadata>, ResolverError> { + check().map_err(ResolverError::Cancelled)?; + if isrc.is_empty() { + return Err(ResolverError::Failed("empty ISRC".into())); + } + let track = self + .search_by_isrc(isrc, check) + .map_err(|error| context(error, "failed to find track by ISRC"))?; + self.get_extended_metadata_for_track(&track, check) + } + + pub fn get_extended_metadata_for_track( + &self, + track: &TrackMetadata, + check: &Check<'_>, + ) -> Result<Arc<AlbumExtendedMetadata>, ResolverError> { + check().map_err(ResolverError::Cancelled)?; + let id = track + .spotify_id + .strip_prefix("deezer:") + .unwrap_or(&track.spotify_id); + if id.is_empty() { + return Err(ResolverError::Failed("track found but no Deezer ID".into())); + } + let album = track + .album_id + .strip_prefix("deezer:") + .unwrap_or(&track.album_id); + if album.bytes().all(|byte| byte.is_ascii_digit()) + && album.parse::<i64>().is_ok_and(|id| id > 0) + { + return self.get_album_extended_metadata(album, check); + } + self.get_extended_metadata_by_track_id(id, check) + } +} + +pub fn parse_url(input: &str) -> Result<(String, String), ResolverError> { + let input = input.trim(); + if input.is_empty() { + return Err(ResolverError::Failed("empty URL".into())); + } + let parsed = + UrlParts::parse(input).ok_or_else(|| ResolverError::Failed("invalid Deezer URL".into()))?; + if parsed.port.is_some() + || !matches!( + parsed.hostname.as_str(), + "www.deezer.com" | "deezer.com" | "deezer.page.link" + ) + { + return Err(ResolverError::Failed("not a Deezer URL".into())); + } + let mut path = parsed.path.as_slice(); + while let Some(rest) = path.strip_prefix(b"/") { + path = rest; + } + while let Some(rest) = path.strip_suffix(b"/") { + path = rest; + } + let mut parts: Vec<_> = path.split(|byte| *byte == b'/').collect(); + if parts.first().is_some_and(|part| part.len() == 2) { + parts.remove(0); + } + if parts.len() < 2 { + return Err(ResolverError::Failed("invalid Deezer URL format".into())); + } + match parts[0] { + b"track" | b"album" | b"artist" | b"playlist" => Ok(( + String::from_utf8_lossy(parts[0]).into_owned(), + String::from_utf8_lossy(parts[1]).into_owned(), + )), + kind => Err(ResolverError::Failed(format!( + "unsupported Deezer resource type: {}", + String::from_utf8_lossy(kind) + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::deezer::cache::Bucket; + + #[test] + fn known_album_skips_track_lookup_and_invalid_album_uses_existing_fallback() { + let network = spotiflac_network::NetworkService::new().unwrap(); + let client = DeezerClient::with_endpoint(&network, "https://127.0.0.1:1").unwrap(); + let known = Arc::new(AlbumExtendedMetadata::default()); + let fallback = Arc::new(AlbumExtendedMetadata::default()); + for (key, value) in [ + ("album_meta:100", Value::Extended(known.clone())), + ("album_meta:200", Value::Extended(fallback.clone())), + ("track_album:42", Value::AlbumId("200".into())), + ] { + client.store(Bucket::Search, key.into(), value); + } + let mut track = TrackMetadata { + spotify_id: "deezer:42".into(), + ..Default::default() + }; + for (album, expected) in [ + ("deezer:100", &known), + ("100", &known), + ("", &fallback), + ("deezer:0", &fallback), + ("0", &fallback), + ("deezer:-100", &fallback), + ("deezer:+100", &fallback), + ("other:100", &fallback), + ("deezer:9223372036854775808", &fallback), + ] { + track.album_id = album.into(); + let result = client + .get_extended_metadata_for_track(&track, &|| Ok(())) + .unwrap(); + assert!(Arc::ptr_eq(&result, expected), "album ID: {album}"); + } + track.album_id = "deezer:100".into(); + assert!(matches!( + client.get_extended_metadata_for_track(&track, &|| Err("cancelled".into())), + Err(ResolverError::Cancelled(_)) + )); + track.spotify_id.clear(); + assert!(matches!( + client.get_extended_metadata_for_track(&track, &|| Ok(())), + Err(ResolverError::Failed(message)) if message == "track found but no Deezer ID" + )); + } +} diff --git a/rust_backend/crates/providers/src/deezer/search.rs b/rust_backend/crates/providers/src/deezer/search.rs new file mode 100644 index 00000000..19a6bc0c --- /dev/null +++ b/rust_backend/crates/providers/src/deezer/search.rs @@ -0,0 +1,182 @@ +use super::{ + Check, DeezerClient, ResolverError, + cache::{Bucket, Value}, +}; +use spotiflac_core::metadata::{ + SearchAlbumResult, SearchAllResult, SearchArtistResult, SearchPlaylistResult, + deezer::{self, AlbumSearch, ArtistSearch, PlaylistSearch, TrackSearch}, +}; +use spotiflac_network::query; +use std::sync::Arc; + +impl DeezerClient { + pub fn search_all( + &self, + query: &str, + track_limit: isize, + artist_limit: isize, + filter: &str, + check: &Check<'_>, + ) -> Result<Arc<SearchAllResult>, ResolverError> { + check().map_err(ResolverError::Cancelled)?; + let (tracks, artists, albums, playlists) = match filter { + "track" => (50, 0, 0, 0), + "artist" => (0, 20, 0, 0), + "album" => (0, 0, 20, 0), + "playlist" => (0, 0, 0, 20), + _ => (track_limit, artist_limit, 5, 5), + }; + if tracks < 0 || artists < 0 { + return Err(ResolverError::Failed("negative search limit".into())); + } + let key = format!("deezer:all:{query}:{tracks}:{artists}:{albums}:{playlists}:{filter}"); + if let Some(Value::Search(result)) = self.cached(Bucket::Search, &key) { + return Ok(result); + } + let mut parameters = query::Query::new(); + query::set(&mut parameters, "q", query); + let encoded = query::encode(¶meters); + let endpoint = |kind: &str, limit: isize| { + format!("https://api.deezer.com/2.0/search/{kind}?{encoded}&limit={limit}") + }; + let mut result = SearchAllResult::default(); + if tracks > 0 { + let response: TrackSearch = self + .get_json(&endpoint("track", tracks), check) + .map_err(|error| context(error, "deezer track search failed"))?; + if let Some(error) = response.error.0 { + return Err(ResolverError::Failed(format!( + "deezer API error: {} (code {})", + error.message, error.code + ))); + } + result.tracks = response + .data + .unwrap_or_default() + .iter() + .map(|track| track.metadata()) + .collect(); + } + let fetch_artists = || { + (artists > 0) + .then(|| self.get_json::<ArtistSearch>(&endpoint("artist", artists), check)) + }; + let fetch_albums = || { + (albums > 0).then(|| self.get_json::<AlbumSearch>(&endpoint("album", albums), check)) + }; + let fetch_playlists = || { + (playlists > 0) + .then(|| self.get_json::<PlaylistSearch>(&endpoint("playlist", playlists), check)) + }; + let parallel = [artists, albums, playlists] + .into_iter() + .filter(|n| *n > 0) + .count() + > 1; + let (artist_response, album_response, playlist_response) = std::thread::scope(|scope| { + let artist = (parallel && artists > 0).then(|| { + std::thread::Builder::new() + .name("search-artists".into()) + .spawn_scoped(scope, fetch_artists) + }); + let album = (parallel && albums > 0).then(|| { + std::thread::Builder::new() + .name("search-albums".into()) + .spawn_scoped(scope, fetch_albums) + }); + let playlist = fetch_playlists(); + let artist = match artist { + Some(Ok(worker)) => worker + .join() + .map_err(|_| ResolverError::Failed("artist search panicked".into()))?, + _ => fetch_artists(), + }; + let album = match album { + Some(Ok(worker)) => worker + .join() + .map_err(|_| ResolverError::Failed("album search panicked".into()))?, + _ => fetch_albums(), + }; + Ok::<_, ResolverError>((artist, album, playlist)) + })?; + if let Some(response) = artist_response { + match response { + Ok(response) if response.error.0.is_none() => { + result.artists = response + .data + .unwrap_or_default() + .iter() + .map(|artist| SearchArtistResult { + id: format!("deezer:{}", artist.id), + name: artist.name.clone(), + images: artist.image(), + followers: artist.nb_fan, + popularity: 0, + }) + .collect(); + } + Err(error @ (ResolverError::Cancelled(_) | ResolverError::Closed)) => { + return Err(error); + } + _ => {} + } + } + if let Some(response) = album_response { + match response { + Ok(response) if response.error.0.is_none() => { + result.albums = response + .data + .unwrap_or_default() + .iter() + .map(|album| SearchAlbumResult { + id: format!("deezer:{}", album.id), + name: album.title.clone(), + artists: album.artist.name.clone(), + images: album.image(), + release_date: album.release_date.clone(), + total_tracks: album.nb_tracks, + album_type: deezer::album_type(&album.record_type), + }) + .collect(); + } + Err(error @ (ResolverError::Cancelled(_) | ResolverError::Closed)) => { + return Err(error); + } + _ => {} + } + } + if let Some(response) = playlist_response { + match response { + Ok(response) if response.error.0.is_none() => { + result.playlists = response + .data + .unwrap_or_default() + .iter() + .map(|playlist| SearchPlaylistResult { + id: format!("deezer:{}", playlist.id), + name: playlist.title.clone(), + owner: playlist.user.name.clone(), + images: playlist.image(), + total_tracks: playlist.nb_tracks, + }) + .collect(); + } + Err(error @ (ResolverError::Cancelled(_) | ResolverError::Closed)) => { + return Err(error); + } + _ => {} + } + } + check().map_err(ResolverError::Cancelled)?; + let result = Arc::new(result); + self.store(Bucket::Search, key, Value::Search(result.clone())); + Ok(result) + } +} + +pub(super) fn context(error: ResolverError, message: &str) -> ResolverError { + match error { + ResolverError::Cancelled(_) | ResolverError::Closed | ResolverError::Busy => error, + _ => ResolverError::Failed(format!("{message}: {error}")), + } +} diff --git a/rust_backend/crates/providers/src/lib.rs b/rust_backend/crates/providers/src/lib.rs new file mode 100644 index 00000000..1895ca01 --- /dev/null +++ b/rust_backend/crates/providers/src/lib.rs @@ -0,0 +1,6 @@ +//! Built-in provider clients and shared lookup services for native app owners. + +pub mod deezer; +pub mod lyrics; +pub mod musicbrainz; +pub mod resolver; diff --git a/rust_backend/crates/providers/src/lyrics.rs b/rust_backend/crates/providers/src/lyrics.rs new file mode 100644 index 00000000..517de394 --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics.rs @@ -0,0 +1,37 @@ +pub mod builtin; +pub mod cache; +mod calls; +mod http; +pub mod lrclib; +mod service; + +pub use calls::{CallGraph, CallNode, CallWait}; +pub use service::{LyricsFetcher, LyricsService, SearchRequest}; + +use spotiflac_core::lyrics::errors::ErrorKind; + +pub type Check<'a> = dyn Fn() -> Result<(), String> + Sync + 'a; + +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +pub enum LyricsError { + #[error("{0}")] + NotFound(String), + #[error("{0}")] + Unavailable(String), + #[error("{0}")] + Other(String), + #[error("{0}")] + Cancelled(String), + #[error("{0}")] + Recursive(String), +} + +impl LyricsError { + pub fn classified(kind: ErrorKind, message: impl Into<String>) -> Self { + match kind { + ErrorKind::NotFound => Self::NotFound(message.into()), + ErrorKind::Unavailable => Self::Unavailable(message.into()), + ErrorKind::Other => Self::Other(message.into()), + } + } +} diff --git a/rust_backend/crates/providers/src/lyrics/builtin.rs b/rust_backend/crates/providers/src/lyrics/builtin.rs new file mode 100644 index 00000000..f05d7dcf --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/builtin.rs @@ -0,0 +1,215 @@ +mod apple; +mod direct; +mod proxy; + +pub use direct::genius_text; + +use super::{ + Check, LyricsError, LyricsFetcher, SearchRequest, + http::{LyricsHttp, Params, Request, pairs}, + lrclib::LrcLibClient, +}; +use spotiflac_core::app_version::AppVersion; +use spotiflac_core::lyrics::{LyricsResponse, errors, matching, models, text_from_bytes}; +use spotiflac_network::NetworkService; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +/// Shared platform resolution remains a separate domain. Its implementation +/// must preserve the full resolver chain and observe the lookup cancellation. +pub trait TrackResolver: Send + Sync + 'static { + fn deezer_id_from_spotify( + &self, + spotify_id: &str, + check: &Check<'_>, + ) -> Result<String, LyricsError>; +} + +pub struct BuiltinLyricsClient { + http: LyricsHttp, + lrclib: LrcLibClient, + resolver: Arc<dyn TrackResolver>, + apple_token: Mutex<String>, +} + +impl BuiltinLyricsClient { + pub fn new( + network: &Arc<NetworkService>, + version: impl Into<AppVersion>, + resolver: Arc<dyn TrackResolver>, + ) -> Self { + Self::with_endpoints(network, version, resolver, BTreeMap::new()) + .expect("built-in provider endpoints") + } + + /// Native-only endpoint injection for integration fixtures or deployments. + /// The application and extension SDK never accept arbitrary override maps. + pub fn with_endpoints( + network: &Arc<NetworkService>, + version: impl Into<AppVersion>, + resolver: Arc<dyn TrackResolver>, + endpoints: BTreeMap<String, String>, + ) -> Result<Self, LyricsError> { + Ok(Self { + lrclib: LrcLibClient::with_endpoint( + network, + endpoints + .get("https://lrclib.net") + .map(String::as_str) + .unwrap_or("https://lrclib.net"), + super::http::BROWSER_UA.into(), + )?, + http: LyricsHttp::new(network, version, endpoints)?, + resolver, + apple_token: Mutex::new(String::new()), + }) + } + + pub fn fetch_once( + &self, + provider: &str, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + check().map_err(LyricsError::Cancelled)?; + match provider { + "lrclib" => self.lrclib.fetch_lyrics(request, check), + "netease" => self.netease(request, check), + "musixmatch" => self.musixmatch(request, check), + "apple_music" => self.apple(request, check), + "spotify" => self.spotify(request, check), + "deezer" => self.deezer(request, check), + "youtube" => self.youtube(request, check), + "lyricsplus" => self.lyrics_plus(request, "", check), + "qqmusic" => self.qqmusic(request, check), + "kugou" => self.kugou(request, check), + "genius" => self.genius(request, check), + _ => Err(LyricsError::Other(format!("unknown provider: {provider}"))), + } + } + + fn proxy_body( + &self, + endpoint: &str, + params: Params, + check: &Check<'_>, + ) -> Result<String, LyricsError> { + let mut request = Request::new(endpoint); + request.params = params; + request.allowed = &[]; + let response = self.http.get(request, check)?; + let text = text_from_bytes(&response.body); + let text = text.trim(); + if let Some(message) = errors::detect_payload(text) { + return Err(if errors::payload_not_found(&message) { + LyricsError::NotFound(message) + } else { + LyricsError::Unavailable(message) + }); + } + if response.status != 200 { + return Err(LyricsError::Unavailable(format!( + "HTTP {}", + response.status + ))); + } + if text.is_empty() { + return Err(LyricsError::Unavailable("empty response".into())); + } + Ok(text.into()) + } +} + +impl LyricsFetcher for BuiltinLyricsClient { + fn fetch( + &self, + provider: &str, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + if provider == "lrclib" { + return self.fetch_once(provider, request, check); + } + let primary = matching::primary_artist(&request.artist); + let primary_differs = primary != request.artist; + let simplified = matching::simplify_track(&request.track); + let mut attempt = request.clone(); + attempt.artist = primary; + let mut result = self.fetch_once(provider, &attempt, check); + if retryable(&result) && primary_differs { + result = self.fetch_once(provider, request, check); + } + if retryable(&result) + && simplified != request.track + && matches!( + provider, + "netease" | "spotify" | "youtube" | "kugou" | "genius" | "lyricsplus" + ) + { + attempt.track = simplified; + if provider == "spotify" { + attempt.spotify_id.clear(); + } + result = self.fetch_once(provider, &attempt, check); + } + result + } +} + +fn retryable(result: &Result<LyricsResponse, LyricsError>) -> bool { + matches!(result, Err(error) if !matches!(error, LyricsError::Unavailable(_) | LyricsError::Cancelled(_))) +} + +fn from_text(text: &str, provider: &str, source: &str) -> Result<LyricsResponse, LyricsError> { + let response = LyricsResponse::from_text(text, provider, source); + if response.has_usable_text() { + Ok(response) + } else { + Err(LyricsError::NotFound(format!( + "no lyrics found on {provider}" + ))) + } +} + +fn artists(artists: &[models::Artist]) -> String { + artists + .iter() + .map(|artist| artist.name.trim()) + .filter(|name| !name.is_empty()) + .collect::<Vec<_>>() + .join(", ") +} + +fn select<'a, T>( + values: &'a [T], + request: &SearchRequest, + fields: impl Fn(&T) -> (String, String, f64, bool), +) -> Option<&'a T> { + let mut best = None; + let mut best_score = -1; + for value in values { + let (track, artist, duration, allowed) = fields(value); + if !allowed { + continue; + } + let score = matching::score( + &track, + &artist, + duration, + &request.track, + &request.artist, + request.duration, + ); + if score > best_score { + best = Some(value); + best_score = score; + } + } + best +} + +fn matches(track: &str, artist: &str, duration: f64, request: &SearchRequest) -> bool { + matching::titles_match(track, &request.track, false) + && matching::artists_match(artist, &request.artist) + && matching::duration_matches(duration, request.duration) +} diff --git a/rust_backend/crates/providers/src/lyrics/builtin/apple.rs b/rust_backend/crates/providers/src/lyrics/builtin/apple.rs new file mode 100644 index 00000000..65055d9f --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/builtin/apple.rs @@ -0,0 +1,204 @@ +use super::*; +use crate::lyrics::http; +use regex::Regex; +use spotiflac_core::lyrics::payloads; +use std::sync::{LazyLock, MutexGuard, TryLockError}; +use std::time::Duration; + +static INDEX: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r#"/assets/index~[^"' <]+\.js"#).unwrap()); +static TOKEN: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+").unwrap()); + +impl BuiltinLyricsClient { + fn lock_apple_token<'a>( + &'a self, + check: &Check<'_>, + ) -> Result<MutexGuard<'a, String>, LyricsError> { + loop { + check().map_err(LyricsError::Cancelled)?; + match self.apple_token.try_lock() { + Ok(guard) => return Ok(guard), + Err(TryLockError::WouldBlock) => std::thread::sleep(Duration::from_millis(25)), + Err(TryLockError::Poisoned(_)) => { + return Err(LyricsError::Other("apple token cache poisoned".into())); + } + } + } + } + + fn apple_token(&self, check: &Check<'_>) -> Result<String, LyricsError> { + let mut token = self.lock_apple_token(check)?; + if !token.is_empty() { + return Ok(token.clone()); + } + let mut page = Request::new("https://beta.music.apple.com"); + page.headers = pairs(&[("User-Agent", http::BROWSER_UA)]); + page.timeout = Duration::from_secs(20); + let html = text_from_bytes(&self.http.get(page, check)?.body); + let index = INDEX + .find(&html) + .ok_or_else(|| LyricsError::Other("apple music index script not found".into()))?; + let url = format!("https://beta.music.apple.com{}", index.as_str()); + let mut script = Request::new(&url); + script.headers = pairs(&[("User-Agent", http::BROWSER_UA)]); + script.timeout = Duration::from_secs(20); + let javascript = text_from_bytes(&self.http.get(script, check)?.body); + let found = TOKEN + .find(&javascript) + .ok_or_else(|| LyricsError::Other("apple music token not found".into()))?; + check().map_err(LyricsError::Cancelled)?; + *token = found.as_str().into(); + Ok(token.clone()) + } + + fn apple_search( + &self, + token: &str, + query: &str, + check: &Check<'_>, + ) -> Result<Option<models::AppleSearch>, LyricsError> { + let mut get = Request::new("https://amp-api.music.apple.com/v1/catalog/us/search"); + get.timeout = Duration::from_secs(20); + get.allowed = &[200, 401]; + get.params = pairs(&[ + ("term", query), + ("types", "songs"), + ("limit", "25"), + ("l", "en-US"), + ("platform", "web"), + ("format[resources]", "map"), + ("include[songs]", "artists"), + ("extend", "artistUrl"), + ]); + get.headers = pairs(&[ + ("Authorization", &format!("Bearer {token}")), + ("Origin", "https://music.apple.com"), + ("Referer", "https://music.apple.com/"), + ( + "User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0", + ), + ("Accept", "application/json"), + ("Accept-Language", "en-US,en;q=0.5"), + ("x-apple-renewal", "true"), + ]); + let response = self.http.get(get, check)?; + if response.status == 401 { + return Ok(None); + } + http::decode(&response.body, true, check).map(Some) + } + + pub(super) fn apple( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let query = format!("{} {}", request.track, request.artist); + if query.trim().is_empty() { + return Err(LyricsError::NotFound("empty search query".into())); + } + let token = self.apple_token(check)?; + let mut search = self.apple_search(&token, query.trim(), check)?; + if search.is_none() { + // A delayed 401 must not discard a newer token obtained by another + // lookup. Refresh remains coalesced by the token cache mutex. + let mut cached = self.lock_apple_token(check)?; + if *cached == token { + cached.clear(); + } + drop(cached); + search = self.apple_search(&self.apple_token(check)?, query.trim(), check)?; + } + let search = search + .ok_or_else(|| LyricsError::Other("apple music catalog search unauthorized".into()))?; + let songs = search + .results + .songs + .and_then(|songs| songs.data) + .unwrap_or_default(); + let resources = search + .resources + .and_then(|resources| resources.songs) + .unwrap_or_default(); + let results: Vec<_> = songs + .iter() + .filter_map(|song| { + resources + .get(&song.id) + .map(|data| (&song.id, &data.attributes)) + }) + .collect(); + let selected = select(&results, request, |(_, song)| { + let duration = song.duration_in_millis as f64 / 1000.0; + ( + song.name.clone(), + song.artist_name.clone(), + duration, + matches(&song.name, &song.artist_name, duration, request), + ) + }) + .filter(|(id, _)| !id.trim().is_empty()) + .ok_or_else(|| LyricsError::NotFound("no songs found on apple music".into()))?; + let mut get = Request::new("https://lyrics.paxsenix.org/apple-music/lyrics"); + get.timeout = Duration::from_secs(20); + get.params = pairs(&[("id", selected.0.trim())]); + let raw = text_from_bytes(&self.http.get(get, check)?.body); + let raw = raw.trim(); + if raw.is_empty() { + return Err(LyricsError::Other( + "empty lyrics response from apple music".into(), + )); + } + if let Some(message) = errors::detect_payload(raw) { + return Err(LyricsError::classified( + errors::classify_payload(0, &message), + format!("apple music proxy returned non-lyric payload: {message}"), + )); + } + let text = match payloads::format_apple( + raw, + request.options.multi_person_word_by_word, + request.options.apple_elrc_word_sync, + ) { + Ok(text) => text, + Err(error) if raw.starts_with(['{', '[']) => return Err(LyricsError::Other(error)), + Err(_) => raw.into(), + }; + from_text(&text, "Apple Music", "Apple Music") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct Resolver; + impl TrackResolver for Resolver { + fn deezer_id_from_spotify(&self, _: &str, _: &Check<'_>) -> Result<String, LyricsError> { + unreachable!("token test must not resolve tracks") + } + } + + #[test] + fn waiting_for_another_token_refresh_remains_cancellable() { + let network = NetworkService::new().unwrap(); + let client = BuiltinLyricsClient::new(&network, "", Arc::new(Resolver)); + let _held = client.apple_token.lock().unwrap(); + let calls = AtomicUsize::new(0); + let result = client.apple_token(&|| { + if calls.fetch_add(1, Ordering::AcqRel) >= 2 { + Err("cancel token waiter".into()) + } else { + Ok(()) + } + }); + assert_eq!( + result, + Err(LyricsError::Cancelled("cancel token waiter".into())) + ); + assert_eq!(calls.load(Ordering::Acquire), 3); + } +} diff --git a/rust_backend/crates/providers/src/lyrics/builtin/direct.rs b/rust_backend/crates/providers/src/lyrics/builtin/direct.rs new file mode 100644 index 00000000..0e1d3b1c --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/builtin/direct.rs @@ -0,0 +1,397 @@ +use super::*; +use crate::lyrics::http; +use base64::{ + Engine, alphabet, + engine::general_purpose::{GeneralPurpose, GeneralPurposeConfig}, +}; +use html5ever::tendril::TendrilSink; +use scraper::{Html, HtmlTreeSink, Node}; +use spotiflac_core::lyrics::lrc; + +fn base64_text(text: &str, raw: bool) -> Result<String, LyricsError> { + // Go accepts nonzero padding bits and ignores CR/LF, but not other spaces. + let input: Vec<_> = text + .bytes() + .filter(|byte| !matches!(byte, b'\r' | b'\n')) + .collect(); + let padding = if raw { + base64::engine::DecodePaddingMode::RequireNone + } else { + base64::engine::DecodePaddingMode::RequireCanonical + }; + let engine = GeneralPurpose::new( + &alphabet::STANDARD, + GeneralPurposeConfig::new() + .with_decode_allow_trailing_bits(true) + .with_decode_padding_mode(padding), + ); + engine + .decode(input) + .map(|bytes| text_from_bytes(&bytes)) + .map_err(|error| LyricsError::Unavailable(format!("invalid base64 lyrics: {error}"))) +} + +impl BuiltinLyricsClient { + fn direct_body( + &self, + endpoint: &str, + params: Params, + qq: bool, + check: &Check<'_>, + ) -> Result<Vec<u8>, LyricsError> { + let mut get = Request::new(endpoint); + get.params = params; + get.max_bytes = 2 << 20; + get.unavailable_errors = true; + if qq { + get.headers + .insert("Referer".into(), "https://y.qq.com/".into()); + get.headers + .insert("User-Agent".into(), http::BROWSER_UA.into()); + } + let response = self.http.get(get, check)?; + if text_from_bytes(&response.body).trim().is_empty() { + return Err(LyricsError::Unavailable("empty response".into())); + } + Ok(response.body) + } + + pub(super) fn qqmusic( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let query = format!("{} {}", request.track, request.artist); + if query.trim().is_empty() { + return Err(LyricsError::NotFound("empty search query".into())); + } + let params = pairs(&[ + ("format", "json"), + ("inCharset", "utf8"), + ("outCharset", "utf8"), + ("platform", "yqq.json"), + ("new_json", "1"), + ("w", query.trim()), + ("p", "1"), + ("n", "20"), + ("t", "0"), + ("aggr", "1"), + ("cr", "1"), + ("catZhida", "1"), + ("lossless", "1"), + ("flag_qc", "0"), + ("remoteplace", "txt.yqq.center"), + ("needNewCode", "0"), + ]); + let response: models::QqSearch = http::decode( + &self.direct_body( + "https://c.y.qq.com/soso/fcgi-bin/client_search_cp", + params, + true, + check, + )?, + false, + check, + )?; + if response.code != 0 { + return Err(LyricsError::Unavailable(format!( + "QQ Music search returned code {}", + response.code + ))); + } + let songs = response.data.song.list.unwrap_or_default(); + let selected = select(&songs, request, |song| { + let artist = artists(song.singer.as_deref().unwrap_or_default()); + let duration = song.interval as f64; + let allowed = matches(&song.name, &artist, duration, request); + (song.name.clone(), artist, duration, allowed) + }) + .filter(|song| !song.mid.trim().is_empty()) + .ok_or_else(|| LyricsError::NotFound("no matching song found on QQ Music".into()))?; + let params = pairs(&[ + ("format", "json"), + ("inCharset", "utf8"), + ("outCharset", "utf-8"), + ("notice", "0"), + ("platform", "yqq.json"), + ("needNewCode", "0"), + ("songmid", &selected.mid), + ("songid", &selected.id.to_string()), + ]); + let response: models::QqLyrics = http::decode( + &self.direct_body( + "https://c.y.qq.com/lyric/fcgi-bin/fcg_query_lyric_new.fcg", + params, + true, + check, + )?, + false, + check, + )?; + if response.code != 0 || response.retcode != 0 { + return Err(LyricsError::Unavailable(format!( + "QQ Music lyrics returned code {}", + response.code + ))); + } + let raw = response.lyric.trim(); + if raw.is_empty() { + return Err(LyricsError::NotFound( + "QQ Music returned empty lyrics".into(), + )); + } + let text = if raw.starts_with('[') { + raw.into() + } else { + base64_text(raw, false).or_else(|_| base64_text(raw, true))? + }; + from_text(&text, "QQ Music", "QQ Music Direct") + } + + pub(super) fn kugou( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let query = format!("{} - {}", request.artist, request.track); + let rounded = (request.duration * 1000.0).round(); + let milliseconds = + if !rounded.is_finite() || rounded >= -(i64::MIN as f64) || rounded < i64::MIN as f64 { + i64::MIN + } else { + rounded as i64 + }; + let params = pairs(&[ + ("ver", "1"), + ("man", "yes"), + ("client", "pc"), + ("keyword", query.trim()), + ("duration", &milliseconds.to_string()), + ("hash", ""), + ]); + let response: models::KugouSearch = http::decode( + &self.direct_body("https://lyrics.kugou.com/search", params, false, check)?, + false, + check, + )?; + if response.status != 200 || (response.errcode != 0 && response.errcode != 200) { + return Err(LyricsError::Unavailable( + if response.errmsg.trim().is_empty() { + format!("status {}/error {}", response.status, response.errcode) + } else { + response.errmsg.trim().into() + }, + )); + } + let songs = response.candidates.unwrap_or_default(); + let song = select(&songs, request, |song| { + let duration = song.duration / 1000.0; + ( + song.song.clone(), + song.singer.clone(), + duration, + matches(&song.song, &song.singer, duration, request), + ) + }) + .filter(|song| !song.id.trim().is_empty() && !song.accesskey.trim().is_empty()) + .ok_or_else(|| LyricsError::NotFound("no matching song found on kugou".into()))?; + let params = pairs(&[ + ("ver", "1"), + ("client", "pc"), + ("id", &song.id), + ("accesskey", &song.accesskey), + ("fmt", "lrc"), + ("charset", "utf8"), + ]); + let response: models::KugouLyrics = http::decode( + &self.direct_body("https://lyrics.kugou.com/download", params, false, check)?, + false, + check, + )?; + if response.status != 200 || response.error_code != 0 { + return Err(LyricsError::Unavailable( + if response.info.trim().is_empty() { + format!("status {}/error {}", response.status, response.error_code) + } else { + response.info.trim().into() + }, + )); + } + from_text( + &base64_text(&response.content, false)?, + "Kugou", + "Kugou Direct", + ) + } + + pub(super) fn genius( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let query = format!("{} {}", request.track, request.artist); + if query.trim().is_empty() { + return Err(LyricsError::NotFound("empty search query".into())); + } + let raw = self.proxy_body( + "https://genius.com/api/search/multi", + pairs(&[("q", query.trim()), ("per_page", "5")]), + check, + )?; + let response: models::GeniusSearch = http::decode(raw.as_bytes(), false, check)?; + let mut best = None; + let mut best_score = -1; + for section in response.response.sections.unwrap_or_default() { + for hit in section.hits.unwrap_or_default() { + let song = hit.result; + if hit.kind != "song" || song.url.trim().is_empty() { + continue; + } + let artist = if song.primary_artist_names.trim().is_empty() { + &song.artist_names + } else { + &song.primary_artist_names + }; + if !matches(&song.title, artist, 0.0, request) { + continue; + } + let score = matching::score( + &song.title, + artist, + 0.0, + &request.track, + &request.artist, + request.duration, + ); + if score > best_score { + best = Some(song.url.trim().to_owned()); + best_score = score; + } + } + } + let url = best.ok_or_else(|| LyricsError::NotFound("no songs found on genius".into()))?; + let mut get = Request::new(&url); + get.max_bytes = 8 << 20; + get.unavailable_errors = true; + get.headers = pairs(&[ + ("Accept", "text/html,application/xhtml+xml"), + ("Accept-Language", "en-US,en;q=0.9"), + ("User-Agent", http::BROWSER_UA), + ]); + let text = genius_text(&text_from_bytes(&self.http.get(get, check)?.body), check)?; + from_text(&text, "Genius", "Genius Direct") + } +} + +pub fn genius_text(text: &str, check: &Check<'_>) -> Result<String, LyricsError> { + check().map_err(LyricsError::Cancelled)?; + if text.len() > 8 << 20 { + return Err(LyricsError::Unavailable( + "Genius page exceeds input limit".into(), + )); + } + let mut parser = + html5ever::parse_document(HtmlTreeSink::new(Html::new_document()), Default::default()); + let mut start = 0; + while start < text.len() { + check().map_err(LyricsError::Cancelled)?; + let end = text.floor_char_boundary((start + 4096).min(text.len())); + parser.process(text[start..end].into()); + start = end; + if parser.tokenizer.sink.sink.0.borrow().tree.nodes().len() > 100_000 { + return Err(LyricsError::Unavailable( + "Genius page exceeds HTML node limit".into(), + )); + } + } + let html = parser.finish(); + let mut found = false; + let mut sections = Vec::new(); + let mut total = 0; + for container in html.tree.root().descendants().filter(|node| { + node.value().as_element().is_some_and(|element| { + element.name() == "div" && element.attr("data-lyrics-container") == Some("true") + }) + }) { + found = true; + let mut text = String::new(); + let mut stack = vec![container]; + while let Some(node) = stack.pop() { + check().map_err(LyricsError::Cancelled)?; + if node + .value() + .as_element() + .is_some_and(|element| element.attr("data-exclude-from-selection") == Some("true")) + { + continue; + } + match node.value() { + Node::Text(value) => text.push_str(&value.text), + Node::Element(element) if element.name() == "br" => text.push('\n'), + _ => stack.extend(node.children().rev()), + } + if text.len() + total > 8 << 20 { + return Err(LyricsError::Unavailable( + "Genius lyrics exceed output limit".into(), + )); + } + } + let normalized = text.replace('\u{a0}', " "); + let normalized = normalized.trim(); + if lrc::has_usable_content(normalized) { + total += normalized.len() + 1; + sections.push(normalized.to_owned()); + } + } + if !found { + return Err(LyricsError::NotFound( + "Genius page has no lyrics container".into(), + )); + } + if sections.is_empty() { + return Err(LyricsError::NotFound( + "Genius page returned empty lyrics".into(), + )); + } + Ok(sections.join("\n")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[test] + fn html_parsing_observes_cancellation_and_bounds_nodes_and_bytes() { + let checks = AtomicUsize::new(0); + let html = "<div>Example</div>".repeat(20_000); + let result = genius_text(&html, &|| { + if checks.fetch_add(1, Ordering::AcqRel) >= 3 { + Err("cancel HTML parser".into()) + } else { + Ok(()) + } + }); + assert_eq!( + result, + Err(LyricsError::Cancelled("cancel HTML parser".into())) + ); + assert!(matches!( + genius_text(&" ".repeat((8 << 20) + 1), &|| Ok(())), + Err(LyricsError::Unavailable(_)) + )); + assert!(matches!( + genius_text(&"<i></i>".repeat(100_001), &|| Ok(())), + Err(LyricsError::Unavailable(_)) + )); + } + + #[test] + fn base64_keeps_go_padding_bits_linebreaks_and_raw_fallback() { + assert_eq!(base64_text("Zh==\r\n", false).unwrap(), "f"); + assert_eq!(base64_text("Zh", true).unwrap(), "f"); + assert!(base64_text("Zh", false).is_err()); + assert!(base64_text("Z h==", false).is_err()); + assert!(base64_text("Zg==", true).is_err()); + } +} diff --git a/rust_backend/crates/providers/src/lyrics/builtin/proxy.rs b/rust_backend/crates/providers/src/lyrics/builtin/proxy.rs new file mode 100644 index 00000000..701b349c --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/builtin/proxy.rs @@ -0,0 +1,370 @@ +use super::*; +use crate::lyrics::http; +use spotiflac_core::lyrics::{decode_document, payloads}; +use spotiflac_core::matching::lowercase; + +pub(super) fn spotify_id(raw: &str) -> String { + let mut raw = raw.trim(); + if lowercase(raw).starts_with("deezer:") { + return String::new(); + } + if lowercase(raw).starts_with("spotify:") { + raw = raw.rsplit(':').next().unwrap_or_default(); + } + if raw.contains("spotify.com/track/") { + raw = raw.split("/track/").nth(1).unwrap_or_default(); + } + raw = raw.split('?').next().unwrap_or_default().trim(); + if raw.len() == 22 && raw.bytes().all(|byte| byte.is_ascii_alphanumeric()) { + raw.into() + } else { + String::new() + } +} + +pub(super) fn deezer_id(raw: &str) -> String { + let mut raw = raw.trim(); + if lowercase(raw).starts_with("deezer:") { + raw = raw[7..].trim(); + } + if raw.contains("deezer.com/") { + raw = raw.rsplit('/').next().unwrap_or_default(); + } + raw = raw.split('?').next().unwrap_or_default().trim(); + if raw.parse::<i64>().is_ok() { + raw.into() + } else { + String::new() + } +} + +impl BuiltinLyricsClient { + pub(super) fn spotify( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let mut id = spotify_id(&request.spotify_id); + if id.is_empty() { + let query = format!("{} {}", request.track, request.artist); + if query.trim().is_empty() { + return Err(LyricsError::NotFound("empty search query".into())); + } + let raw = self.proxy_body( + "https://lyrics.paxsenix.org/spotify/search", + pairs(&[("q", query.trim())]), + check, + )?; + let songs: Option<Vec<models::SpotifySong>> = + http::decode(raw.as_bytes(), false, check)?; + let songs = songs.unwrap_or_default(); + let selected = select(&songs, request, |song| { + let duration = matching::clock_duration(&song.duration); + ( + song.name.clone(), + song.artist_name.clone(), + duration, + matches(&song.name, &song.artist_name, duration, request), + ) + }) + .filter(|song| !song.track_id.trim().is_empty()) + .ok_or_else(|| LyricsError::NotFound("no songs found on spotify".into()))?; + id = selected.track_id.trim().into(); + } + let raw = self.proxy_body( + "https://lyrics.paxsenix.org/spotify/lyrics", + pairs(&[("id", &id)]), + check, + )?; + payloads::parse_proxy(&raw, "Spotify", false).map_err(LyricsError::Unavailable) + } + + pub(super) fn deezer( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let mut id = deezer_id(&request.spotify_id); + if id.is_empty() { + let spotify = spotify_id(&request.spotify_id); + if spotify.is_empty() { + return Err(LyricsError::NotFound( + "deezer provider needs a deezer id or spotify id".into(), + )); + } + id = deezer_id(&self.resolver.deezer_id_from_spotify(&spotify, check)?); + } + if id.is_empty() { + return Err(LyricsError::Other("deezer id unavailable".into())); + } + let raw = self.proxy_body( + "https://lyrics.paxsenix.org/deezer/lyrics", + pairs(&[("id", &id)]), + check, + )?; + payloads::parse_proxy(&raw, "Deezer", true).map_err(LyricsError::Unavailable) + } + + pub(super) fn youtube( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let query = format!("{} {}", request.track, request.artist); + if query.trim().is_empty() { + return Err(LyricsError::NotFound("empty search query".into())); + } + let raw = self.proxy_body( + "https://lyrics.paxsenix.org/youtube/search", + pairs(&[("q", query.trim())]), + check, + )?; + let songs: Option<Vec<models::YouTubeSong>> = http::decode(raw.as_bytes(), false, check)?; + let songs = songs.unwrap_or_default(); + let selected = select(&songs, request, |song| { + let duration = matching::clock_duration(&song.duration); + let allowed = matching::titles_match(&song.title, &request.track, true) + && (matching::artists_match(&song.author, &request.artist) + || matching::artist_in_title(&song.title, &request.artist)) + && matching::duration_matches(duration, request.duration); + (song.title.clone(), song.author.clone(), duration, allowed) + }) + .filter(|song| !song.video_id.trim().is_empty()) + .ok_or_else(|| LyricsError::NotFound("no songs found on youtube".into()))?; + let raw = self.proxy_body( + "https://lyrics.paxsenix.org/youtube/lyrics", + pairs(&[("id", selected.video_id.trim())]), + check, + )?; + payloads::parse_proxy(&raw, "YouTube", false).map_err(LyricsError::Unavailable) + } + + pub(super) fn netease( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let query = format!("{} {}", request.track, request.artist); + if query.trim().is_empty() { + return Err(LyricsError::NotFound("empty search query".into())); + } + let headers = pairs(&[ + ("Accept", "application/json"), + ("Accept-Language", "en-US,en;q=0.9"), + ("Cache-Control", "max-age=0"), + ]); + let mut search = Request::new("https://lyrics.paxsenix.org/netease/search"); + search.params = pairs(&[("q", &query)]); + search.headers = headers.clone(); + let body = self.http.get(search, check)?.body; + let response: models::NeteaseSearch = http::decode(&body, true, check)?; + if response.code != 0 && response.code != 200 { + let message = if !response.message.trim().is_empty() { + response.message.trim() + } else if !response.msg.trim().is_empty() { + response.msg.trim() + } else { + "unexpected response code" + }; + return Err(LyricsError::Unavailable(format!( + "netease search unavailable: code {}: {message}", + response.code + ))); + } + let songs = response.result.songs.unwrap_or_default(); + if response.result.song_count == 0 || songs.is_empty() { + return Err(LyricsError::NotFound("no songs found on netease".into())); + } + let song = select(&songs, request, |song| { + let artist = artists(song.artists.as_deref().unwrap_or_default()); + let allowed = matches(&song.name, &artist, 0.0, request); + (song.name.clone(), artist, 0.0, allowed) + }) + .filter(|song| song.id != 0) + .ok_or_else(|| LyricsError::NotFound("no matching songs found on netease".into()))?; + let mut get = Request::new("https://lyrics.paxsenix.org/netease/lyrics"); + get.params = pairs(&[("id", &song.id.to_string())]); + get.headers = headers; + let response: models::NeteaseLyrics = + http::decode(&self.http.get(get, check)?.body, true, check)?; + let mut text = response + .lrc + .filter(|line| !line.lyric.trim().is_empty()) + .ok_or_else(|| LyricsError::NotFound("no lyrics available on netease".into()))? + .lyric; + for (include, lines) in [ + (request.options.include_translation_netease, response.tlyric), + ( + request.options.include_romanization_netease, + response.romalrc, + ), + ] { + if include + && let Some(lines) = lines + && !lines.lyric.trim().is_empty() + { + text.push_str("\n\n"); + text.push_str(&lines.lyric); + } + } + from_text(&text, "Netease", "Netease") + .map_err(|_| LyricsError::Other("netease returned empty lyrics".into())) + } + + fn musixmatch_payload( + &self, + request: &SearchRequest, + kind: &str, + language: &str, + check: &Check<'_>, + ) -> Result<String, LyricsError> { + if request.track.trim().is_empty() || request.artist.trim().is_empty() { + return Err(LyricsError::NotFound("empty track or artist name".into())); + } + let mut get = Request::new("https://lyrics.paxsenix.org/musixmatch/lyrics"); + get.allowed = &[]; + get.params = pairs(&[ + ("t", &request.track), + ("a", &request.artist), + ("type", kind), + ("format", "lrc"), + ]); + if request.duration > 0.0 { + let rounded = request.duration.round(); + let duration = if rounded >= -(isize::MIN as f64) { + isize::MIN + } else { + rounded as isize + }; + get.params.insert("d".into(), duration.to_string()); + } + if !language.trim().is_empty() { + get.params.insert("l".into(), lowercase(language.trim())); + } + let response = self.http.get(get, check)?; + let text = text_from_bytes(&response.body); + let text = text.trim(); + if response.status != 200 { + let kind = errors::detect_payload(text).map_or_else( + || errors::http_status(response.status), + |message| errors::classify_payload(response.status, &message), + ); + return Err(LyricsError::classified( + kind, + format!("musixmatch proxy returned HTTP {}", response.status), + )); + } + if let Ok(decoded) = decode_document::<Option<String>>(&response.body) { + let decoded = decoded.unwrap_or_default(); + if decoded.trim().is_empty() { + return Err(LyricsError::Other("empty musixmatch lyrics payload".into())); + } + return Ok(decoded.trim().into()); + } + if let Some(message) = errors::detect_payload(text) { + return Err(LyricsError::classified( + errors::classify_payload(0, &message), + message, + )); + } + if !text.is_empty() && !text.starts_with('{') { + return Ok(text.into()); + } + Err(LyricsError::Other( + "failed to decode musixmatch response".into(), + )) + } + + pub(super) fn musixmatch( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let language = lowercase(request.options.musixmatch_language.trim()); + if !language.is_empty() { + match self.musixmatch_payload(request, "translate", &language, check) { + Ok(text) => { + if let Ok(lyrics) = + from_text(&text, "Musixmatch", &format!("Musixmatch ({language})")) + { + return Ok(lyrics); + } + } + Err(error @ LyricsError::Cancelled(_)) => return Err(error), + _ => {} + } + } + from_text( + &self.musixmatch_payload(request, "word", "", check)?, + "Musixmatch", + "Musixmatch", + ) + } + + pub fn lyrics_plus( + &self, + request: &SearchRequest, + isrc: &str, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + if request.track.trim().is_empty() || request.artist.trim().is_empty() { + return Err(LyricsError::Other( + "lyricsplus: missing track or artist".into(), + )); + } + let mut last = LyricsError::NotFound("lyricsplus: no lyrics found".into()); + for endpoint in [ + "https://lyricsplus.prjktla.workers.dev/v2/lyrics/get", + "https://lyricsplus.binimum.org/v2/lyrics/get", + ] { + match self.lyrics_plus_server(endpoint, request, isrc, check) { + Ok(lyrics) if lyrics.has_usable_text() => return Ok(lyrics), + Err(error @ LyricsError::Cancelled(_)) => return Err(error), + Err(error) => last = error, + _ => {} + } + } + Err(last) + } + + fn lyrics_plus_server( + &self, + endpoint: &str, + request: &SearchRequest, + isrc: &str, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let mut get = Request::new(endpoint); + get.allowed = &[200, 404]; + get.params = pairs(&[("title", &request.track), ("artist", &request.artist)]); + if request.duration > 0.0 { + get.params + .insert("duration".into(), format!("{:.3}", request.duration)); + } + if !isrc.trim().is_empty() { + get.params.insert("isrc".into(), isrc.trim().into()); + } + let response = self.http.get(get, check)?; + if response.status == 404 { + if !isrc.trim().is_empty() { + return self.lyrics_plus_server(endpoint, request, "", check); + } + return Err(LyricsError::NotFound("lyrics not found".into())); + } + let payload: payloads::KpoeResponse = http::decode(&response.body, true, check)?; + if payload.lyrics.as_ref().is_none_or(Vec::is_empty) { + return Err(LyricsError::Other("lyricsplus returned no lines".into())); + } + let text = payloads::format_kpoe( + &payload, + request.options.multi_person_word_by_word, + request.options.apple_elrc_word_sync, + ); + if text.trim().is_empty() { + return Err(LyricsError::Other( + "lyricsplus produced empty lyrics".into(), + )); + } + Ok(LyricsResponse::from_text(&text, "LyricsPlus", "LyricsPlus")) + } +} diff --git a/rust_backend/crates/providers/src/lyrics/cache.rs b/rust_backend/crates/providers/src/lyrics/cache.rs new file mode 100644 index 00000000..58dbe964 --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/cache.rs @@ -0,0 +1,307 @@ +//! Go-compatible successful-lyrics cache with bounded, atomic persistence. + +use serde::{Deserialize, Serialize}; +use spotiflac_core::lyrics::LyricsResponse; +use std::collections::BTreeMap; +use std::fs::{self, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +pub const MAX_ENTRIES: usize = 500; +pub const TTL: Duration = Duration::from_secs(24 * 60 * 60); +const MAX_PERSISTED_BYTES: u64 = 64 << 20; + +#[derive(Clone)] +struct Entry { + response: Arc<LyricsResponse>, + expires_at: SystemTime, +} + +#[derive(Default, Deserialize, Serialize)] +#[serde(default)] +struct Snapshot { + version: u32, + entries: BTreeMap<String, PersistedEntry>, +} + +#[derive(Deserialize, Serialize)] +struct PersistedEntry { + response: Option<LyricsResponse>, + expires_at: i64, +} + +#[derive(Default)] +struct State { + entries: BTreeMap<String, Entry>, + path: Option<PathBuf>, + pending: Option<(PathBuf, BTreeMap<String, Entry>)>, + writing: bool, + flush: bool, + error: Option<String>, +} + +#[derive(Default)] +struct Inner { + state: Mutex<State>, + ready: Condvar, +} + +#[derive(Default)] +pub struct LyricsCache { + inner: Arc<Inner>, +} + +fn clone_response(response: &LyricsResponse) -> LyricsResponse { + let mut result = response.clone(); + // append([]LyricsLine(nil), empty...) in Go collapses [] to null. + if result.lines().is_empty() { + result.lines = None; + } + result +} + +impl LyricsCache { + pub fn get(&self, key: &str, now: SystemTime) -> Option<LyricsResponse> { + let state = self.inner.state.lock().expect("lyrics cache lock"); + state + .entries + .get(key) + .filter(|entry| now <= entry.expires_at) + .map(|entry| clone_response(&entry.response)) + } + + pub fn set(&self, key: String, response: &LyricsResponse, now: SystemTime) { + let mut state = self.inner.state.lock().expect("lyrics cache lock"); + if state.entries.len() >= MAX_ENTRIES { + state.entries.retain(|_, entry| now <= entry.expires_at); + while state.entries.len() >= MAX_ENTRIES { + let oldest = state + .entries + .iter() + .min_by_key(|(_, entry)| entry.expires_at) + .map(|(key, _)| key.clone()) + .expect("full lyrics cache"); + state.entries.remove(&oldest); + } + } + state.entries.insert( + key, + Entry { + response: Arc::new(clone_response(response)), + expires_at: now + TTL, + }, + ); + self.schedule(&mut state); + } + + pub fn len(&self) -> usize { + self.inner + .state + .lock() + .expect("lyrics cache lock") + .entries + .len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn clean_expired(&self, now: SystemTime) -> usize { + let mut state = self.inner.state.lock().expect("lyrics cache lock"); + let before = state.entries.len(); + state.entries.retain(|_, entry| now <= entry.expires_at); + before - state.entries.len() + } + + pub fn clear(&self) -> usize { + let mut state = self.inner.state.lock().expect("lyrics cache lock"); + let count = state.entries.len(); + state.entries.clear(); + self.schedule(&mut state); + count + } + + /// Retain the pending disk snapshot even if memory pressure arrives during + /// the debounce window. A later process can still restore successful data. + pub fn drop_memory(&self) -> usize { + let mut state = self.inner.state.lock().expect("lyrics cache lock"); + let count = state.entries.len(); + state.entries.clear(); + count + } + + /// The path must come from the native application's private cache directory. + /// Missing, invalid or expired files are ignored, matching Go restoration. + pub fn set_persistence_path(&self, path: &Path, now: SystemTime) { + if path.as_os_str().is_empty() || path == Path::new(".") { + return; + } + let loaded = read_snapshot(path).unwrap_or_default(); + let mut state = self.inner.state.lock().expect("lyrics cache lock"); + state.path = Some(path.to_owned()); + if loaded.version == 1 { + for (key, entry) in loaded.entries { + if state.entries.len() >= MAX_ENTRIES { + break; + } + let Some(response) = entry.response else { + continue; + }; + let Some(expires_at) = u64::try_from(entry.expires_at) + .ok() + .and_then(|seconds| UNIX_EPOCH.checked_add(Duration::from_secs(seconds))) + else { + continue; + }; + if now < expires_at { + state.entries.entry(key).or_insert_with(|| Entry { + response: Arc::new(clone_response(&response)), + expires_at, + }); + } + } + } + if state.pending.is_some() { + self.schedule(&mut state); + } + } + + /// Flush the last scheduled snapshot and report any persistence error. + pub fn flush(&self) -> Result<(), String> { + let mut state = self.inner.state.lock().expect("lyrics cache lock"); + state.flush = true; + self.inner.ready.notify_all(); + while state.writing { + state = self.inner.ready.wait(state).expect("lyrics cache lock"); + } + state.flush = false; + state.error.clone().map_or(Ok(()), Err) + } + + fn schedule(&self, state: &mut State) { + let Some(path) = state.path.clone() else { + return; + }; + // Share immutable response bodies across the pending snapshot; writes + // during the debounce window must not copy every track's lyrics again. + state.pending = Some((path, state.entries.clone())); + if state.writing { + return; + } + state.writing = true; + let inner = Arc::clone(&self.inner); + if let Err(error) = std::thread::Builder::new() + .name("lyrics-cache".into()) + .spawn(move || persist(inner)) + { + state.writing = false; + state.error = Some(error.to_string()); + } + } +} + +impl Drop for LyricsCache { + fn drop(&mut self) { + let _ = self.flush(); + } +} + +fn read_snapshot(path: &Path) -> Option<Snapshot> { + let file = OpenOptions::new() + .read(true) + .custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32) + .open(path) + .ok()?; + let metadata = file.metadata().ok()?; + if !metadata.is_file() || metadata.len() > MAX_PERSISTED_BYTES { + return None; + } + let mut bytes = Vec::new(); + file.take(MAX_PERSISTED_BYTES + 1) + .read_to_end(&mut bytes) + .ok()?; + if bytes.len() as u64 > MAX_PERSISTED_BYTES { + return None; + } + serde_json::from_slice(&bytes).ok() +} + +fn persist(inner: Arc<Inner>) { + let mut delay = Duration::from_millis(500); + let mut state = inner.state.lock().expect("lyrics cache lock"); + loop { + state = inner + .ready + .wait_timeout_while(state, delay, |state| !state.flush) + .expect("lyrics cache lock") + .0; + let Some((path, snapshot)) = state.pending.take() else { + break; + }; + drop(state); + let result = write_snapshot(&path, &snapshot); + state = inner.state.lock().expect("lyrics cache lock"); + state.error = result.err().map(|error| error.to_string()); + if state.pending.is_none() { + break; + } + delay = Duration::from_millis(100); + } + state.writing = false; + inner.ready.notify_all(); +} + +fn write_snapshot(path: &Path, entries: &BTreeMap<String, Entry>) -> std::io::Result<()> { + #[derive(Serialize)] + struct BorrowedEntry<'a> { + response: &'a LyricsResponse, + expires_at: i64, + } + #[derive(Serialize)] + struct BorrowedSnapshot<'a> { + version: u32, + entries: BTreeMap<&'a str, BorrowedEntry<'a>>, + } + let snapshot = BorrowedSnapshot { + version: 1, + entries: entries + .iter() + .map(|(key, entry)| { + ( + key.as_str(), + BorrowedEntry { + response: &entry.response, + expires_at: entry + .expires_at + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64, + }, + ) + }) + .collect(), + }; + let parent = path + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(parent)?; + let bytes = serde_json::to_vec(&snapshot)?; + if bytes.len() as u64 > MAX_PERSISTED_BYTES { + return Err(std::io::Error::other( + "lyrics cache exceeds persistence size limit", + )); + } + let mut temporary = tempfile::NamedTempFile::new_in(parent)?; + temporary.write_all(&bytes)?; + temporary.as_file().sync_all()?; + temporary.persist(path).map_err(|error| error.error)?; + Ok(()) +} diff --git a/rust_backend/crates/providers/src/lyrics/calls.rs b/rust_backend/crates/providers/src/lyrics/calls.rs new file mode 100644 index 00000000..edaf2e1d --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/calls.rs @@ -0,0 +1,123 @@ +//! Native wait dependencies shared by lyrics flights and their extension VMs. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Default)] +struct State { + next: u64, + edges: BTreeMap<u64, BTreeMap<u64, usize>>, +} + +#[derive(Clone, Debug, Default)] +pub struct CallGraph(Arc<Mutex<State>>); + +#[derive(Debug)] +struct Node { + graph: CallGraph, + id: u64, +} + +/// A native capability; never deserialize a caller-supplied node identifier. +#[derive(Clone, Debug)] +pub struct CallNode(Arc<Node>); + +pub struct CallWait { + caller: CallNode, + target: CallNode, +} + +impl CallGraph { + pub fn node(&self) -> CallNode { + let mut state = self.0.lock().expect("lyrics dependency lock"); + let id = state.next; + state.next = state + .next + .checked_add(1) + .expect("lyrics node IDs exhausted"); + CallNode(Arc::new(Node { + graph: self.clone(), + id, + })) + } +} + +impl CallNode { + /// Register before queueing or joining work, and retain until it completes. + /// The graph lock is never held while waiting for a VM or service result. + pub fn wait_for(&self, target: &Self) -> Result<CallWait, String> { + if !Arc::ptr_eq(&self.0.graph.0, &target.0.graph.0) { + return Err("lyrics caller belongs to a different backend".into()); + } + let mut state = self.0.graph.0.lock().expect("lyrics dependency lock"); + let mut pending = vec![target.0.id]; + let mut visited = BTreeSet::new(); + while let Some(id) = pending.pop() { + if id == self.0.id { + return Err("recursive lyrics call would deadlock".into()); + } + if visited.insert(id) + && let Some(edges) = state.edges.get(&id) + { + pending.extend(edges.keys().copied()); + } + } + *state + .edges + .entry(self.0.id) + .or_default() + .entry(target.0.id) + .or_default() += 1; + Ok(CallWait { + caller: self.clone(), + target: target.clone(), + }) + } +} + +impl Drop for CallWait { + fn drop(&mut self) { + let mut state = self + .caller + .0 + .graph + .0 + .lock() + .expect("lyrics dependency lock"); + let edges = state.edges.get_mut(&self.caller.0.id).expect("caller edge"); + let count = edges.get_mut(&self.target.0.id).expect("target edge"); + *count -= 1; + if *count == 0 { + edges.remove(&self.target.0.id); + } + if edges.is_empty() { + state.edges.remove(&self.caller.0.id); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_cycles_and_keeps_shared_edges_until_the_last_waiter_leaves() { + let graph = CallGraph::default(); + let a = graph.node(); + let b = graph.node(); + let c = graph.node(); + assert!(a.wait_for(&a).is_err()); + let ab = a.wait_for(&b).unwrap(); + let second_ab = a.wait_for(&b).unwrap(); + let bc = b.wait_for(&c).unwrap(); + assert!(c.wait_for(&a).is_err()); + drop(ab); + assert!(c.wait_for(&a).is_err()); + drop(second_ab); + let ca = c.wait_for(&a).unwrap(); + assert!(a.wait_for(&b).is_err()); + assert!(a.wait_for(&CallGraph::default().node()).is_err()); + drop((bc, ca)); + assert!(graph.0.lock().unwrap().edges.is_empty()); + } +} diff --git a/rust_backend/crates/providers/src/lyrics/http.rs b/rust_backend/crates/providers/src/lyrics/http.rs new file mode 100644 index 00000000..0a3de5e2 --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/http.rs @@ -0,0 +1,208 @@ +use super::{Check, LyricsError}; +use serde::de::DeserializeOwned; +use spotiflac_core::app_version::AppVersion; +use spotiflac_core::lyrics::{decode_document, decode_response, errors}; +use spotiflac_network::{HttpRequest, HttpStream, NetworkService, NetworkSession}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; +use url::Url; + +pub type Params = BTreeMap<String, String>; +pub const BROWSER_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"; + +pub fn pairs(values: &[(&str, &str)]) -> Params { + values + .iter() + .map(|(key, value)| ((*key).into(), (*value).into())) + .collect() +} + +pub struct Request<'a> { + pub endpoint: &'a str, + pub params: Params, + pub headers: Params, + pub timeout: Duration, + pub max_bytes: usize, + /// Empty accepts any status and reads its payload. Explicit non-200 allowed + /// statuses return immediately without reading a potentially stalled body. + pub allowed: &'static [u16], + pub unavailable_errors: bool, +} + +impl<'a> Request<'a> { + pub fn new(endpoint: &'a str) -> Self { + Self { + endpoint, + params: Params::new(), + headers: pairs(&[("Accept", "application/json")]), + timeout: Duration::from_secs(15), + max_bytes: 16 << 20, + allowed: &[200], + unavailable_errors: false, + } + } +} + +pub struct Response { + pub status: u16, + pub body: Vec<u8>, +} + +pub struct LyricsHttp { + session: Arc<NetworkSession>, + app_version: AppVersion, + overrides: BTreeMap<String, Url>, +} + +impl LyricsHttp { + pub fn new( + network: &Arc<NetworkService>, + version: impl Into<AppVersion>, + overrides: BTreeMap<String, String>, + ) -> Result<Self, LyricsError> { + let overrides = overrides + .into_iter() + .map(|(origin, replacement)| { + let url = Url::parse(&replacement) + .map_err(|error| LyricsError::Other(error.to_string()))?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + { + return Err(LyricsError::Other( + "provider override requires HTTPS without credentials".into(), + )); + } + Ok((origin, url)) + }) + .collect::<Result<_, _>>()?; + Ok(Self { + session: network.native_session(Duration::from_secs(20)), + app_version: version.into(), + overrides, + }) + } + + pub fn get(&self, request: Request<'_>, check: &Check<'_>) -> Result<Response, LyricsError> { + check().map_err(LyricsError::Cancelled)?; + let mut url = + Url::parse(request.endpoint).map_err(|error| LyricsError::Other(error.to_string()))?; + if !request.params.is_empty() { + url.query_pairs_mut().extend_pairs(&request.params); + } + if let Some(base) = self.overrides.get(&url.origin().ascii_serialization()) { + let mut replaced = base.clone(); + replaced.set_path(url.path()); + replaced.set_query(url.query()); + replaced.set_fragment(None); + url = replaced; + } + self.session + .validate_url(url.as_str()) + .map_err(LyricsError::Other)?; + let mut stream = self + .session + .open_stream( + HttpRequest { + url: url.into(), + method: "GET".into(), + body: String::new(), + headers: request.headers, + default_json: false, + user_agent: self.app_version.user_agent(), + }, + request.timeout, + request.timeout, + check, + ) + .map_err(|error| transport_error(error, check))?; + let status = stream.response.status; + if !request.allowed.is_empty() && !request.allowed.contains(&status) { + let message = format!("HTTP {status}"); + return Err(if request.unavailable_errors { + LyricsError::Unavailable(message) + } else { + LyricsError::classified(errors::http_status(status), message) + }); + } + if status != 200 && !request.allowed.is_empty() { + return Ok(Response { + status, + body: Vec::new(), + }); + } + let body = read(&mut stream, request.max_bytes, check).map_err(|error| { + if request.unavailable_errors && matches!(error, LyricsError::Other(_)) { + LyricsError::Unavailable(error.to_string()) + } else { + error + } + })?; + Ok(Response { status, body }) + } +} + +fn transport_error(error: String, check: &Check<'_>) -> LyricsError { + if let Err(cancelled) = check() { + return LyricsError::Cancelled(cancelled); + } + if error == "network policy changed" { + return LyricsError::Cancelled(error); + } + if error.starts_with("invalid ") || error == "blocking HTTP host called from async executor" { + LyricsError::Other(error) + } else { + LyricsError::Unavailable(error) + } +} + +fn read(stream: &mut HttpStream, limit: usize, check: &Check<'_>) -> Result<Vec<u8>, LyricsError> { + let mut result = Vec::new(); + let mut buffer = [0; 16 << 10]; + loop { + let capacity = buffer.len().min(limit + 1 - result.len()); + let count = stream + .read(&mut buffer[..capacity], check) + .map_err(|error| transport_error(error, check))?; + if count == 0 { + break; + } + result.extend_from_slice(&buffer[..count]); + if result.len() > limit { + return Err(LyricsError::Other(format!( + "response exceeds {limit} bytes" + ))); + } + } + check().map_err(LyricsError::Cancelled)?; + Ok(result) +} + +pub fn decode<T: DeserializeOwned>( + bytes: &[u8], + streaming: bool, + check: &Check<'_>, +) -> Result<T, LyricsError> { + check().map_err(LyricsError::Cancelled)?; + let result = if streaming { + decode_response(bytes) + } else { + decode_document(bytes) + }; + check().map_err(LyricsError::Cancelled)?; + result.map_err(|error| { + let message = format!("failed to decode lyrics response: {error}"); + if streaming + && error.is_eof() + && bytes + .iter() + .any(|byte| !matches!(byte, b' ' | b'\t' | b'\n' | b'\r')) + { + LyricsError::Unavailable(message) + } else { + LyricsError::Other(message) + } + }) +} diff --git a/rust_backend/crates/providers/src/lyrics/lrclib.rs b/rust_backend/crates/providers/src/lyrics/lrclib.rs new file mode 100644 index 00000000..5768be91 --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/lrclib.rs @@ -0,0 +1,226 @@ +use super::{Check, LyricsError, LyricsFetcher, SearchRequest}; +use serde::de::DeserializeOwned; +use spotiflac_core::lyrics::{LyricsResponse, decode_response, errors, lrclib, matching}; +use spotiflac_network::{HttpRequest, NetworkService, NetworkSession}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; +use url::Url; + +pub struct LrcLibClient { + session: Arc<NetworkSession>, + endpoint: Url, + user_agent: String, +} + +impl LrcLibClient { + pub fn new(network: &Arc<NetworkService>, user_agent: String) -> Self { + Self::with_endpoint(network, "https://lrclib.net", user_agent) + .expect("built-in LRCLIB endpoint") + } + + /// Native configuration only; JavaScript cannot choose provider endpoints. + pub fn with_endpoint( + network: &Arc<NetworkService>, + endpoint: &str, + user_agent: String, + ) -> Result<Self, LyricsError> { + let endpoint = + Url::parse(endpoint).map_err(|error| LyricsError::Other(error.to_string()))?; + if endpoint.scheme() != "https" + || endpoint.host_str().is_none() + || !endpoint.username().is_empty() + || endpoint.password().is_some() + { + return Err(LyricsError::Other( + "LRCLIB requires an HTTPS endpoint without credentials".into(), + )); + } + Ok(Self { + session: network.native_session(Duration::from_secs(15)), + endpoint, + user_agent, + }) + } + + fn get<T: DeserializeOwned>( + &self, + path: &str, + parameters: &[(&str, &str)], + check: &Check<'_>, + ) -> Result<T, LyricsError> { + check().map_err(LyricsError::Cancelled)?; + let mut url = self.endpoint.clone(); + url.set_path(path); + url.set_query(None); + url.set_fragment(None); + url.query_pairs_mut() + .extend_pairs(parameters.iter().copied()); + self.session + .validate_url(url.as_str()) + .map_err(LyricsError::Other)?; + let response = self + .session + .request( + HttpRequest { + url: url.into(), + method: "GET".into(), + body: String::new(), + headers: BTreeMap::new(), + default_json: false, + user_agent: self.user_agent.clone(), + }, + check, + ) + .map_err(|error| { + if let Err(cancelled) = check() { + return LyricsError::Cancelled(cancelled); + } + if error == "network policy changed" { + return LyricsError::Cancelled(error); + } + if error.starts_with("response body exceeds ") + || error.starts_with("invalid ") + || error == "blocking HTTP host called from async executor" + { + LyricsError::Other(format!("failed to fetch lyrics: {error}")) + } else { + LyricsError::Unavailable(format!("failed to fetch lyrics: {error}")) + } + })?; + check().map_err(LyricsError::Cancelled)?; + if response.status == 404 { + return Err(LyricsError::NotFound("lyrics not found".into())); + } + if response.status != 200 { + return Err(LyricsError::classified( + errors::http_status(response.status), + format!("unexpected status code: {}", response.status), + )); + } + let result = decode_response(&response.body).map_err(|error| { + let message = format!("failed to decode response: {error}"); + // Go's streaming decoder reports a truncated JSON value as + // io.ErrUnexpectedEOF, which its connectivity classifier cools down. + // An entirely empty body instead produces io.EOF and remains Other. + if error.is_eof() + && response + .body + .iter() + .any(|byte| !matches!(byte, b' ' | b'\t' | b'\r' | b'\n')) + { + LyricsError::Unavailable(message) + } else { + LyricsError::Other(message) + } + }); + check().map_err(LyricsError::Cancelled)?; + result + } + + pub fn metadata( + &self, + artist: &str, + track: &str, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let response: lrclib::LrcLibResponse = self.get( + "/api/get", + &[("artist_name", artist), ("track_name", track)], + check, + )?; + if !matching::titles_match(response.track_name(), track, false) + || !matching::artists_match(&response.artist_name, artist) + { + return Err(LyricsError::NotFound( + "LRCLIB returned mismatched track metadata".into(), + )); + } + Ok(response.into_lyrics()) + } + + pub fn search( + &self, + query: &str, + track: &str, + artist: &str, + duration: f64, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let results: Option<Vec<lrclib::LrcLibResponse>> = + self.get("/api/search", &[("q", query)], check)?; + let results = results.unwrap_or_default(); + if results.is_empty() { + return Err(LyricsError::NotFound("no lyrics found".into())); + } + lrclib::select(&results, query, track, artist, duration) + .map(lrclib::LrcLibResponse::into_lyrics) + .ok_or_else(|| LyricsError::NotFound("no matching lyrics found".into())) + } + + pub fn fetch_lyrics( + &self, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + let primary = matching::primary_artist(&request.artist); + let simplified = matching::simplify_track(&request.track); + let mut metadata = vec![(primary.as_str(), request.track.as_str(), "LRCLIB")]; + if primary != request.artist { + metadata.push((&request.artist, &request.track, "LRCLIB")); + } + if simplified != request.track { + metadata.push((&primary, &simplified, "LRCLIB (simplified)")); + } + for (artist, track, source) in metadata { + if let Some(lyrics) = accepted(self.metadata(artist, track, check), source)? { + return Ok(lyrics); + } + } + let mut searches = vec![(request.track.as_str(), "LRCLIB Search")]; + if simplified != request.track { + searches.push((&simplified, "LRCLIB Search (simplified)")); + } + for (track, source) in searches { + let result = self.search( + &format!("{primary} {track}"), + track, + &primary, + request.duration, + check, + ); + if let Some(lyrics) = accepted(result, source)? { + return Ok(lyrics); + } + } + Err(LyricsError::NotFound("LRCLIB: no lyrics found".into())) + } +} + +fn accepted( + result: Result<LyricsResponse, LyricsError>, + source: &str, +) -> Result<Option<LyricsResponse>, LyricsError> { + match result { + Ok(mut lyrics) if !lyrics.lines().is_empty() || lyrics.instrumental => { + lyrics.source = source.into(); + Ok(Some(lyrics)) + } + Err(error @ (LyricsError::Unavailable(_) | LyricsError::Cancelled(_))) => Err(error), + _ => Ok(None), + } +} + +impl LyricsFetcher for LrcLibClient { + fn fetch( + &self, + provider: &str, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError> { + if provider != "lrclib" { + return Err(LyricsError::Other(format!("unknown provider: {provider}"))); + } + self.fetch_lyrics(request, check) + } +} diff --git a/rust_backend/crates/providers/src/lyrics/service.rs b/rust_backend/crates/providers/src/lyrics/service.rs new file mode 100644 index 00000000..832cc50c --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/service.rs @@ -0,0 +1,567 @@ +use super::{CallGraph, CallNode, Check, LyricsError, cache::LyricsCache}; +use spotiflac_core::lyrics::{LyricsResponse, config, config::FetchOptions, matching}; +use spotiflac_core::matching::lowercase; +use std::collections::BTreeMap; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::Path; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex, mpsc}; +use std::time::{Duration, Instant, SystemTime}; + +const POLL: Duration = Duration::from_millis(25); +const NEGATIVE_TTL: Duration = Duration::from_secs(5 * 60); +const UNAVAILABLE_COOLDOWN: Duration = Duration::from_secs(10 * 60); +const PRIORITY_GRACE: Duration = Duration::from_secs(5); +const MAX_NEGATIVE: usize = 500; +const MAX_FLIGHTS: usize = 64; + +#[derive(Clone, Debug, Default)] +pub struct SearchRequest { + pub spotify_id: String, + pub track: String, + pub artist: String, + pub duration: f64, + pub options: FetchOptions, + /// Native wait dependency, replaced by the service's flight before dispatch. + pub caller: Option<CallNode>, +} + +/// Built-in clients and the installed extension manager implement this boundary. +/// Calls execute on native workers. Implementations must observe `check` during +/// blocking work and must not reenter the same JavaScript VM from a lyrics call. +pub trait LyricsFetcher: Send + Sync + 'static { + fn call_graph(&self) -> Option<CallGraph> { + None + } + + /// Pure owner-liveness check, also applied before cached/heuristic results. + /// This can run while the service state is locked; do not block or call a VM. + fn check(&self) -> Result<(), LyricsError> { + Ok(()) + } + + fn extensions(&self) -> Vec<String> { + Vec::new() + } + + fn fetch( + &self, + provider: &str, + request: &SearchRequest, + check: &Check<'_>, + ) -> Result<LyricsResponse, LyricsError>; +} + +type FetchResult = Result<LyricsResponse, LyricsError>; + +struct Flight { + node: CallNode, + waiters: AtomicUsize, + result: Mutex<Option<FetchResult>>, + ready: Condvar, +} + +struct Waiter(Arc<Flight>); +impl Drop for Waiter { + fn drop(&mut self) { + self.0.waiters.fetch_sub(1, Ordering::AcqRel); + } +} + +#[derive(Default)] +struct State { + // Retain an empty selection separately from the effective default order. + providers: Vec<String>, + options: FetchOptions, + generation: u64, + negative: BTreeMap<String, Instant>, + health: BTreeMap<String, Instant>, + flights: BTreeMap<(u64, String), Arc<Flight>>, + active: usize, +} + +struct Inner { + fetcher: Arc<dyn LyricsFetcher>, + calls: CallGraph, + state: Mutex<State>, + idle: Condvar, + closed: AtomicBool, + cache: LyricsCache, +} + +pub struct LyricsService { + inner: Arc<Inner>, +} + +impl LyricsService { + pub fn new(fetcher: Arc<dyn LyricsFetcher>) -> Self { + let calls = fetcher.call_graph().unwrap_or_default(); + Self { + inner: Arc::new(Inner { + fetcher, + calls, + state: Mutex::default(), + idle: Condvar::new(), + closed: AtomicBool::new(false), + cache: LyricsCache::default(), + }), + } + } + + pub fn providers(&self) -> Vec<String> { + config::provider_order( + &self + .inner + .state + .lock() + .expect("lyrics service lock") + .providers, + ) + } + + pub fn set_providers(&self, providers: &[String]) -> Result<(), LyricsError> { + let normalized = config::normalize_provider_order(providers); + let mut state = self.inner.state.lock().expect("lyrics service lock"); + self.inner.check(&|| Ok(()))?; + state.health.clear(); + if state.providers != normalized { + state.providers = normalized; + state.generation = state.generation.wrapping_add(1); + self.inner.cache.clear(); + } + Ok(()) + } + + pub fn options(&self) -> FetchOptions { + self.inner + .state + .lock() + .expect("lyrics service lock") + .options + .clone() + } + + pub fn set_options(&self, mut options: FetchOptions) -> Result<(), LyricsError> { + options.normalize(); + let mut state = self.inner.state.lock().expect("lyrics service lock"); + self.inner.check(&|| Ok(()))?; + if state.options != options { + state.options = options; + state.generation = state.generation.wrapping_add(1); + self.inner.cache.clear(); + } + Ok(()) + } + + pub fn set_persistence_path(&self, path: &Path) -> Result<(), LyricsError> { + let _state = self.inner.state.lock().expect("lyrics service lock"); + self.inner.check(&|| Ok(()))?; + self.inner + .cache + .set_persistence_path(path, SystemTime::now()); + Ok(()) + } + + pub fn cache_size(&self) -> usize { + self.inner.cache.len() + } + + pub fn clear_cache(&self) -> Result<usize, LyricsError> { + let mut state = self.inner.state.lock().expect("lyrics service lock"); + self.inner.check(&|| Ok(()))?; + state.generation = state.generation.wrapping_add(1); + Ok(self.inner.cache.clear()) + } + + pub fn drop_memory(&self) -> Result<usize, LyricsError> { + let mut state = self.inner.state.lock().expect("lyrics service lock"); + self.inner.check(&|| Ok(()))?; + state.generation = state.generation.wrapping_add(1); + state.negative.clear(); + state.health.clear(); + Ok(self.inner.cache.drop_memory()) + } + + pub fn fetch(&self, mut request: SearchRequest, check: &Check<'_>) -> FetchResult { + self.inner.check(check)?; + let extensions = self.inner.fetcher.extensions(); + let (flight, start, flight_key, providers, _dependency) = { + let mut state = self.inner.state.lock().expect("lyrics service lock"); + self.inner.check(check)?; + request.options = state.options.clone(); + let providers = config::provider_order(&state.providers); + let key = config::cache_key( + &request.spotify_id, + &request.track, + &request.artist, + request.duration, + &providers, + &extensions, + &request.options, + ); + let now = Instant::now(); + if state.negative.get(&key).is_some_and(|expiry| now < *expiry) { + return Err(LyricsError::NotFound("lyrics not found (cached)".into())); + } + state.negative.remove(&key); + let flight_key = (state.generation, key); + let existing = state + .flights + .get(&flight_key) + .filter(|flight| flight.waiters.load(Ordering::Acquire) > 0) + .cloned(); + let (flight, start) = if let Some(flight) = existing { + (flight, false) + } else { + if state.active >= MAX_FLIGHTS { + return Err(LyricsError::Other( + "too many concurrent lyrics searches".into(), + )); + } + let flight = Arc::new(Flight { + node: self.inner.calls.node(), + waiters: AtomicUsize::new(0), + result: Mutex::new(None), + ready: Condvar::new(), + }); + (flight, true) + }; + let dependency = request + .caller + .as_ref() + .map(|caller| caller.wait_for(&flight.node)) + .transpose() + .map_err(LyricsError::Recursive)?; + if start { + state + .flights + .insert(flight_key.clone(), Arc::clone(&flight)); + state.active += 1; + } + request.caller = Some(flight.node.clone()); + flight.waiters.fetch_add(1, Ordering::AcqRel); + (flight, start, flight_key, providers, dependency) + }; + let _waiter = Waiter(Arc::clone(&flight)); + if start { + let inner = Arc::clone(&self.inner); + let worker_flight = Arc::clone(&flight); + let worker_key = flight_key.clone(); + let spawned = std::thread::Builder::new() + .name("lyrics-search".into()) + .spawn(move || { + let check = || { + if inner.closed.load(Ordering::Acquire) { + Err("lyrics service is closed".into()) + } else if worker_flight.waiters.load(Ordering::Acquire) == 0 { + Err("lyrics request cancelled".into()) + } else { + Ok(()) + } + }; + let result = catch_unwind(AssertUnwindSafe(|| { + inner.fetch_uncached( + &request, + &providers, + &extensions, + worker_key.0, + &check, + ) + })) + .unwrap_or_else(|_| Err(LyricsError::Other("lyrics provider panicked".into()))); + inner.finish(&worker_key, &worker_flight, result); + }); + if let Err(error) = spawned { + self.inner.finish( + &flight_key, + &flight, + Err(LyricsError::Other(error.to_string())), + ); + } + } + loop { + self.inner.check(check)?; + let result = flight.result.lock().expect("lyrics flight lock"); + if let Some(value) = result.as_ref() { + let value = value.clone(); + drop(result); + self.inner.check(check)?; + return value; + } + drop( + flight + .ready + .wait_timeout(result, POLL) + .expect("lyrics flight lock"), + ); + } + } + + /// Cancel workers, wait for their HTTP/extension calls to unwind, then flush + /// successful data. Native owners call this before releasing the service. + pub fn shutdown(&self) -> Result<(), String> { + self.inner.closed.store(true, Ordering::Release); + let mut state = self.inner.state.lock().expect("lyrics service lock"); + while state.active > 0 { + state = self.inner.idle.wait(state).expect("lyrics service lock"); + } + drop(state); + self.inner.cache.flush() + } +} + +impl Drop for LyricsService { + fn drop(&mut self) { + let _ = self.shutdown(); + } +} + +impl Inner { + fn check(&self, check: &Check<'_>) -> Result<(), LyricsError> { + if self.closed.load(Ordering::Acquire) { + return Err(LyricsError::Cancelled("lyrics service is closed".into())); + } + check().map_err(LyricsError::Cancelled)?; + self.fetcher.check() + } + + fn finish(&self, key: &(u64, String), flight: &Arc<Flight>, result: FetchResult) { + let mut state = self.state.lock().expect("lyrics service lock"); + if state.generation == key.0 && !self.closed.load(Ordering::Acquire) { + if result.as_ref().is_err_and(|error| { + !matches!(error, LyricsError::Cancelled(_) | LyricsError::Recursive(_)) + }) { + let now = Instant::now(); + state.negative.retain(|_, expiry| now < *expiry); + while state.negative.len() >= MAX_NEGATIVE { + state.negative.pop_first(); + } + state.negative.insert(key.1.clone(), now + NEGATIVE_TTL); + } else if result.is_ok() { + state.negative.remove(&key.1); + } + } + *flight.result.lock().expect("lyrics flight lock") = Some(result); + flight.ready.notify_all(); + if state + .flights + .get(key) + .is_some_and(|current| Arc::ptr_eq(current, flight)) + { + state.flights.remove(key); + } + state.active -= 1; + self.idle.notify_all(); + } + + fn fetch_uncached( + &self, + request: &SearchRequest, + providers: &[String], + extensions: &[String], + generation: u64, + check: &Check<'_>, + ) -> FetchResult { + self.check(check)?; + let key = config::cache_key( + "", + &request.track, + &request.artist, + request.duration, + providers, + extensions, + &request.options, + ); + if matching::is_likely_instrumental(&request.track) { + let result = LyricsResponse { + instrumental: true, + source: "Heuristic: Instrumental".into(), + ..LyricsResponse::default() + }; + self.store(generation, key, &result, check)?; + return Ok(result); + } + let providers = config::resolve_order(providers, extensions); + let mut fallback = None; + if let Some(mut cached) = self.cache.get(&key, SystemTime::now()) { + let extension = cached.source.strip_prefix("Extension:"); + let selected = extension.is_some_and(|id| { + providers.contains(&format!("extension:{}", lowercase(id.trim()))) + }); + let has_extensions = providers.iter().any(|name| name.starts_with("extension:")); + if (extension.is_none() && !has_extensions) || selected { + cached.source.push_str(" (cached)"); + return Ok(cached); + } + if extension.is_none() { + fallback = Some(cached); + } + } + let result = self.search(&providers, request, generation, check, PRIORITY_GRACE); + self.check(check)?; + if let Err(error @ LyricsError::Cancelled(_)) = result { + return Err(error); + } + let recursive = match &result { + Err(error @ LyricsError::Recursive(_)) => Some(error.clone()), + _ => None, + }; + if let Ok(lyrics) = result + && lyrics.has_usable_text() + { + self.store(generation, key, &lyrics, check)?; + return Ok(lyrics); + } + if let Some(mut cached) = fallback { + cached.source.push_str(" (cached fallback)"); + return Ok(cached); + } + if let Some(error) = recursive { + return Err(error); + } + Err(LyricsError::Other( + "lyrics not found from any source".into(), + )) + } + + fn store( + &self, + generation: u64, + key: String, + response: &LyricsResponse, + check: &Check<'_>, + ) -> Result<(), LyricsError> { + let state = self.state.lock().expect("lyrics service lock"); + self.check(check)?; + if state.generation == generation { + self.cache.set(key, response, SystemTime::now()); + } + Ok(()) + } + + fn search( + &self, + providers: &[String], + request: &SearchRequest, + generation: u64, + check: &Check<'_>, + grace: Duration, + ) -> FetchResult { + let candidates: Vec<_> = { + let mut state = self.state.lock().expect("lyrics service lock"); + let now = Instant::now(); + state.health.retain(|_, expiry| now < *expiry); + providers + .iter() + .enumerate() + .filter(|(_, name)| !state.health.contains_key(*name)) + .collect() + }; + let stopped = AtomicBool::new(false); + let next = AtomicUsize::new(0); + let control = || { + if stopped.load(Ordering::Acquire) { + Err("lyrics provider search cancelled".into()) + } else { + check() + } + }; + std::thread::scope(|scope| { + let (sender, receiver) = mpsc::channel(); + for _ in 0..candidates.len().min(3) { + let sender = sender.clone(); + let candidates = &candidates; + let control = &control; + let next = &next; + scope.spawn(move || { + loop { + if control().is_err() { + break; + } + let Some(&(index, name)) = + candidates.get(next.fetch_add(1, Ordering::AcqRel)) + else { + break; + }; + let result = catch_unwind(AssertUnwindSafe(|| { + self.fetcher.fetch(name, request, control) + })) + .unwrap_or_else(|_| { + Err(LyricsError::Other("lyrics provider panicked".into())) + }); + if control().is_ok() { + let mut state = self.state.lock().expect("lyrics service lock"); + if state.generation == generation { + match &result { + Ok(lyrics) if lyrics.has_usable_text() => { + state.health.remove(name); + } + Err(LyricsError::Unavailable(_)) => { + state.health.insert( + name.clone(), + Instant::now() + UNAVAILABLE_COOLDOWN, + ); + } + _ => {} + } + } + } + if sender.send((index, result)).is_err() { + break; + } + } + }); + } + drop(sender); + let mut completed = vec![false; providers.len()]; + let mut best: Option<(usize, LyricsResponse)> = None; + let mut deadline = None; + let mut last_error = None; + let mut recursive = None; + let result = loop { + if let Err(error) = self.check(check) { + break Err(error); + } + if let Some((index, _)) = &best { + let pending = candidates + .iter() + .any(|(earlier, _)| earlier < index && !completed[*earlier]); + if !pending || deadline.is_some_and(|limit| Instant::now() >= limit) { + break Ok(best.take().expect("best lyrics result").1); + } + } + match receiver.recv_timeout(POLL) { + Ok((index, result)) => { + completed[index] = true; + if let Err(error @ LyricsError::Recursive(_)) = &result { + recursive = Some(error.clone()); + } + if let Err(error) = &result { + last_error = Some(error.clone()); + } + if let Ok(lyrics) = result + && lyrics.has_usable_text() + && best.as_ref().is_none_or(|(current, _)| index < *current) + { + best = Some((index, lyrics)); + deadline = Some(Instant::now() + grace); + } + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + break best.map(|(_, lyrics)| lyrics).ok_or_else(|| { + recursive.or(last_error).unwrap_or_else(|| { + LyricsError::Other("lyrics not found from any source".into()) + }) + }); + } + } + }; + stopped.store(true, Ordering::Release); + result + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/rust_backend/crates/providers/src/lyrics/service/tests.rs b/rust_backend/crates/providers/src/lyrics/service/tests.rs new file mode 100644 index 00000000..93bc5502 --- /dev/null +++ b/rust_backend/crates/providers/src/lyrics/service/tests.rs @@ -0,0 +1,453 @@ +use super::*; +use std::thread; + +type Handler = dyn Fn(&str, &SearchRequest, &Check<'_>) -> FetchResult + Send + Sync; +struct Mock { + handler: Box<Handler>, + extensions: Vec<String>, +} +impl LyricsFetcher for Mock { + fn extensions(&self) -> Vec<String> { + self.extensions.clone() + } + fn fetch(&self, provider: &str, request: &SearchRequest, check: &Check<'_>) -> FetchResult { + (self.handler)(provider, request, check) + } +} +fn service( + handler: impl Fn(&str, &SearchRequest, &Check<'_>) -> FetchResult + Send + Sync + 'static, +) -> Arc<LyricsService> { + let service = Arc::new(LyricsService::new(Arc::new(Mock { + handler: Box::new(handler), + extensions: vec!["example.lyrics".into()], + }))); + service.set_providers(&["lrclib".into()]).unwrap(); + service +} +fn request() -> SearchRequest { + SearchRequest { + track: "Song".into(), + artist: "Artist".into(), + duration: 180.0, + ..SearchRequest::default() + } +} +fn lyrics(source: &str) -> LyricsResponse { + LyricsResponse::from_text("[00:01.00]Line", "Example", source) +} +fn until(condition: impl Fn() -> bool) { + let deadline = Instant::now() + Duration::from_secs(3); + while !condition() { + assert!(Instant::now() < deadline, "condition timed out"); + thread::sleep(Duration::from_millis(2)); + } +} +fn waiters(service: &LyricsService) -> usize { + service + .inner + .state + .lock() + .unwrap() + .flights + .values() + .map(|flight| flight.waiters.load(Ordering::Acquire)) + .sum() +} +fn wait_gate(gate: &AtomicBool, check: &Check<'_>) -> Result<(), LyricsError> { + while !gate.load(Ordering::Acquire) { + check().map_err(LyricsError::Cancelled)?; + thread::sleep(Duration::from_millis(2)); + } + check().map_err(LyricsError::Cancelled) +} + +#[test] +fn eight_callers_share_work_and_cancelled_waiter_does_not_cancel_others() { + let calls = Arc::new(AtomicUsize::new(0)); + let gate = Arc::new(AtomicBool::new(false)); + let service = service({ + let calls = calls.clone(); + let gate = gate.clone(); + move |_, _, check| { + calls.fetch_add(1, Ordering::AcqRel); + wait_gate(&gate, check)?; + Ok(lyrics("LRCLIB")) + } + }); + let cancelled = Arc::new(AtomicBool::new(false)); + let mut workers = Vec::new(); + for index in 0..8 { + let service = service.clone(); + let cancelled = cancelled.clone(); + workers.push(thread::spawn(move || { + service.fetch(request(), &|| { + if index == 0 && cancelled.load(Ordering::Acquire) { + Err("caller cancelled".into()) + } else { + Ok(()) + } + }) + })); + } + until(|| waiters(&service) == 8); + cancelled.store(true, Ordering::Release); + assert_eq!( + workers.remove(0).join().unwrap(), + Err(LyricsError::Cancelled("caller cancelled".into())) + ); + gate.store(true, Ordering::Release); + for worker in workers { + assert_eq!(worker.join().unwrap().unwrap().source, "LRCLIB"); + } + assert_eq!(calls.load(Ordering::Acquire), 1); + assert_eq!( + service.fetch(request(), &|| Ok(())).unwrap().source, + "LRCLIB (cached)" + ); +} + +#[test] +fn abandoning_all_waiters_cancels_work_without_negative_caching() { + let calls = Arc::new(AtomicUsize::new(0)); + let service = service({ + let calls = calls.clone(); + move |_, _, check| { + if calls.fetch_add(1, Ordering::AcqRel) == 0 { + loop { + check().map_err(LyricsError::Cancelled)?; + thread::sleep(Duration::from_millis(2)); + } + } + Ok(lyrics("LRCLIB")) + } + }); + let cancelled = Arc::new(AtomicBool::new(false)); + let worker = { + let service = service.clone(); + let cancelled = cancelled.clone(); + thread::spawn(move || { + service.fetch(request(), &|| { + if cancelled.load(Ordering::Acquire) { + Err("cancelled".into()) + } else { + Ok(()) + } + }) + }) + }; + until(|| calls.load(Ordering::Acquire) == 1); + cancelled.store(true, Ordering::Release); + assert!(matches!( + worker.join().unwrap(), + Err(LyricsError::Cancelled(_)) + )); + until(|| service.inner.state.lock().unwrap().active == 0); + assert_eq!( + service.fetch(request(), &|| Ok(())).unwrap().source, + "LRCLIB" + ); + assert_eq!(calls.load(Ordering::Acquire), 2); +} + +#[test] +fn provider_priority_waits_within_grace_then_cancels_slow_workers() { + let calls = Arc::new(AtomicUsize::new(0)); + let cancelled = Arc::new(AtomicBool::new(false)); + let service = service({ + let calls = calls.clone(); + let cancelled = cancelled.clone(); + move |name, _, check| { + calls.fetch_add(1, Ordering::AcqRel); + if name == "lrclib" { + while check().is_ok() { + thread::sleep(Duration::from_millis(2)); + } + cancelled.store(true, Ordering::Release); + return Err(LyricsError::Cancelled("cancelled loser".into())); + } + Ok(lyrics("Extension:example.lyrics")) + } + }); + service + .set_providers(&["lrclib".into(), "extension:example.lyrics".into()]) + .unwrap(); + let start = Instant::now(); + assert_eq!( + service.fetch(request(), &|| Ok(())).unwrap().source, + "Extension:example.lyrics" + ); + assert!(start.elapsed() >= PRIORITY_GRACE); + assert!(start.elapsed() < PRIORITY_GRACE + Duration::from_secs(2)); + assert!(cancelled.load(Ordering::Acquire)); + assert_eq!(calls.load(Ordering::Acquire), 2); + + let preferred = service_for_preference(); + assert_eq!( + preferred.fetch(request(), &|| Ok(())).unwrap().source, + "LRCLIB" + ); +} + +fn service_for_preference() -> Arc<LyricsService> { + let service = service(|name, _, check| { + if name == "lrclib" { + thread::sleep(Duration::from_millis(60)); + } + check().map_err(LyricsError::Cancelled)?; + Ok(lyrics(if name == "lrclib" { + "LRCLIB" + } else { + "Extension:example.lyrics" + })) + }); + service + .set_providers(&["lrclib".into(), "extension:example.lyrics".into()]) + .unwrap(); + service +} + +#[test] +fn at_most_three_providers_run_and_shutdown_joins_cancelled_calls() { + let active = Arc::new(AtomicUsize::new(0)); + let entered = Arc::new(AtomicUsize::new(0)); + let service = service({ + let active = active.clone(); + let entered = entered.clone(); + move |_, _, check| { + active.fetch_add(1, Ordering::AcqRel); + entered.fetch_add(1, Ordering::AcqRel); + let result = loop { + if let Err(error) = check() { + break Err(LyricsError::Cancelled(error)); + } + thread::sleep(Duration::from_millis(2)); + }; + active.fetch_sub(1, Ordering::AcqRel); + result + } + }); + service + .set_providers(&[ + "lrclib".into(), + "apple_music".into(), + "netease".into(), + "qqmusic".into(), + "extension:example.lyrics".into(), + ]) + .unwrap(); + let worker = { + let service = service.clone(); + thread::spawn(move || service.fetch(request(), &|| Ok(()))) + }; + until(|| active.load(Ordering::Acquire) == 3); + service.shutdown().unwrap(); + assert_eq!(entered.load(Ordering::Acquire), 3); + assert_eq!(active.load(Ordering::Acquire), 0); + assert!(matches!( + worker.join().unwrap(), + Err(LyricsError::Cancelled(_)) + )); + assert!(matches!( + service.fetch(request(), &|| Ok(())), + Err(LyricsError::Cancelled(_)) + )); +} + +#[test] +fn unavailable_cooldown_negative_cache_expiry_and_reset_match_go() { + let calls = Arc::new(AtomicUsize::new(0)); + let service = service({ + let calls = calls.clone(); + move |_, _, _| { + calls.fetch_add(1, Ordering::AcqRel); + Err(LyricsError::Unavailable("HTTP 503".into())) + } + }); + assert!(service.fetch(request(), &|| Ok(())).is_err()); + assert_eq!( + service.fetch(request(), &|| Ok(())), + Err(LyricsError::NotFound("lyrics not found (cached)".into())) + ); + let mut second = request(); + second.track = "Second".into(); + assert!(service.fetch(second, &|| Ok(())).is_err()); + assert_eq!(calls.load(Ordering::Acquire), 1); + { + let mut state = service.inner.state.lock().unwrap(); + for expiry in state.negative.values_mut() { + *expiry = Instant::now(); + } + for expiry in state.health.values_mut() { + *expiry = Instant::now(); + } + } + assert!(service.fetch(request(), &|| Ok(())).is_err()); + assert_eq!(calls.load(Ordering::Acquire), 2); + service.set_providers(&["lrclib".into()]).unwrap(); + let mut third = request(); + third.track = "Third".into(); + assert!(service.fetch(third, &|| Ok(())).is_err()); + assert_eq!(calls.load(Ordering::Acquire), 3); + for index in 0..510 { + let mut request = request(); + request.track = format!("Missing {index}"); + assert!(service.fetch(request, &|| Ok(())).is_err()); + } + assert_eq!( + service.inner.state.lock().unwrap().negative.len(), + MAX_NEGATIVE + ); +} + +#[test] +fn config_changes_prevent_late_results_from_repopulating_current_cache() { + let gate = Arc::new(AtomicBool::new(false)); + let calls = Arc::new(AtomicUsize::new(0)); + let service = service({ + let gate = gate.clone(); + let calls = calls.clone(); + move |_, request, check| { + calls.fetch_add(1, Ordering::AcqRel); + if request.options.musixmatch_language.is_empty() { + wait_gate(&gate, check)?; + } + Ok(lyrics(&request.options.musixmatch_language)) + } + }); + let old = { + let service = service.clone(); + thread::spawn(move || service.fetch(request(), &|| Ok(()))) + }; + until(|| calls.load(Ordering::Acquire) == 1); + service + .set_options(FetchOptions { + musixmatch_language: " EN! ".into(), + ..FetchOptions::default() + }) + .unwrap(); + assert_eq!(service.fetch(request(), &|| Ok(())).unwrap().source, "en"); + gate.store(true, Ordering::Release); + assert_eq!(old.join().unwrap().unwrap().source, ""); + assert_eq!(service.cache_size(), 1); + assert_eq!( + service.fetch(request(), &|| Ok(())).unwrap().source, + "en (cached)" + ); + assert_eq!(calls.load(Ordering::Acquire), 2); +} + +#[test] +fn selected_extension_gets_a_chance_before_builtin_cache_and_has_cached_fallback() { + let mode = Arc::new(AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); + let service = service({ + let mode = mode.clone(); + let calls = calls.clone(); + move |name, _, _| { + calls.fetch_add(1, Ordering::AcqRel); + match (mode.load(Ordering::Acquire), name) { + (0, "lrclib") => Ok(lyrics("LRCLIB")), + (2, "extension:example.lyrics") => Ok(lyrics("Extension:example.lyrics")), + _ => Err(LyricsError::NotFound("missing".into())), + } + } + }); + service + .set_providers(&["extension:example.lyrics".into(), "lrclib".into()]) + .unwrap(); + assert_eq!( + service.fetch(request(), &|| Ok(())).unwrap().source, + "LRCLIB" + ); + mode.store(1, Ordering::Release); + assert_eq!( + service.fetch(request(), &|| Ok(())).unwrap().source, + "LRCLIB (cached fallback)" + ); + mode.store(2, Ordering::Release); + assert_eq!( + service.fetch(request(), &|| Ok(())).unwrap().source, + "Extension:example.lyrics" + ); + let previous = calls.load(Ordering::Acquire); + assert_eq!( + service.fetch(request(), &|| Ok(())).unwrap().source, + "Extension:example.lyrics (cached)" + ); + assert_eq!(calls.load(Ordering::Acquire), previous); +} + +#[test] +fn instrumental_heuristic_skips_network_and_provider_panics_release_workers() { + let service = service(|_, _, _| panic!("fixture provider failure")); + let mut instrumental = request(); + instrumental.track = "Song (Instrumental)".into(); + let response = service.fetch(instrumental, &|| Ok(())).unwrap(); + assert!(response.instrumental); + assert_eq!(response.source, "Heuristic: Instrumental"); + assert!(service.fetch(request(), &|| Ok(())).is_err()); + service.shutdown().unwrap(); + assert_eq!(service.inner.state.lock().unwrap().active, 0); +} + +#[test] +fn transport_policy_cancellation_is_not_cached_as_missing_lyrics() { + let calls = Arc::new(AtomicUsize::new(0)); + let service = service({ + let calls = calls.clone(); + move |_, _, _| { + if calls.fetch_add(1, Ordering::AcqRel) == 0 { + Err(LyricsError::Cancelled("network policy changed".into())) + } else { + Ok(lyrics("LRCLIB")) + } + } + }); + assert_eq!( + service.fetch(request(), &|| Ok(())), + Err(LyricsError::Cancelled("network policy changed".into())) + ); + assert_eq!( + service.fetch(request(), &|| Ok(())).unwrap().source, + "LRCLIB" + ); + assert_eq!(calls.load(Ordering::Acquire), 2); +} + +#[test] +fn shutdown_flushes_cache_and_rejects_retained_mutations() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("lyrics.json"); + let replacement = directory.path().join("replacement.json"); + let service = service(|_, _, _| Ok(lyrics("LRCLIB"))); + service.set_persistence_path(&path).unwrap(); + service.fetch(request(), &|| Ok(())).unwrap(); + service.shutdown().unwrap(); + let persisted = std::fs::read(&path).unwrap(); + let options = service.options(); + let providers = service.providers(); + let closed = LyricsError::Cancelled("lyrics service is closed".into()); + assert_eq!( + service.set_providers(&["apple_music".into()]), + Err(closed.clone()) + ); + assert_eq!( + service.set_options(FetchOptions { + musixmatch_language: "id".into(), + ..Default::default() + }), + Err(closed.clone()) + ); + assert_eq!( + service.set_persistence_path(&replacement), + Err(closed.clone()) + ); + assert_eq!(service.clear_cache(), Err(closed.clone())); + assert_eq!(service.drop_memory(), Err(closed)); + assert_eq!(service.options(), options); + assert_eq!(service.providers(), providers); + assert_eq!(service.cache_size(), 1); + service.shutdown().unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), persisted); + assert!(!replacement.exists()); +} diff --git a/rust_backend/crates/providers/src/musicbrainz.rs b/rust_backend/crates/providers/src/musicbrainz.rs new file mode 100644 index 00000000..f19bd523 --- /dev/null +++ b/rust_backend/crates/providers/src/musicbrainz.rs @@ -0,0 +1,333 @@ +//! Shared recording snapshots for native genre and album-artist enrichment. + +mod http; +#[cfg(test)] +mod tests; + +use crate::resolver::{Check, ResolverError}; +use spotiflac_core::matching::uppercase; +use spotiflac_core::metadata::musicbrainz::Response; +use spotiflac_network::NetworkService; +use std::collections::BTreeMap; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +const POLL: Duration = Duration::from_millis(25); +const MAX_CACHE: usize = 256; +const MAX_FLIGHTS: usize = 64; +type Outcome = Result<Arc<Response>, ResolverError>; + +/// Trusted native/test configuration. These options are not exposed to JS/FFI. +#[derive(Clone)] +pub struct MusicBrainzOptions { + pub endpoint: String, + pub http_timeout: Duration, + pub retry_delay: Duration, + pub positive_ttl: Duration, + pub negative_ttl: Duration, +} + +impl Default for MusicBrainzOptions { + fn default() -> Self { + Self { + endpoint: "https://musicbrainz.org".into(), + http_timeout: Duration::from_secs(10), + retry_delay: Duration::from_secs(2), + positive_ttl: Duration::from_secs(6 * 60 * 60), + negative_ttl: Duration::from_secs(10 * 60), + } + } +} + +trait Fetch: Send + Sync + 'static { + fn fetch(&self, isrc: &str, check: &Check<'_>) -> Outcome; +} + +struct Entry { + outcome: Outcome, + expires: Instant, +} + +#[derive(Default)] +struct Cache(BTreeMap<String, Entry>); + +impl Cache { + fn get(&self, key: &str, now: Instant) -> Option<Outcome> { + self.0 + .get(key) + .filter(|entry| now < entry.expires) + .map(|entry| entry.outcome.clone()) + } + + fn put(&mut self, key: &str, outcome: Outcome, now: Instant, options: &MusicBrainzOptions) { + if self.0.len() >= MAX_CACHE { + self.0.retain(|_, entry| now <= entry.expires); + if self.0.len() >= MAX_CACHE { + self.0.clear(); + } + } + let ttl = if outcome.is_ok() { + options.positive_ttl + } else { + options.negative_ttl + }; + self.0.insert( + key.into(), + Entry { + outcome, + expires: now + ttl, + }, + ); + } +} + +#[derive(Default)] +struct Flight { + waiters: AtomicUsize, + result: Mutex<Option<Outcome>>, + ready: Condvar, +} + +struct Waiter(Arc<Flight>); +impl Drop for Waiter { + fn drop(&mut self) { + self.0.waiters.fetch_sub(1, Ordering::AcqRel); + } +} + +#[derive(Default)] +struct State { + active: usize, + flights: BTreeMap<String, Arc<Flight>>, + workers: Vec<JoinHandle<()>>, + cache: Cache, +} + +struct Inner { + fetcher: Arc<dyn Fetch>, + options: MusicBrainzOptions, + closed: AtomicBool, + state: Mutex<State>, + idle: Condvar, + shutdown: Mutex<()>, +} + +pub struct MusicBrainzClient { + inner: Arc<Inner>, +} + +impl MusicBrainzClient { + pub fn with_options( + network: &Arc<NetworkService>, + options: MusicBrainzOptions, + ) -> Result<Self, ResolverError> { + let fetcher = Arc::new(http::Http::new(network, &options)?); + Ok(Self::with_fetcher(fetcher, options)) + } + + fn with_fetcher(fetcher: Arc<dyn Fetch>, options: MusicBrainzOptions) -> Self { + Self { + inner: Arc::new(Inner { + fetcher, + options, + closed: AtomicBool::new(false), + state: Mutex::new(State::default()), + idle: Condvar::new(), + shutdown: Mutex::new(()), + }), + } + } + + pub fn genre(&self, isrc: &str, check: &Check<'_>) -> Result<String, ResolverError> { + let (isrc, snapshot) = self.snapshot(isrc, check)?; + let result = snapshot.genre(&isrc).map_err(ResolverError::Failed); + self.inner.check(check)?; + result + } + + pub fn album_artist( + &self, + isrc: &str, + album_name: &str, + check: &Check<'_>, + ) -> Result<String, ResolverError> { + let (isrc, snapshot) = self.snapshot(isrc, check)?; + let result = snapshot + .album_artist(&isrc, album_name) + .map_err(ResolverError::Failed); + self.inner.check(check)?; + result + } + + fn snapshot( + &self, + isrc: &str, + check: &Check<'_>, + ) -> Result<(String, Arc<Response>), ResolverError> { + self.inner.check(check)?; + if isrc.len() > 64 << 10 { + return Err(ResolverError::Failed( + "MusicBrainz ISRC exceeds limit".into(), + )); + } + let key = uppercase(isrc.trim()); + let flight = { + let mut state = self.inner.state.lock().unwrap(); + self.inner.check(&|| Ok(()))?; + // Reap completed threads on cache hits too, keeping retained handles bounded. + let mut pending = Vec::new(); + for worker in state.workers.drain(..) { + if worker.is_finished() { + let _ = worker.join(); + } else { + pending.push(worker); + } + } + state.workers = pending; + if let Some(value) = state.cache.get(&key, Instant::now()) { + drop(state); + self.inner.check(check)?; + return value.map(|value| (key, value)); + } + if let Some(flight) = state + .flights + .get(&key) + .filter(|flight| { + flight + .waiters + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + (count > 0).then_some(count + 1) + }) + .is_ok() + }) + .cloned() + { + flight + } else { + if state.active >= MAX_FLIGHTS { + return Err(ResolverError::Failed( + "MusicBrainz lookup limit reached".into(), + )); + } + let flight = Arc::new(Flight::default()); + flight.waiters.store(1, Ordering::Release); + state.flights.insert(key.clone(), flight.clone()); + state.active += 1; + let inner = self.inner.clone(); + let worker_key = key.clone(); + let worker_flight = flight.clone(); + // Register the handle under the state lock so shutdown cannot miss it. + match std::thread::Builder::new() + .name("musicbrainz".into()) + .spawn(move || { + let check = || { + if inner.closed.load(Ordering::Acquire) { + Err("MusicBrainz client closed".into()) + } else if worker_flight.waiters.load(Ordering::Acquire) == 0 { + Err("MusicBrainz request cancelled".into()) + } else { + Ok(()) + } + }; + let result = catch_unwind(AssertUnwindSafe(|| { + if worker_key.is_empty() { + Err(ResolverError::Failed("no ISRC provided".into())) + } else { + inner.fetcher.fetch(&worker_key, &check) + } + })) + .unwrap_or_else(|_| { + Err(ResolverError::Failed("MusicBrainz request panicked".into())) + }); + inner.finish( + &worker_key, + &worker_flight, + check().map_err(ResolverError::Cancelled).and(result), + ); + }) { + Ok(worker) => state.workers.push(worker), + Err(error) => { + state.flights.remove(&key); + state.active -= 1; + *flight.result.lock().unwrap() = + Some(Err(ResolverError::Failed(error.to_string()))); + self.inner.idle.notify_all(); + } + } + flight + } + }; + let _waiter = Waiter(flight.clone()); + loop { + self.inner.check(check)?; + let result_guard = flight.result.lock().unwrap(); + if let Some(result) = result_guard.as_ref() { + let result = result.clone(); + drop(result_guard); + self.inner.check(check)?; + return result.map(|value| (key, value)); + } + drop(flight.ready.wait_timeout(result_guard, POLL).unwrap()); + } + } + + pub fn shutdown(&self) { + let _shutdown = self.inner.shutdown.lock().unwrap(); + self.inner.closed.store(true, Ordering::Release); + let mut state = self.inner.state.lock().unwrap(); + while state.active != 0 { + state = self.inner.idle.wait(state).unwrap(); + } + state.cache.0.clear(); + let workers = std::mem::take(&mut state.workers); + drop(state); + for worker in workers { + let _ = worker.join(); + } + } +} + +impl Drop for MusicBrainzClient { + fn drop(&mut self) { + self.shutdown(); + } +} + +impl Inner { + fn check(&self, check: &Check<'_>) -> Result<(), ResolverError> { + if self.closed.load(Ordering::Acquire) { + Err(ResolverError::Cancelled("MusicBrainz client closed".into())) + } else { + check().map_err(ResolverError::Cancelled) + } + } + + fn finish(&self, key: &str, flight: &Arc<Flight>, result: Outcome) { + let mut state = self.state.lock().unwrap(); + let current = state + .flights + .get(key) + .is_some_and(|current| Arc::ptr_eq(current, flight)); + if current { + state.flights.remove(key); + if !self.closed.load(Ordering::Acquire) + && flight.waiters.load(Ordering::Acquire) > 0 + && !matches!( + result, + Err(ResolverError::Cancelled(_) | ResolverError::Closed | ResolverError::Busy) + ) + { + state + .cache + .put(key, result.clone(), Instant::now(), &self.options); + } + } + *flight.result.lock().unwrap() = Some(result); + flight.ready.notify_all(); + state.active -= 1; + self.idle.notify_all(); + } +} diff --git a/rust_backend/crates/providers/src/musicbrainz/http.rs b/rust_backend/crates/providers/src/musicbrainz/http.rs new file mode 100644 index 00000000..1a435e31 --- /dev/null +++ b/rust_backend/crates/providers/src/musicbrainz/http.rs @@ -0,0 +1,200 @@ +use super::{Fetch, MusicBrainzOptions, Outcome, POLL}; +use crate::resolver::{Check, ResolverError, urls::query_escape}; +use spotiflac_core::lyrics::decode_response; +use spotiflac_network::{ + HttpRequest, HttpStream, NetworkService, NetworkSession, random_user_agent, url::UrlParts, +}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const MAX_BODY: usize = 16 << 20; + +pub(super) struct Http { + session: Arc<NetworkSession>, + endpoint: String, + timeout: Duration, + retry_delay: Duration, +} + +impl Http { + pub fn new( + network: &Arc<NetworkService>, + options: &MusicBrainzOptions, + ) -> Result<Self, ResolverError> { + let endpoint = UrlParts::parse(&options.endpoint) + .ok_or_else(|| ResolverError::Failed("invalid MusicBrainz origin".into()))?; + if endpoint.scheme != "https" + || endpoint.hostname.is_empty() + || endpoint.has_credentials + || !matches!(endpoint.raw_path.as_str(), "" | "/") + || !endpoint.raw_query.is_empty() + || !endpoint.fragment.is_empty() + { + return Err(ResolverError::Failed( + "MusicBrainz endpoint requires an HTTPS origin without credentials".into(), + )); + } + if options.http_timeout.is_zero() { + return Err(ResolverError::Failed( + "MusicBrainz HTTP timeout must be positive".into(), + )); + } + Ok(Self { + session: network.native_session(options.http_timeout), + endpoint: options.endpoint.trim_end_matches('/').into(), + timeout: options.http_timeout, + retry_delay: options.retry_delay, + }) + } + + fn wait_retry(&self, check: &Check<'_>) -> Result<(), ResolverError> { + let started = Instant::now(); + while started.elapsed() < self.retry_delay { + check().map_err(ResolverError::Cancelled)?; + std::thread::sleep(self.retry_delay.saturating_sub(started.elapsed()).min(POLL)); + } + check().map_err(ResolverError::Cancelled) + } +} + +fn failure(message: String, check: &Check<'_>) -> ResolverError { + if let Err(cancelled) = check() { + ResolverError::Cancelled(cancelled) + } else if message == "network policy changed" { + ResolverError::Cancelled(message) + } else { + ResolverError::Transport(message) + } +} + +impl Fetch for Http { + fn fetch(&self, isrc: &str, check: &Check<'_>) -> Outcome { + let url = format!( + "{}/ws/2/recording?query={}&fmt=json&inc=tags+releases+artist-credits", + self.endpoint, + query_escape(&format!("isrc:{isrc}")) + ); + // Go chooses a browser UA once per lookup and reuses it for retries. + let user_agent = random_user_agent(); + for attempt in 0..3 { + if attempt > 0 { + self.wait_retry(check)?; + } + check().map_err(ResolverError::Cancelled)?; + let stream = self.session.open_stream( + HttpRequest { + url: url.clone(), + method: "GET".into(), + body: String::new(), + headers: BTreeMap::new(), + default_json: false, + user_agent: user_agent.clone(), + }, + self.timeout, + self.timeout, + check, + ); + let mut stream = match stream { + Ok(stream) => stream, + Err(message) => { + let error = failure(message, check); + if matches!(error, ResolverError::Cancelled(_)) || attempt == 2 { + return Err(error); + } + continue; + } + }; + let status = stream.response.status; + if status != 200 { + // Failed statuses retry without waiting for their body to finish. + drop(stream); + if attempt == 2 { + return Err(ResolverError::Failed(format!( + "MusicBrainz API returned status: {status}" + ))); + } + continue; + } + // Decode/body failures occur after Client.Do in Go and are not retried. + let body = read_json(&mut stream, check)?; + let result = decode_response(&body) + .map(Arc::new) + .map_err(|error| ResolverError::Failed(error.to_string())); + check().map_err(ResolverError::Cancelled)?; + return result; + } + unreachable!("last MusicBrainz attempt returns its result") + } +} + +// Locate the first complete compound/string JSON value in one linear scan. +// Go's Decoder.Decode can return before EOF, including a trailing stalled body. +// Full syntax and field validation remain the typed decoder's responsibility. +#[derive(Default)] +struct Boundary { + started: bool, + compound: bool, + depth: usize, + quoted: bool, + escaped: bool, +} + +impl Boundary { + fn byte(&mut self, byte: u8) -> bool { + if !self.started { + if matches!(byte, b' ' | b'\t' | b'\r' | b'\n') { + return false; + } + self.started = true; + self.compound = matches!(byte, b'{' | b'[' | b'"'); + } + if self.quoted { + if self.escaped { + self.escaped = false; + } else if byte == b'\\' { + self.escaped = true; + } else if byte == b'"' { + self.quoted = false; + return self.depth == 0; + } + } else { + match byte { + b'"' => self.quoted = true, + b'{' | b'[' => self.depth += 1, + b'}' | b']' => { + self.depth = self.depth.saturating_sub(1); + return self.depth == 0; + } + b' ' | b'\t' | b'\r' | b'\n' if !self.compound => return true, + _ => {} + } + } + false + } +} + +fn read_json(stream: &mut HttpStream, check: &Check<'_>) -> Result<Vec<u8>, ResolverError> { + let mut boundary = Boundary::default(); + let mut body = Vec::new(); + let mut buffer = [0; 16 << 10]; + loop { + let count = stream + .read(&mut buffer, check) + .map_err(|message| failure(message, check))?; + if count == 0 { + return Ok(body); + } + let end = buffer[..count].iter().position(|byte| boundary.byte(*byte)); + let length = end.map_or(count, |index| index + 1); + if body.len() + length > MAX_BODY { + return Err(ResolverError::Failed( + "MusicBrainz response exceeds limit".into(), + )); + } + body.extend_from_slice(&buffer[..length]); + if end.is_some() { + return Ok(body); + } + } +} diff --git a/rust_backend/crates/providers/src/musicbrainz/tests.rs b/rust_backend/crates/providers/src/musicbrainz/tests.rs new file mode 100644 index 00000000..e0c84f9e --- /dev/null +++ b/rust_backend/crates/providers/src/musicbrainz/tests.rs @@ -0,0 +1,298 @@ +use super::*; +use spotiflac_core::metadata::musicbrainz::{ArtistCredit, Recording, Release, Tag}; +use std::thread; + +fn response() -> Arc<Response> { + Arc::new(Response { + recordings: Some(vec![Recording { + tags: Some(vec![Tag { + count: 1, + name: "rock".into(), + }]), + releases: Some(vec![Release { + title: "Album".into(), + artist_credit: Some(vec![ArtistCredit { + name: "Example Artist".into(), + joinphrase: String::new(), + }]), + }]), + }]), + }) +} + +fn until(ready: impl Fn() -> bool) { + let started = Instant::now(); + while !ready() { + assert!( + started.elapsed() < Duration::from_secs(3), + "fixture did not become ready" + ); + thread::sleep(Duration::from_millis(1)); + } +} + +#[derive(Default)] +struct Probe { + calls: AtomicUsize, + active: AtomicUsize, + cancelled: AtomicUsize, + release: AtomicBool, + hold_cancelled: AtomicBool, + failure: AtomicBool, +} + +struct Active<'a>(&'a AtomicUsize); +impl Drop for Active<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + +impl Fetch for Probe { + fn fetch(&self, isrc: &str, check: &Check<'_>) -> Outcome { + self.calls.fetch_add(1, Ordering::AcqRel); + self.active.fetch_add(1, Ordering::AcqRel); + let _active = Active(&self.active); + let started = Instant::now(); + while !self.release.load(Ordering::Acquire) { + if let Err(message) = check() { + self.cancelled.fetch_add(1, Ordering::AcqRel); + while self.hold_cancelled.load(Ordering::Acquire) + && started.elapsed() < Duration::from_secs(3) + { + thread::sleep(Duration::from_millis(1)); + } + return Err(ResolverError::Cancelled(message)); + } + if started.elapsed() >= Duration::from_secs(3) { + return Err(ResolverError::Failed("fixture timed out".into())); + } + thread::sleep(Duration::from_millis(1)); + } + if self.failure.load(Ordering::Acquire) { + Err(ResolverError::Transport("fixture transport failed".into())) + } else if isrc == "EMPTY" { + Ok(Arc::new(Response::default())) + } else { + Ok(response()) + } + } +} + +fn client(probe: &Arc<Probe>) -> Arc<MusicBrainzClient> { + Arc::new(MusicBrainzClient::with_fetcher( + probe.clone(), + MusicBrainzOptions::default(), + )) +} + +#[test] +fn cache_ttls_expiry_and_capacity_follow_go_snapshot_outcomes() { + let options = MusicBrainzOptions::default(); + let now = Instant::now(); + let mut cache = Cache::default(); + cache.put("empty", Ok(Arc::new(Response::default())), now, &options); + cache.put( + "failed", + Err(ResolverError::Failed("failed".into())), + now, + &options, + ); + assert!( + cache + .get("empty", now + Duration::from_secs(11 * 60)) + .unwrap() + .is_ok() + ); + assert!(cache.get("empty", now + options.positive_ttl).is_none()); + assert!( + cache + .get( + "failed", + now + options.negative_ttl - Duration::from_nanos(1) + ) + .unwrap() + .is_err() + ); + assert!(cache.get("failed", now + options.negative_ttl).is_none()); + cache.0.clear(); + for index in 0..MAX_CACHE { + cache.put(&index.to_string(), Ok(response()), now, &options); + } + cache.put("overflow", Ok(response()), now, &options); + assert_eq!( + cache.0.len(), + 1, + "Go resets a full cache when cleanup cannot reclaim entries" + ); + for index in 0..MAX_CACHE - 1 { + cache.put( + &index.to_string(), + Err(ResolverError::Failed("failed".into())), + now, + &options, + ); + } + cache.put( + "after-expiry", + Ok(response()), + now + options.negative_ttl + Duration::from_nanos(1), + &options, + ); + assert_eq!(cache.0.len(), 2); + assert!(cache.get("overflow", now).is_some()); +} + +#[test] +fn genre_and_album_artist_share_a_flight_without_cancelling_other_callers() { + let probe = Arc::new(Probe::default()); + let client = client(&probe); + let cancel = Arc::new(AtomicBool::new(false)); + let first = { + let (client, cancel) = (client.clone(), cancel.clone()); + thread::spawn(move || { + client.genre(" usaa00000101 ", &|| { + if cancel.load(Ordering::Acquire) { + Err("caller cancelled".into()) + } else { + Ok(()) + } + }) + }) + }; + until(|| probe.calls.load(Ordering::Acquire) == 1); + let others: Vec<_> = (0..8) + .map(|index| { + let client = client.clone(); + thread::spawn(move || { + if index % 2 == 0 { + client.genre("USAA00000101", &|| Ok(())) + } else { + client.album_artist("USAA00000101", "Album", &|| Ok(())) + } + }) + }) + .collect(); + until(|| { + client.inner.state.lock().unwrap().flights["USAA00000101"] + .waiters + .load(Ordering::Acquire) + == 9 + }); + cancel.store(true, Ordering::Release); + assert_eq!( + first.join().unwrap(), + Err(ResolverError::Cancelled("caller cancelled".into())) + ); + assert_eq!(probe.cancelled.load(Ordering::Acquire), 0); + assert_eq!(probe.calls.load(Ordering::Acquire), 1); + probe.release.store(true, Ordering::Release); + for (index, worker) in others.into_iter().enumerate() { + assert_eq!( + worker.join().unwrap().unwrap(), + if index % 2 == 0 { + "Rock" + } else { + "Example Artist" + } + ); + } + assert_eq!( + client + .album_artist("USAA00000101", "Other", &|| Ok(())) + .unwrap(), + "Example Artist" + ); + assert_eq!(probe.calls.load(Ordering::Acquire), 1); +} + +#[test] +fn abandoned_flight_cannot_publish_over_its_replacement() { + let probe = Arc::new(Probe::default()); + probe.hold_cancelled.store(true, Ordering::Release); + let client = client(&probe); + let cancel = Arc::new(AtomicBool::new(false)); + let first = { + let (client, cancel) = (client.clone(), cancel.clone()); + thread::spawn(move || { + client.genre("same", &|| { + if cancel.load(Ordering::Acquire) { + Err("cancelled".into()) + } else { + Ok(()) + } + }) + }) + }; + until(|| probe.calls.load(Ordering::Acquire) == 1); + cancel.store(true, Ordering::Release); + assert!(first.join().unwrap().is_err()); + until(|| probe.cancelled.load(Ordering::Acquire) == 1); + let second = { + let client = client.clone(); + thread::spawn(move || client.genre("same", &|| Ok(()))) + }; + until(|| probe.calls.load(Ordering::Acquire) == 2); + probe.release.store(true, Ordering::Release); + assert_eq!(second.join().unwrap().unwrap(), "Rock"); + probe.hold_cancelled.store(false, Ordering::Release); + until(|| client.inner.state.lock().unwrap().active == 0); + assert_eq!(client.genre("same", &|| Ok(())).unwrap(), "Rock"); + assert_eq!(probe.calls.load(Ordering::Acquire), 2); +} + +#[test] +fn fetch_failures_and_empty_recordings_are_cached_before_derived_getter_errors() { + let probe = Arc::new(Probe::default()); + probe.release.store(true, Ordering::Release); + probe.failure.store(true, Ordering::Release); + let client = client(&probe); + assert!(client.genre("failed", &|| Ok(())).is_err()); + probe.failure.store(false, Ordering::Release); + assert!(client.album_artist("FAILED", "Album", &|| Ok(())).is_err()); + assert_eq!(probe.calls.load(Ordering::Acquire), 1); + assert_eq!( + client.genre("empty", &|| Ok(())).unwrap_err().to_string(), + "no recordings found for ISRC: EMPTY" + ); + assert_eq!( + client + .album_artist("empty", "Album", &|| Ok(())) + .unwrap_err() + .to_string(), + "no MusicBrainz album artist found for ISRC: EMPTY" + ); + assert_eq!(probe.calls.load(Ordering::Acquire), 2); + assert_eq!( + client.genre("", &|| Ok(())).unwrap_err().to_string(), + "no ISRC provided" + ); + assert_eq!( + client + .album_artist(" \t ", "Album", &|| Ok(())) + .unwrap_err() + .to_string(), + "no ISRC provided" + ); + assert_eq!(probe.calls.load(Ordering::Acquire), 2); +} + +#[test] +fn shutdown_joins_workers_and_rejects_retained_cached_calls() { + let probe = Arc::new(Probe::default()); + let client = client(&probe); + let worker = { + let client = client.clone(); + thread::spawn(move || client.genre("active", &|| Ok(()))) + }; + until(|| probe.calls.load(Ordering::Acquire) == 1); + let started = Instant::now(); + client.shutdown(); + assert!(worker.join().unwrap().is_err()); + assert!(started.elapsed() < Duration::from_secs(2)); + assert_eq!(probe.active.load(Ordering::Acquire), 0); + assert!(client.inner.state.lock().unwrap().workers.is_empty()); + assert!(client.genre("active", &|| Ok(())).is_err()); + assert!(client.album_artist("", "Album", &|| Ok(())).is_err()); + client.shutdown(); +} diff --git a/rust_backend/crates/providers/src/resolver.rs b/rust_backend/crates/providers/src/resolver.rs new file mode 100644 index 00000000..00d3bd99 --- /dev/null +++ b/rust_backend/crates/providers/src/resolver.rs @@ -0,0 +1,68 @@ +//! Active platform resolver chain and cancellation-aware native ownership. + +pub mod availability; +mod client; +pub(crate) mod http; +mod rate; +mod service; +pub mod urls; + +pub use client::{PlatformResolverChain, ResolverOptions}; +use serde::Serialize; +pub use service::PlatformResolverService; +pub use spotiflac_core::resolver::Metadata; +use std::collections::BTreeMap; + +pub type Check<'a> = dyn Fn() -> Result<(), String> + Sync + 'a; + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ResolverError { + #[error("{0}")] + Failed(String), + #[error("{0}")] + Transport(String), + #[error("{0}")] + Cancelled(String), + #[error("platform resolver closed")] + Closed, + #[error("platform resolver lookup limit reached")] + Busy, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct Resolution { + pub links: BTreeMap<String, String>, + pub metadata: Metadata, +} + +pub trait Resolver: Send + Sync + 'static { + /// Implementations must observe `check` during blocking work. Shutdown + /// waits for calls to release their network streams and other resources. + fn resolve( + &self, + url: &str, + hint: &Metadata, + check: &Check<'_>, + ) -> Result<Resolution, ResolverError>; +} + +fn check_active(check: &Check<'_>) -> Result<(), ResolverError> { + check().map_err(ResolverError::Cancelled) +} + +fn add_source(links: &mut BTreeMap<String, String>, input: &str) { + let platform = urls::platform(input); + let direct = urls::direct(platform, input); + if !direct.is_empty() { + links.entry(platform.into()).or_insert(direct); + } +} + +fn useful(links: &BTreeMap<String, String>) -> bool { + links.len() >= 4 + && ["deezer", "tidal", "amazonMusic", "qobuz"] + .iter() + .filter(|platform| links.get(**platform).is_some_and(|link| !link.is_empty())) + .count() + >= 2 +} diff --git a/rust_backend/crates/providers/src/resolver/availability.rs b/rust_backend/crates/providers/src/resolver/availability.rs new file mode 100644 index 00000000..9b03e450 --- /dev/null +++ b/rust_backend/crates/providers/src/resolver/availability.rs @@ -0,0 +1,220 @@ +//! Application-facing track/album availability and platform-link contracts. + +mod cache; +use super::{Check, Metadata, PlatformResolverService, ResolverError, urls}; +pub use cache::AvailabilityService; +use serde::{Deserialize, Serialize}; +use spotiflac_core::metadata::TrackMetadata; +use spotiflac_network::{query, url::UrlParts}; +use std::collections::BTreeMap; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct TrackAvailability { + pub spotify_id: String, + pub tidal: bool, + pub amazon: bool, + pub qobuz: bool, + pub deezer: bool, + pub youtube: bool, + #[serde(skip_serializing_if = "String::is_empty")] + pub tidal_url: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub amazon_url: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub qobuz_url: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub deezer_url: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub youtube_url: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub deezer_id: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub qobuz_id: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub tidal_id: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub youtube_id: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct AlbumAvailability { + pub spotify_id: String, + pub deezer: bool, + #[serde(skip_serializing_if = "String::is_empty")] + pub deezer_url: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub deezer_id: String, +} + +pub fn from_links(spotify_id: &str, links: &BTreeMap<String, String>) -> TrackAvailability { + let link = |platform: &str| links.get(platform).cloned().unwrap_or_default(); + let mut result = TrackAvailability { + spotify_id: spotify_id.into(), + tidal_url: link("tidal"), + amazon_url: link("amazonMusic"), + qobuz_url: link("qobuz"), + deezer_url: link("deezer"), + youtube_url: link("youtubeMusic"), + ..TrackAvailability::default() + }; + if result.spotify_id.is_empty() { + result.spotify_id = spotify_id_from_url(&link("spotify")); + } + if result.youtube_url.is_empty() { + result.youtube_url = link("youtube"); + } + result.tidal = !result.tidal_url.is_empty(); + result.amazon = !result.amazon_url.is_empty(); + result.qobuz = !result.qobuz_url.is_empty(); + result.deezer = !result.deezer_url.is_empty(); + result.youtube = !result.youtube_url.is_empty(); + result.tidal_id = tidal_id_from_url(&result.tidal_url); + result.qobuz_id = qobuz_id_from_url(&result.qobuz_url); + result.deezer_id = deezer_id_from_url(&result.deezer_url); + result.youtube_id = youtube_id_from_url(&result.youtube_url); + result +} + +fn before(value: &str, marker: char) -> &str { + value + .find(marker) + .filter(|index| *index > 0) + .map_or(value, |index| &value[..index]) +} + +pub fn deezer_id_from_url(value: &str) -> String { + before(value.rsplit('/').next().unwrap_or_default(), '?').into() +} +pub fn spotify_id_from_url(value: &str) -> String { + before(value.split("/track/").nth(1).unwrap_or_default(), '?').into() +} +fn numeric(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) +} + +pub fn tidal_id_from_url(value: &str) -> String { + let value = before( + before(value.split("/track/").nth(1).unwrap_or_default(), '?'), + '/', + ) + .trim(); + if numeric(value) { + value.into() + } else { + String::new() + } +} + +pub fn qobuz_id_from_url(value: &str) -> String { + let track = tidal_id_from_url(value); + if !track.is_empty() { + return track; + } + if let Some(id) = value.split("trackId=").nth(1) { + let id = before(id, '&').trim(); + if numeric(id) { + return id.into(); + } + } + value + .rsplit('/') + .map(|part| before(part, '?').trim()) + .find(|part| numeric(part)) + .unwrap_or_default() + .into() +} + +pub fn youtube_id_from_url(value: &str) -> String { + if let Some(id) = value.split("youtu.be/").nth(1) { + return before(before(id, '?'), '&').trim().into(); + } + let Some(url) = UrlParts::parse(value) else { + return String::new(); + }; + if let Some(id) = query::parse(&url.raw_query) + .get(b"v".as_slice()) + .and_then(|values| values.first()) + && !id.is_empty() + { + return spotiflac_core::lyrics::text_from_bytes(id); + } + let path = spotiflac_core::lyrics::text_from_bytes(&url.path); + path.split("/embed/") + .nth(1) + .unwrap_or_default() + .split('/') + .next() + .unwrap_or_default() + .into() +} + +pub fn deezer_id_from_metadata(track: &TrackMetadata) -> String { + if let Some(id) = track.spotify_id.trim().strip_prefix("deezer:") + && !id.trim().is_empty() + { + return id.trim().into(); + } + deezer_id_from_url(track.external_urls.trim()) +} + +fn resolved_links( + resolver: &PlatformResolverService, + input: &str, + check: &Check<'_>, +) -> Result<BTreeMap<String, String>, ResolverError> { + resolver + .resolve_url(input, &Metadata::default(), check) + .map(|result| result.links) +} + +fn from_deezer( + resolver: &PlatformResolverService, + id: &str, + check: &Check<'_>, +) -> Result<TrackAvailability, ResolverError> { + if id.is_empty() { + return Err(ResolverError::Failed("deezer track ID is empty".into())); + } + let input = format!("https://www.deezer.com/track/{id}"); + let mut result = from_links("", &resolved_links(resolver, &input, check)?); + result.deezer = true; + result.deezer_id = id.into(); + if result.deezer_url.is_empty() { + result.deezer_url = input; + } + Ok(result) +} + +fn album( + resolver: &PlatformResolverService, + id: &str, + check: &Check<'_>, +) -> Result<AlbumAvailability, ResolverError> { + let links = resolved_links( + resolver, + &format!("https://open.spotify.com/album/{id}"), + check, + )?; + let url = links.get("deezer").cloned().unwrap_or_default(); + Ok(AlbumAvailability { + spotify_id: id.into(), + deezer: !url.is_empty(), + deezer_id: deezer_id_from_url(&url), + deezer_url: url, + }) +} + +fn by_platform( + resolver: &PlatformResolverService, + platform: &str, + kind: &str, + id: &str, + check: &Check<'_>, +) -> Result<TrackAvailability, ResolverError> { + if id.is_empty() { + return Err(ResolverError::Failed(format!("{platform} ID is empty"))); + } + let input = urls::from_id(platform, kind, id).map_err(ResolverError::Failed)?; + Ok(from_links("", &resolved_links(resolver, &input, check)?)) +} diff --git a/rust_backend/crates/providers/src/resolver/availability/cache.rs b/rust_backend/crates/providers/src/resolver/availability/cache.rs new file mode 100644 index 00000000..5ac3de11 --- /dev/null +++ b/rust_backend/crates/providers/src/resolver/availability/cache.rs @@ -0,0 +1,849 @@ +use super::{ + AlbumAvailability, TrackAvailability, album, by_platform, deezer_id_from_metadata, from_deezer, + from_links, resolved_links, +}; +use crate::deezer::MetadataLookup; +use crate::lyrics::{LyricsError, builtin::TrackResolver}; +use crate::resolver::{Check, PlatformResolverService, Resolver, ResolverError}; +use std::collections::BTreeMap; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Kind { + Availability, + Links, +} + +#[derive(Clone)] +enum Value { + Availability(Box<TrackAvailability>), + Links(BTreeMap<String, String>), +} +type Outcome = Result<Value, ResolverError>; + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +struct Key { + kind: Kind, + region: String, + id: String, + isrc: bool, +} + +struct Entry { + result: Outcome, + expires: Instant, +} +#[derive(Default)] +struct Flight { + waiters: AtomicUsize, + result: Mutex<Option<Outcome>>, + ready: Condvar, +} +struct Waiter(Arc<Flight>); +impl Drop for Waiter { + fn drop(&mut self) { + self.0.waiters.fetch_sub(1, Ordering::AcqRel); + } +} + +struct State { + generation: u64, + region: String, + cache: BTreeMap<Key, Entry>, + flights: BTreeMap<(u64, Key), Arc<Flight>>, + active: usize, +} + +struct Inner { + resolver: PlatformResolverService, + metadata: Arc<dyn MetadataLookup>, + state: Mutex<State>, + closed: AtomicBool, + idle: Condvar, +} + +/// One native owner for raw resolver work and both application caches. +pub struct AvailabilityService { + inner: Arc<Inner>, +} + +impl AvailabilityService { + pub fn new(resolver: Arc<dyn Resolver>, metadata: Arc<dyn MetadataLookup>) -> Self { + Self { + inner: Arc::new(Inner { + resolver: PlatformResolverService::new(resolver), + metadata, + state: Mutex::new(State { + generation: 0, + region: "US".into(), + cache: BTreeMap::new(), + flights: BTreeMap::new(), + active: 0, + }), + closed: AtomicBool::new(false), + idle: Condvar::new(), + }), + } + } + + pub fn set_region(&self, region: &str) -> Result<(), ResolverError> { + let mut state = self.inner.state.lock().unwrap(); + self.inner.check(&|| Ok(()))?; + let region = spotiflac_core::matching::uppercase(region.trim()); + state.region = if region.len() == 2 && region.bytes().all(|byte| byte.is_ascii_uppercase()) + { + region + } else { + "US".into() + }; + Ok(()) + } + + pub fn region(&self) -> Result<String, ResolverError> { + self.inner.check(&|| Ok(()))?; + Ok(self.inner.state.lock().unwrap().region.clone()) + } + + pub fn clear_cache(&self) -> Result<usize, ResolverError> { + let mut state = self.inner.state.lock().unwrap(); + self.inner.check(&|| Ok(()))?; + state.generation = state.generation.wrapping_add(1); + let count = state.cache.len(); + state.cache.clear(); + Ok(count) + } + + pub fn check_track( + &self, + spotify_id: &str, + isrc: &str, + check: &Check<'_>, + ) -> Result<TrackAvailability, ResolverError> { + match self.lookup(Kind::Availability, spotify_id, isrc, check)? { + Value::Availability(value) => Ok(*value), + _ => unreachable!("availability cache type"), + } + } + + pub fn track_platform_links( + &self, + spotify_id: &str, + isrc: &str, + check: &Check<'_>, + ) -> Result<BTreeMap<String, String>, ResolverError> { + match self.lookup(Kind::Links, spotify_id, isrc, check)? { + Value::Links(value) => Ok(value), + _ => unreachable!("links cache type"), + } + } + + pub fn check_album( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<AlbumAvailability, ResolverError> { + self.with_raw(check, |resolver, check| album(resolver, id, check)) + } + + pub fn check_from_deezer( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<TrackAvailability, ResolverError> { + self.with_raw(check, |resolver, check| from_deezer(resolver, id, check)) + } + + pub fn check_by_platform( + &self, + platform: &str, + kind: &str, + id: &str, + check: &Check<'_>, + ) -> Result<TrackAvailability, ResolverError> { + self.with_raw(check, |resolver, check| { + by_platform(resolver, platform, kind, id, check) + }) + } + + pub fn check_from_url( + &self, + url: &str, + check: &Check<'_>, + ) -> Result<TrackAvailability, ResolverError> { + self.with_raw(check, |resolver, check| { + Ok(from_links("", &resolved_links(resolver, url, check)?)) + }) + } + + pub fn streaming_urls( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<BTreeMap<String, String>, ResolverError> { + let availability = self.check_track(id, "", check)?; + Ok([ + ("tidal", availability.tidal_url), + ("amazon", availability.amazon_url), + ] + .into_iter() + .filter(|(_, value)| !value.is_empty()) + .map(|(key, value)| (key.into(), value)) + .collect()) + } + + pub fn deezer_album_id(&self, id: &str, check: &Check<'_>) -> Result<String, ResolverError> { + let album = self.check_album(id, check)?; + if album.deezer && !album.deezer_id.is_empty() { + Ok(album.deezer_id) + } else { + Err(ResolverError::Failed("album not found on Deezer".into())) + } + } + + pub fn youtube_url_from_spotify( + &self, + id: &str, + check: &Check<'_>, + ) -> Result<String, ResolverError> { + let track = self.check_track(id, "", check)?; + if track.youtube && !track.youtube_url.is_empty() { + Ok(track.youtube_url) + } else { + Err(ResolverError::Failed("track not found on YouTube".into())) + } + } + + pub fn platform_from_deezer( + &self, + id: &str, + platform: &str, + check: &Check<'_>, + ) -> Result<String, ResolverError> { + let track = self.check_from_deezer(id, check)?; + let (value, name) = match platform { + "spotify" => (track.spotify_id, "Spotify"), + "tidal" => (track.tidal_url, "Tidal"), + "amazon" => (track.amazon_url, "Amazon Music"), + "youtube" => (track.youtube_url, "YouTube"), + _ => { + return Err(ResolverError::Failed( + "unsupported availability platform".into(), + )); + } + }; + if value.is_empty() { + Err(ResolverError::Failed(format!("track not found on {name}"))) + } else { + Ok(value) + } + } + + fn with_raw<T>( + &self, + check: &Check<'_>, + work: impl FnOnce(&PlatformResolverService, &Check<'_>) -> Result<T, ResolverError>, + ) -> Result<T, ResolverError> { + self.inner.check(check)?; + let result = work(&self.inner.resolver, &|| { + self.inner.check(check).map_err(|error| error.to_string()) + }); + self.inner.check(check)?; + result + } + + fn lookup(&self, kind: Kind, spotify_id: &str, isrc: &str, check: &Check<'_>) -> Outcome { + self.inner.check(check)?; + let spotify_id = spotify_id.trim(); + let isrc = spotiflac_core::matching::uppercase(isrc.trim()); + let (id, isrc) = if !spotify_id.is_empty() { + (spotify_id.to_owned(), false) + } else if !isrc.is_empty() { + (isrc, true) + } else { + return Err(ResolverError::Failed( + "spotify track ID and ISRC are empty".into(), + )); + }; + if id.len() > 64 << 10 { + return Err(ResolverError::Failed( + "availability input exceeds limit".into(), + )); + } + let (key, generation, flight, start) = { + let mut state = self.inner.state.lock().unwrap(); + self.inner.check(check)?; + let key = Key { + kind, + region: state.region.clone(), + id, + isrc, + }; + if let Some(entry) = state.cache.get(&key) + && Instant::now() <= entry.expires + { + return entry.result.clone().map_err(|_| { + ResolverError::Failed( + if kind == Kind::Availability { + "track availability unavailable (cached)" + } else { + "track platform links unavailable (cached)" + } + .into(), + ) + }); + } + state.cache.remove(&key); + let generation = state.generation; + let flight_key = (generation, key.clone()); + let existing = state + .flights + .get(&flight_key) + .filter(|flight| flight.waiters.load(Ordering::Acquire) > 0) + .cloned(); + let (flight, start) = if let Some(flight) = existing { + (flight, false) + } else { + if state.active >= 64 { + return Err(ResolverError::Busy); + } + let flight = Arc::new(Flight::default()); + state.flights.insert(flight_key, flight.clone()); + state.active += 1; + (flight, true) + }; + flight.waiters.fetch_add(1, Ordering::AcqRel); + (key, generation, flight, start) + }; + let _waiter = Waiter(flight.clone()); + if start { + let inner = self.inner.clone(); + let worker_key = key.clone(); + let worker_flight = flight.clone(); + let spawned = std::thread::Builder::new() + .name("platform-availability".into()) + .spawn(move || { + let check = || { + if inner.closed.load(Ordering::Acquire) { + Err("availability service closed".into()) + } else if worker_flight.waiters.load(Ordering::Acquire) == 0 { + Err("availability request cancelled".into()) + } else { + Ok(()) + } + }; + let result = + catch_unwind(AssertUnwindSafe(|| inner.fetch(&worker_key, &check))) + .unwrap_or_else(|_| { + Err(ResolverError::Failed("availability worker panicked".into())) + }); + let result = check().map_err(ResolverError::Cancelled).and(result); + inner.finish(generation, &worker_key, &worker_flight, result); + }); + if let Err(error) = spawned { + self.inner.finish( + generation, + &key, + &flight, + Err(ResolverError::Failed(error.to_string())), + ); + } + } + loop { + self.inner.check(check)?; + let result = flight.result.lock().unwrap(); + if let Some(value) = result.as_ref() { + let value = value.clone(); + drop(result); + self.inner.check(check)?; + return value; + } + drop( + flight + .ready + .wait_timeout(result, Duration::from_millis(25)) + .unwrap(), + ); + } + } + + pub fn shutdown(&self) { + self.inner.closed.store(true, Ordering::Release); + let mut state = self.inner.state.lock().unwrap(); + while state.active != 0 { + state = self.inner.idle.wait(state).unwrap(); + } + state.cache.clear(); + drop(state); + self.inner.resolver.shutdown(); + } +} + +impl Drop for AvailabilityService { + fn drop(&mut self) { + self.shutdown(); + } +} + +impl TrackResolver for AvailabilityService { + fn deezer_id_from_spotify(&self, id: &str, check: &Check<'_>) -> Result<String, LyricsError> { + let track = self + .check_track(id, "", check) + .map_err(|error| match error { + ResolverError::Cancelled(message) => LyricsError::Cancelled(message), + ResolverError::Closed => LyricsError::Cancelled(error.to_string()), + ResolverError::Transport(message) => LyricsError::Unavailable(message), + _ => LyricsError::Other(error.to_string()), + })?; + if track.deezer && !track.deezer_id.is_empty() { + Ok(track.deezer_id) + } else { + Err(LyricsError::Other("track not found on Deezer".into())) + } + } +} + +impl Inner { + fn check(&self, check: &Check<'_>) -> Result<(), ResolverError> { + if self.closed.load(Ordering::Acquire) { + Err(ResolverError::Closed) + } else { + check().map_err(ResolverError::Cancelled) + } + } + + fn fetch(&self, key: &Key, check: &Check<'_>) -> Outcome { + let id = if key.isrc { + let started = Instant::now(); + let track = self.metadata.search_by_isrc(&key.id, &|| { + check()?; + if started.elapsed() >= Duration::from_secs(30) { + Err("ISRC lookup deadline exceeded".into()) + } else { + Ok(()) + } + })?; + let id = deezer_id_from_metadata(&track); + if id.is_empty() { + return Err(ResolverError::Failed(format!( + "failed to resolve Deezer track ID from ISRC {}", + key.id + ))); + } + id + } else { + key.id.clone() + }; + if key.kind == Kind::Availability && key.isrc { + return from_deezer(&self.resolver, &id, check) + .map(|value| Value::Availability(Box::new(value))); + } + let url = if key.isrc { + format!("https://www.deezer.com/track/{id}") + } else { + format!("https://open.spotify.com/track/{id}") + }; + let links = resolved_links(&self.resolver, &url, check)?; + if key.kind == Kind::Availability { + return Ok(Value::Availability(Box::new(from_links(&id, &links)))); + } + let links: BTreeMap<_, _> = links + .into_iter() + .filter_map(|(key, value)| { + let value = value.trim(); + (value.starts_with("https://") || value.starts_with("http://")) + .then(|| (key, value.into())) + }) + .collect(); + if links.is_empty() { + Err(ResolverError::Failed("no platform links found".into())) + } else { + Ok(Value::Links(links)) + } + } + + fn finish(&self, generation: u64, key: &Key, flight: &Arc<Flight>, result: Outcome) { + let mut state = self.state.lock().unwrap(); + if generation == state.generation + && !self.closed.load(Ordering::Acquire) + && !matches!( + result, + Err(ResolverError::Cancelled(_) | ResolverError::Closed | ResolverError::Busy) + ) + { + let limit = if key.kind == Kind::Availability { + 500 + } else { + 200 + }; + if !state.cache.contains_key(key) + && state + .cache + .keys() + .filter(|entry| entry.kind == key.kind) + .count() + >= limit + && let Some(oldest) = state + .cache + .iter() + .filter(|(entry, _)| entry.kind == key.kind) + .min_by_key(|(_, value)| value.expires) + .map(|(key, _)| key.clone()) + { + state.cache.remove(&oldest); + } + let ttl = Duration::from_secs(if result.is_ok() { 30 * 60 } else { 5 * 60 }); + state.cache.insert( + key.clone(), + Entry { + result: result.clone(), + expires: Instant::now() + ttl, + }, + ); + } + *flight.result.lock().unwrap() = Some(result); + flight.ready.notify_all(); + let flight_key = (generation, key.clone()); + if state + .flights + .get(&flight_key) + .is_some_and(|value| Arc::ptr_eq(value, flight)) + { + state.flights.remove(&flight_key); + } + state.active -= 1; + self.idle.notify_all(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::resolver::{Metadata, Resolution}; + use spotiflac_core::metadata::TrackMetadata; + + #[derive(Default)] + struct Probe { + calls: AtomicUsize, + metadata_calls: AtomicUsize, + release: AtomicBool, + metadata_blocked: AtomicBool, + failed: AtomicBool, + } + + impl Resolver for Probe { + fn resolve( + &self, + _: &str, + _: &Metadata, + check: &Check<'_>, + ) -> Result<Resolution, ResolverError> { + self.calls.fetch_add(1, Ordering::AcqRel); + while !self.release.load(Ordering::Acquire) { + check().map_err(ResolverError::Cancelled)?; + std::thread::sleep(Duration::from_millis(2)); + } + if self.failed.load(Ordering::Acquire) { + return Err(ResolverError::Failed("fixture failure".into())); + } + Ok(Resolution { + links: BTreeMap::from([( + "deezer".into(), + "https://www.deezer.com/track/101".into(), + )]), + ..Resolution::default() + }) + } + } + + impl MetadataLookup for Probe { + fn search_by_isrc( + &self, + _: &str, + check: &Check<'_>, + ) -> Result<TrackMetadata, ResolverError> { + self.metadata_calls.fetch_add(1, Ordering::AcqRel); + while self.metadata_blocked.load(Ordering::Acquire) { + check().map_err(ResolverError::Cancelled)?; + std::thread::sleep(Duration::from_millis(2)); + } + Ok(TrackMetadata { + spotify_id: "deezer:42".into(), + ..TrackMetadata::default() + }) + } + } + + fn until(predicate: impl Fn() -> bool) { + let started = Instant::now(); + while !predicate() { + assert!( + started.elapsed() < Duration::from_secs(3), + "availability test timed out" + ); + std::thread::sleep(Duration::from_millis(2)); + } + } + + #[test] + fn shared_requests_cancel_independently_and_return_owned_cached_responses() { + let probe = Arc::new(Probe::default()); + let service = Arc::new(AvailabilityService::new(probe.clone(), probe.clone())); + let cancelled = Arc::new(AtomicBool::new(false)); + let first = { + let service = service.clone(); + let cancelled = cancelled.clone(); + std::thread::spawn(move || { + service.check_track("example", "", &|| { + if cancelled.load(Ordering::Acquire) { + Err("first cancelled".into()) + } else { + Ok(()) + } + }) + }) + }; + until(|| probe.calls.load(Ordering::Acquire) == 1); + let second = { + let service = service.clone(); + std::thread::spawn(move || service.check_track("example", "", &|| Ok(()))) + }; + until(|| { + service + .inner + .state + .lock() + .unwrap() + .flights + .values() + .next() + .unwrap() + .waiters + .load(Ordering::Acquire) + == 2 + }); + cancelled.store(true, Ordering::Release); + assert_eq!( + first.join().unwrap(), + Err(ResolverError::Cancelled("first cancelled".into())) + ); + probe.release.store(true, Ordering::Release); + let mut track = second.join().unwrap().unwrap(); + track.deezer_id = "caller changed".into(); + assert_eq!( + service + .check_track("example", "", &|| Ok(())) + .unwrap() + .deezer_id, + "101" + ); + assert_eq!(probe.calls.load(Ordering::Acquire), 1); + let mut links = service + .track_platform_links("example", "", &|| Ok(())) + .unwrap(); + links.insert("deezer".into(), "changed".into()); + assert_ne!( + service + .track_platform_links("example", "", &|| Ok(())) + .unwrap(), + links + ); + assert_eq!(probe.calls.load(Ordering::Acquire), 2); + assert_eq!( + service + .check_track("", "usaa00000001", &|| Ok(())) + .unwrap() + .deezer_id, + "42" + ); + service + .track_platform_links("", "USAA00000001", &|| Ok(())) + .unwrap(); + service + .check_track("", " USAA00000001 ", &|| Ok(())) + .unwrap(); + assert_eq!(probe.metadata_calls.load(Ordering::Acquire), 2); + } + + #[test] + fn shutdown_cancels_isrc_and_uncached_calls_and_rejects_retained_apis() { + let probe = Arc::new(Probe::default()); + probe.metadata_blocked.store(true, Ordering::Release); + let service = Arc::new(AvailabilityService::new(probe.clone(), probe.clone())); + let pending = { + let service = service.clone(); + std::thread::spawn(move || service.check_track("", "USAA00000001", &|| Ok(()))) + }; + until(|| probe.metadata_calls.load(Ordering::Acquire) == 1); + let raw = { + let service = service.clone(); + std::thread::spawn(move || service.check_album("example", &|| Ok(()))) + }; + until(|| probe.calls.load(Ordering::Acquire) == 1); + service.shutdown(); + assert_eq!(pending.join().unwrap(), Err(ResolverError::Closed)); + assert_eq!(raw.join().unwrap(), Err(ResolverError::Closed)); + assert_eq!( + service.check_track("", "", &|| Ok(())), + Err(ResolverError::Closed) + ); + assert_eq!( + service.track_platform_links("", "", &|| Ok(())), + Err(ResolverError::Closed) + ); + assert_eq!( + service.check_from_deezer("", &|| Ok(())), + Err(ResolverError::Closed) + ); + assert_eq!(service.set_region("ID"), Err(ResolverError::Closed)); + assert_eq!(service.region(), Err(ResolverError::Closed)); + assert_eq!(service.clear_cache(), Err(ResolverError::Closed)); + assert_eq!(service.inner.state.lock().unwrap().active, 0); + } + + #[test] + fn cache_clear_and_expiry_keep_stale_and_cancelled_results_out() { + let probe = Arc::new(Probe::default()); + let service = Arc::new(AvailabilityService::new(probe.clone(), probe.clone())); + let pending = { + let service = service.clone(); + std::thread::spawn(move || service.check_track("old", "", &|| Ok(()))) + }; + until(|| probe.calls.load(Ordering::Acquire) == 1); + service.clear_cache().unwrap(); + probe.release.store(true, Ordering::Release); + pending.join().unwrap().unwrap(); + assert!(service.inner.state.lock().unwrap().cache.is_empty()); + service.check_track("old", "", &|| Ok(())).unwrap(); + probe.failed.store(true, Ordering::Release); + assert!( + service + .track_platform_links("failed", "", &|| Ok(())) + .is_err() + ); + assert!( + service + .track_platform_links("failed", "", &|| Ok(())) + .is_err() + ); + assert_eq!(probe.calls.load(Ordering::Acquire), 3); + { + let mut state = service.inner.state.lock().unwrap(); + let positive = state + .cache + .values() + .find(|entry| entry.result.is_ok()) + .unwrap(); + let negative = state + .cache + .values() + .find(|entry| entry.result.is_err()) + .unwrap(); + assert!( + positive.expires.duration_since(negative.expires) > Duration::from_secs(24 * 60) + ); + for entry in state.cache.values_mut() { + entry.expires = Instant::now() - Duration::from_secs(1); + } + } + probe.failed.store(false, Ordering::Release); + service + .track_platform_links("failed", "", &|| Ok(())) + .unwrap(); + assert_eq!(probe.calls.load(Ordering::Acquire), 4); + probe.release.store(false, Ordering::Release); + let started = Instant::now(); + assert!(matches!( + service.check_track( + "cancel", + "", + &|| if started.elapsed() >= Duration::from_millis(40) { + Err("cancel".into()) + } else { + Ok(()) + } + ), + Err(ResolverError::Cancelled(_)) + )); + until(|| service.inner.state.lock().unwrap().active == 0); + assert!( + !service + .inner + .state + .lock() + .unwrap() + .cache + .keys() + .any(|key| key.id == "cancel") + ); + } + + #[test] + fn independent_cache_limits_region_keys_and_lookup_limit_are_enforced() { + let probe = Arc::new(Probe::default()); + let service = Arc::new(AvailabilityService::new(probe.clone(), probe.clone())); + let mut waiters = Vec::new(); + for index in 0..64 { + let service = service.clone(); + waiters.push(std::thread::spawn(move || { + service.check_track(&index.to_string(), "", &|| Ok(())) + })); + } + until(|| service.inner.state.lock().unwrap().active == 64); + assert_eq!( + service.check_track("over-limit", "", &|| Ok(())), + Err(ResolverError::Busy) + ); + probe.release.store(true, Ordering::Release); + for waiter in waiters { + waiter.join().unwrap().unwrap(); + } + service.clear_cache().unwrap(); + for index in 0..503 { + service + .check_track(&index.to_string(), "", &|| Ok(())) + .unwrap(); + } + for index in 0..203 { + service + .track_platform_links(&index.to_string(), "", &|| Ok(())) + .unwrap(); + } + { + let state = service.inner.state.lock().unwrap(); + assert_eq!( + state + .cache + .keys() + .filter(|key| key.kind == Kind::Availability) + .count(), + 500 + ); + assert_eq!( + state + .cache + .keys() + .filter(|key| key.kind == Kind::Links) + .count(), + 200 + ); + assert!(!state.cache.keys().any(|key| key.id == "0")); + } + service.set_region("id").unwrap(); + service.check_track("502", "", &|| Ok(())).unwrap(); + service.set_region("bad region").unwrap(); + assert_eq!(service.region().unwrap(), "US"); + assert!( + service + .inner + .state + .lock() + .unwrap() + .cache + .keys() + .any(|key| key.region == "ID") + ); + } +} diff --git a/rust_backend/crates/providers/src/resolver/client.rs b/rust_backend/crates/providers/src/resolver/client.rs new file mode 100644 index 00000000..26a4d3df --- /dev/null +++ b/rust_backend/crates/providers/src/resolver/client.rs @@ -0,0 +1,579 @@ +use super::{ + Check, Metadata, Resolution, Resolver, ResolverError, add_source, check_active, urls, useful, +}; +use super::{http::ResolverHttp, rate::RateLimiter}; +use html5ever::tendril::TendrilSink; +use scraper::{Html, HtmlTreeSink}; +use serde::de::DeserializeOwned; +use spotiflac_core::{lyrics, resolver as models}; +use spotiflac_network::{NetworkService, url::UrlParts}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const CHAIN_DEADLINE: &str = "platform resolver deadline exceeded"; + +pub struct ResolverOptions { + pub endpoints: BTreeMap<String, String>, + pub request_timeout: Duration, + pub chain_timeout: Duration, + /// Song.link Web, UniTune, MusicBrainz, Squigly, respectively. + pub rate_limits: [(usize, Duration); 4], +} + +impl Default for ResolverOptions { + fn default() -> Self { + Self { + endpoints: BTreeMap::new(), + request_timeout: Duration::from_secs(6), + chain_timeout: Duration::from_secs(12), + rate_limits: [ + (20, Duration::from_secs(60)), + (30, Duration::from_secs(60)), + (1, Duration::from_secs(1)), + (18, Duration::from_secs(60)), + ], + } + } +} + +pub struct PlatformResolverChain { + http: ResolverHttp, + rates: [RateLimiter; 4], + timeout: Duration, +} + +impl PlatformResolverChain { + pub fn new(network: &Arc<NetworkService>) -> Self { + Self::with_options(network, ResolverOptions::default()).expect("default resolver options") + } + + /// Configuration belongs to a native owner, never to JavaScript input. + pub fn with_options( + network: &Arc<NetworkService>, + options: ResolverOptions, + ) -> Result<Self, ResolverError> { + if options.request_timeout.is_zero() + || options.chain_timeout.is_zero() + || options + .rate_limits + .iter() + .any(|(max, window)| *max == 0 || window.is_zero()) + { + return Err(ResolverError::Failed( + "resolver limits must be positive".into(), + )); + } + Ok(Self { + http: ResolverHttp::new(network, options.endpoints, options.request_timeout)?, + rates: options + .rate_limits + .map(|(max, window)| RateLimiter::new(max, window)), + timeout: options.chain_timeout, + }) + } + + pub fn resolve_one( + &self, + provider: &str, + input: &str, + hint: &Metadata, + check: &Check<'_>, + ) -> Result<Resolution, ResolverError> { + check_active(check)?; + if input.len() > 64 << 10 || hint.title.len() + hint.artist.len() > 64 << 10 { + return Err(ResolverError::Failed("resolver input exceeds limit".into())); + } + match provider { + "songlink" => self.songlink(input, check), + "unitune" => self.unitune(input, check), + "musicbrainz" => self.musicbrainz(hint, check), + "squigly" => self.squigly(input, check), + _ => Err(ResolverError::Failed("unknown resolver".into())), + } + } + + fn songlink(&self, input: &str, check: &Check<'_>) -> Result<Resolution, ResolverError> { + if urls::direct(urls::platform(input), input).is_empty() { + return Err(ResolverError::Failed("unsupported source URL".into())); + } + self.rates[0].wait(check)?; + let response = self.http.request( + &format!("https://song.link/{}", urls::path_escape(input)), + None, + true, + check, + )?; + let hostname = UrlParts::parse(&response.url) + .map(|url| url.hostname.to_ascii_lowercase()) + .unwrap_or_default(); + if !matches!( + hostname.as_str(), + "song.link" | "album.link" | "artist.link" | "odesli.co" | "www.odesli.co" + ) { + return Err(ResolverError::Failed( + "web page redirected to an unexpected host".into(), + )); + } + songlink_html(&response.body, input, check) + } + + fn unitune(&self, input: &str, check: &Check<'_>) -> Result<Resolution, ResolverError> { + self.rates[1].wait(check)?; + let body = self + .http + .request( + &format!( + "https://api.unitune.art/v1-alpha.1/links?url={}", + urls::query_escape(input) + ), + None, + false, + check, + )? + .body; + let payload: models::UniTune = decode(&body, false, check)?; + let mut result = Resolution::default(); + for (platform, link) in payload.links.0 { + check_active(check)?; + let canonical = urls::canonical(&platform); + let direct = urls::direct(canonical, &link.url); + if !direct.is_empty() { + result.links.insert(canonical.into(), direct); + } + } + // Go's unspecified map iteration can choose any entity if its ID is + // missing. Rust uses the first sorted key for deterministic fallback. + if let Some(entity) = payload + .entities + .0 + .get(&payload.entity_id) + .or_else(|| payload.entities.0.values().next()) + { + result.metadata = Metadata { + title: entity.title.clone(), + artist: entity.artist.clone(), + }; + } + if result.links.is_empty() && result.metadata.title.is_empty() { + return Err(ResolverError::Failed( + "API returned no direct platform links or metadata".into(), + )); + } + Ok(result) + } + + fn musicbrainz(&self, hint: &Metadata, check: &Check<'_>) -> Result<Resolution, ResolverError> { + let (title, artist) = (hint.title.trim(), hint.artist.trim()); + if title.is_empty() || artist.is_empty() { + return Err(ResolverError::Failed( + "title and artist metadata are required".into(), + )); + } + let escape = |value: &str| value.replace('\\', "\\\\").replace('"', "\\\""); + let query = format!( + "recording:\"{}\" AND artist:\"{}\"", + escape(title), + escape(artist) + ); + self.rates[2].wait(check)?; + let body = self + .http + .request( + &format!( + "https://musicbrainz.org/ws/2/recording?fmt=json&limit=5&query={}", + urls::query_escape(&query) + ), + None, + false, + check, + )? + .body; + let payload: models::Recordings = decode(&body, false, check)?; + let mut id = String::new(); + for candidate in payload.recordings.unwrap_or_default() { + check_active(check)?; + let found = candidate + .artists + .as_deref() + .unwrap_or_default() + .first() + .map(|artist| artist.name.as_str()) + .unwrap_or_default(); + if candidate.score >= 90 + && lyrics::matching::normalize_title(&candidate.title) + == lyrics::matching::normalize_title(title) + && models::artists_match(artist, found) + { + id = candidate.id; + break; + } + } + if id.is_empty() { + return Err(ResolverError::Failed("no verified recording match".into())); + } + self.rates[2].wait(check)?; + let body = self + .http + .request( + &format!( + "https://musicbrainz.org/ws/2/recording/{}?fmt=json&inc=url-rels+isrcs", + urls::path_escape(&id) + ), + None, + false, + check, + )? + .body; + let payload: models::Relations = decode(&body, false, check)?; + let mut result = Resolution { + metadata: hint.clone(), + ..Resolution::default() + }; + for relation in payload.relations.unwrap_or_default() { + check_active(check)?; + let platform = urls::platform(&relation.url.resource); + let direct = urls::direct(platform, &relation.url.resource); + if !direct.is_empty() { + result.links.entry(platform.into()).or_insert(direct); + } + } + if result.links.is_empty() { + return Err(ResolverError::Failed( + "recording has no supported platform relations".into(), + )); + } + Ok(result) + } + + fn squigly(&self, input: &str, check: &Check<'_>) -> Result<Resolution, ResolverError> { + self.rates[3].wait(check)?; + let body = serde_json::to_string(&BTreeMap::from([("url", input)])) + .unwrap() + .replace('&', "\\u0026") + .replace('<', "\\u003c") + .replace('>', "\\u003e") + .replace('\u{2028}', "\\u2028") + .replace('\u{2029}', "\\u2029"); + let created: models::Created = decode( + &self + .http + .request("https://squigly.link/api/create", Some(body), false, check)? + .body, + false, + check, + )?; + let page = created.full_url.trim(); + let parsed = UrlParts::parse(page) + .filter(|url| url.scheme == "https" && url.hostname == "squigly.link") + .ok_or_else(|| { + ResolverError::Failed("create endpoint returned an invalid page URL".into()) + })?; + // Preserve encoded paths and query ordering in the page URL. + let response = self + .http + .request(&parsed.display_url(), None, true, check)?; + const MARKER: &[u8] = b"window.__SQUIGLY_LINK__ ="; + let offset = response + .body + .windows(MARKER.len()) + .position(|value| value == MARKER) + .ok_or_else(|| { + ResolverError::Failed("result page contains no resolver payload".into()) + })?; + let payload: models::Page = decode(&response.body[offset + MARKER.len()..], true, check)?; + let mut result = Resolution { + metadata: Metadata { + title: if payload.data.title.is_empty() { + created.title + } else { + payload.data.title + }, + artist: if payload.data.artist.is_empty() { + created.artist + } else { + payload.data.artist + }, + }, + ..Resolution::default() + }; + for (platform, service) in payload.data.services.0 { + check_active(check)?; + if let Some(service) = service { + let platform = urls::canonical(&platform); + let direct = urls::direct(platform, &service.url); + if !direct.is_empty() { + result.links.insert(platform.into(), direct); + } + } + } + if result.links.is_empty() { + return Err(ResolverError::Failed( + "result page returned no direct platform links".into(), + )); + } + Ok(result) + } +} + +impl Resolver for PlatformResolverChain { + fn resolve( + &self, + input: &str, + hint: &Metadata, + check: &Check<'_>, + ) -> Result<Resolution, ResolverError> { + resolve_chain(input, hint, check, self.timeout, |provider, hint, check| { + self.resolve_one(provider, input, hint, check) + }) + } +} + +fn resolve_chain( + input: &str, + hint: &Metadata, + check: &Check<'_>, + timeout: Duration, + mut resolve: impl FnMut(&str, &Metadata, &Check<'_>) -> Result<Resolution, ResolverError>, +) -> Result<Resolution, ResolverError> { + if input.len() > 64 << 10 || hint.title.len() + hint.artist.len() > 64 << 10 { + return Err(ResolverError::Failed("resolver input exceeds limit".into())); + } + let started = Instant::now(); + let bounded = || { + check()?; + if started.elapsed() >= timeout { + Err(CHAIN_DEADLINE.into()) + } else { + Ok(()) + } + }; + let mut result = Resolution { + metadata: hint.clone(), + ..Resolution::default() + }; + let mut errors = Vec::new(); + for (provider, name) in [ + ("songlink", "Song.link Web"), + ("unitune", "UniTune"), + ("musicbrainz", "MusicBrainz"), + ("squigly", "Squigly"), + ] { + match resolve(provider, &result.metadata, &bounded) { + Ok(resolved) => { + for (platform, link) in resolved.links { + let direct = urls::direct(&platform, &link); + if !direct.is_empty() { + result.links.entry(platform).or_insert(direct); + } + } + if result.metadata.title.is_empty() { + result.metadata.title = resolved.metadata.title.trim().into(); + } + if result.metadata.artist.is_empty() { + result.metadata.artist = resolved.metadata.artist.trim().into(); + } + if useful(&result.links) { + break; + } + } + Err(ResolverError::Cancelled(message)) if message == CHAIN_DEADLINE => { + // The chain's own deadline ends fallback work, but must not + // discard links already found. Caller cancellation and + // transport policy changes still invalidate the operation. + check_active(check)?; + errors.push(format!("{name}: {message}")); + break; + } + Err(error @ ResolverError::Cancelled(_)) => return Err(error), + Err(error) => errors.push(format!("{name}: {error}")), + } + } + check_active(check)?; + add_source(&mut result.links, input); + if !result.links.is_empty() { + return Ok(result); + } + Err(ResolverError::Failed(errors.join("\n"))) +} + +fn decode<T: DeserializeOwned>( + bytes: &[u8], + streaming: bool, + check: &Check<'_>, +) -> Result<T, ResolverError> { + check_active(check)?; + let value = if streaming { + lyrics::decode_response(bytes) + } else { + lyrics::decode_document(bytes) + }; + check_active(check)?; + value.map_err(|error| ResolverError::Failed(error.to_string())) +} + +pub fn songlink_html( + bytes: &[u8], + input: &str, + check: &Check<'_>, +) -> Result<Resolution, ResolverError> { + check_active(check)?; + if bytes.len() > 8 << 20 { + return Err(ResolverError::Failed("HTML input exceeds limit".into())); + } + let text = lyrics::text_from_bytes(bytes); + let mut parser = + html5ever::parse_document(HtmlTreeSink::new(Html::new_document()), Default::default()); + let mut start = 0; + while start < text.len() { + check_active(check)?; + let end = text.floor_char_boundary((start + 4096).min(text.len())); + parser.process(text[start..end].into()); + start = end; + if parser.tokenizer.sink.sink.0.borrow().tree.nodes().len() > 100_000 { + return Err(ResolverError::Failed("HTML node limit exceeded".into())); + } + } + let html = parser.finish(); + let mut result = Resolution::default(); + for node in html.tree.root().descendants() { + check_active(check)?; + if let Some(element) = node.value().as_element() + && element.name() == "a" + && let Some(href) = element.attr("href") + { + let platform = urls::platform(href); + let direct = urls::direct(platform, href); + if !direct.is_empty() { + result.links.entry(platform.into()).or_insert(direct); + } + } + } + add_source(&mut result.links, input); + if result.links.len() < 2 { + return Err(ResolverError::Failed( + "web page returned no cross-platform links".into(), + )); + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + #[test] + fn chain_keeps_web_links_when_a_later_resolver_times_out() { + let mut calls = Vec::new(); + let source = "https://open.spotify.com/track/source"; + let result = resolve_chain( + source, + &Metadata::default(), + &|| Ok(()), + Duration::from_millis(10), + |provider, _, check| { + calls.push(provider.to_owned()); + if provider == "songlink" { + return songlink_html( + br#"<a href="https://music.amazon.com/tracks/EXAMPLE?x=1&y=2">Track</a>"#, + source, + &|| Ok(()), + ); + } + // Exercise the same deadline check used while waiting for an + // upstream request or a rate-limit slot, without live servers. + std::thread::sleep(Duration::from_millis(10)); + check_active(check)?; + panic!("fallback should have reached the chain deadline"); + }, + ) + .unwrap(); + assert_eq!(calls, ["songlink", "unitune"]); + let availability = super::super::availability::from_links("source", &result.links); + assert!(availability.amazon); + assert_eq!( + availability.amazon_url, + "https://music.amazon.com/tracks/EXAMPLE?x=1&y=2" + ); + assert_eq!(result.links["spotify"], source); + } + + #[test] + fn chain_deadline_keeps_source_but_cancellation_discards_partial_links() { + let source = "https://open.spotify.com/track/source"; + let result = resolve_chain( + source, + &Metadata::default(), + &|| Ok(()), + Duration::ZERO, + |_, _, check| { + check_active(check)?; + unreachable!() + }, + ) + .unwrap(); + assert_eq!( + result.links, + BTreeMap::from([("spotify".into(), source.into())]) + ); + + for message in ["request cancelled", "network policy changed"] { + let cancelled = AtomicBool::new(false); + let mut calls = 0; + let result = resolve_chain( + source, + &Metadata::default(), + &|| { + if cancelled.load(Ordering::Acquire) { + Err(message.into()) + } else { + Ok(()) + } + }, + Duration::ZERO, + |_, _, _| { + calls += 1; + if calls == 1 { + return Ok(Resolution { + links: BTreeMap::from([( + "amazonMusic".into(), + "https://music.amazon.com/tracks/EXAMPLE".into(), + )]), + ..Resolution::default() + }); + } + if message == "request cancelled" { + cancelled.store(true, Ordering::Release); + Err(ResolverError::Cancelled(CHAIN_DEADLINE.into())) + } else { + Err(ResolverError::Cancelled(message.into())) + } + }, + ); + assert_eq!(calls, 2); + assert_eq!(result, Err(ResolverError::Cancelled(message.into()))); + } + } + + #[test] + fn html_parsing_is_cancellable_and_bounds_bytes_and_nodes() { + let checks = AtomicUsize::new(0); + let html = "<a>Example</a>".repeat(20_000); + assert!(matches!( + songlink_html(html.as_bytes(), "", &|| { + if checks.fetch_add(1, Ordering::AcqRel) >= 3 { + Err("cancel parser".into()) + } else { + Ok(()) + } + }), + Err(ResolverError::Cancelled(_)) + )); + assert!(songlink_html(&vec![b' '; (8 << 20) + 1], "", &|| Ok(())).is_err()); + assert!( + matches!(songlink_html("<i></i>".repeat(100_001).as_bytes(), "", &|| Ok(())), + Err(ResolverError::Failed(message)) if message == "HTML node limit exceeded") + ); + } +} diff --git a/rust_backend/crates/providers/src/resolver/http.rs b/rust_backend/crates/providers/src/resolver/http.rs new file mode 100644 index 00000000..dd29f7c0 --- /dev/null +++ b/rust_backend/crates/providers/src/resolver/http.rs @@ -0,0 +1,197 @@ +use super::{Check, ResolverError, check_active}; +use spotiflac_network::{HttpRequest, NetworkService, NetworkSession, url::UrlParts}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +pub struct ResolverHttp { + session: Arc<NetworkSession>, + overrides: BTreeMap<String, String>, + timeout: Duration, +} + +pub struct Response { + pub url: String, + pub body: Vec<u8>, + pub status: u16, + pub headers: BTreeMap<String, Vec<String>>, +} + +impl ResolverHttp { + pub fn new( + network: &Arc<NetworkService>, + overrides: BTreeMap<String, String>, + timeout: Duration, + ) -> Result<Self, ResolverError> { + for (origin, replacement) in &overrides { + for value in [origin, replacement] { + let url = UrlParts::parse(value) + .ok_or_else(|| ResolverError::Failed("invalid resolver origin".into()))?; + if url.scheme != "https" + || url.hostname.is_empty() + || url.has_credentials + || !matches!(url.raw_path.as_str(), "" | "/") + || !url.raw_query.is_empty() + || !url.fragment.is_empty() + { + return Err(ResolverError::Failed( + "resolver overrides require HTTPS origins without credentials".into(), + )); + } + } + } + Ok(Self { + session: network.native_session(timeout), + overrides, + timeout, + }) + } + + pub fn request( + &self, + endpoint: &str, + body: Option<String>, + html: bool, + check: &Check<'_>, + ) -> Result<Response, ResolverError> { + self.execute(endpoint, body, html, None, check) + } + + pub fn metadata( + &self, + endpoint: &str, + language: &str, + check: &Check<'_>, + ) -> Result<Response, ResolverError> { + self.execute(endpoint, None, false, Some(language), check) + } + + fn execute( + &self, + endpoint: &str, + body: Option<String>, + html: bool, + language: Option<&str>, + check: &Check<'_>, + ) -> Result<Response, ResolverError> { + check_active(check)?; + let parts = UrlParts::parse(endpoint) + .ok_or_else(|| ResolverError::Failed("invalid resolver URL".into()))?; + let origin = format!("{}://{}", parts.scheme, parts.authority()); + let replacement = self.overrides.get(&origin); + let endpoint = if let Some(base) = replacement { + format!( + "{}{}", + base.trim_end_matches('/'), + &endpoint[origin.len()..] + ) + } else { + endpoint.into() + }; + let mut headers = BTreeMap::new(); + if let Some(language) = language { + headers.insert("Accept".into(), "application/json".into()); + headers.insert("Accept-Language".into(), language.into()); + } + if html && origin == "https://song.link" { + headers.insert("Accept".into(), "text/html,application/xhtml+xml".into()); + } + if body.is_some() { + headers.insert("Content-Type".into(), "application/json".into()); + } + let post = body.is_some(); + let failure = |message: String| { + if let Err(cancelled) = check() { + ResolverError::Cancelled(cancelled) + } else if message == "network policy changed" { + ResolverError::Cancelled(message) + } else if message.starts_with("invalid ") + || message == "blocking HTTP host called from async executor" + { + ResolverError::Failed(message) + } else { + ResolverError::Transport(message) + } + }; + let mut stream = self + .session + .open_stream( + HttpRequest { + url: endpoint, + method: if post { "POST" } else { "GET" }.into(), + body: body.unwrap_or_default(), + headers, + default_json: false, + user_agent: if language.is_some() { + "Go-http-client/1.1".into() + } else { + spotiflac_network::random_user_agent() + }, + }, + self.timeout, + self.timeout, + check, + ) + .map_err(&failure)?; + let status = stream.response.status; + if language.is_none() && status != 200 && !(post && status == 201) { + return Err(ResolverError::Failed(format!("HTTP {status}"))); + } + let mut url = stream.response.url.clone(); + if let Some(base) = replacement { + let base = base.trim_end_matches('/'); + if let Some(suffix) = url.strip_prefix(base) + && (suffix.is_empty() || suffix.starts_with(['/', '?', '#'])) + { + url = format!("{origin}{suffix}"); + } + } + if origin == "https://song.link" { + let hostname = UrlParts::parse(&url) + .map(|parts| parts.hostname.to_ascii_lowercase()) + .unwrap_or_default(); + if !matches!( + hostname.as_str(), + "song.link" | "album.link" | "artist.link" | "odesli.co" | "www.odesli.co" + ) { + return Err(ResolverError::Failed( + "web page redirected to an unexpected host".into(), + )); + } + } + let limit = if language.is_some() { + 16 << 20 + } else if html { + 8 << 20 + } else { + 2 << 20 + }; + let mut body = Vec::new(); + let mut buffer = [0; 16 << 10]; + loop { + let capacity = buffer.len().min(limit + 1 - body.len()); + let read = stream + .read(&mut buffer[..capacity], check) + .map_err(&failure)?; + if read == 0 { + break; + } + body.extend_from_slice(&buffer[..read]); + if body.len() > limit { + return Err(ResolverError::Failed(format!( + "response exceeds {limit} bytes" + ))); + } + } + check_active(check)?; + if body.is_empty() && language.is_none() { + return Err(ResolverError::Failed("response body is empty".into())); + } + Ok(Response { + url, + body, + status, + headers: stream.response.headers.clone(), + }) + } +} diff --git a/rust_backend/crates/providers/src/resolver/rate.rs b/rust_backend/crates/providers/src/resolver/rate.rs new file mode 100644 index 00000000..09a3b1e0 --- /dev/null +++ b/rust_backend/crates/providers/src/resolver/rate.rs @@ -0,0 +1,77 @@ +use super::{Check, ResolverError, check_active}; +use std::collections::VecDeque; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +pub struct RateLimiter { + timestamps: Mutex<VecDeque<Instant>>, + maximum: usize, + window: Duration, +} + +impl RateLimiter { + pub fn new(maximum: usize, window: Duration) -> Self { + Self { + timestamps: Mutex::new(VecDeque::new()), + maximum, + window, + } + } + + pub fn wait(&self, check: &Check<'_>) -> Result<(), ResolverError> { + loop { + check_active(check)?; + let mut timestamps = self.timestamps.lock().unwrap(); + let now = Instant::now(); + while timestamps + .front() + .is_some_and(|stamp| now.duration_since(*stamp) >= self.window) + { + timestamps.pop_front(); + } + if timestamps.len() < self.maximum { + timestamps.push_back(now); + return Ok(()); + } + let remaining = self + .window + .saturating_sub(now.duration_since(*timestamps.front().unwrap())); + drop(timestamps); + std::thread::sleep(remaining.min(Duration::from_millis(25))); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn concurrent_waiters_reserve_distinct_slots_and_can_cancel() { + let limiter = RateLimiter::new(1, Duration::from_millis(30)); + let stamps = Mutex::new(Vec::new()); + std::thread::scope(|scope| { + for _ in 0..4 { + scope.spawn(|| { + limiter.wait(&|| Ok(())).unwrap(); + stamps.lock().unwrap().push(Instant::now()); + }); + } + }); + let mut stamps = stamps.into_inner().unwrap(); + stamps.sort_unstable(); + assert!(stamps.last().unwrap().duration_since(stamps[0]) >= Duration::from_millis(85)); + let limiter = RateLimiter::new(1, Duration::from_secs(60)); + limiter.wait(&|| Ok(())).unwrap(); + let started = Instant::now(); + assert_eq!( + limiter.wait(&|| if started.elapsed() >= Duration::from_millis(30) { + Err("cancel rate wait".into()) + } else { + Ok(()) + }), + Err(ResolverError::Cancelled("cancel rate wait".into())) + ); + assert!(started.elapsed() < Duration::from_secs(1)); + } +} diff --git a/rust_backend/crates/providers/src/resolver/service.rs b/rust_backend/crates/providers/src/resolver/service.rs new file mode 100644 index 00000000..2df6418b --- /dev/null +++ b/rust_backend/crates/providers/src/resolver/service.rs @@ -0,0 +1,533 @@ +use super::{Check, Metadata, Resolution, Resolver, ResolverError, check_active}; +use crate::lyrics::{LyricsError, builtin::TrackResolver}; +use std::collections::BTreeMap; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +const POSITIVE_TTL: Duration = Duration::from_secs(30 * 60); +const NEGATIVE_TTL: Duration = Duration::from_secs(5 * 60); +const MAX_CACHE: usize = 500; +const MAX_FLIGHTS: usize = 64; +const POLL: Duration = Duration::from_millis(25); +type Outcome = Result<Resolution, ResolverError>; + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +struct Key { + generation: u64, + region: String, + url: String, + title: String, + artist: String, + cached: bool, +} + +#[derive(Default)] +struct Flight { + waiters: AtomicUsize, + result: Mutex<Option<Outcome>>, + ready: Condvar, +} + +struct Waiter(Arc<Flight>); +impl Drop for Waiter { + fn drop(&mut self) { + self.0.waiters.fetch_sub(1, Ordering::AcqRel); + } +} + +struct CacheEntry { + value: Outcome, + expires: Instant, +} + +struct State { + region: String, + generation: u64, + active: usize, + flights: BTreeMap<Key, Arc<Flight>>, + cache: BTreeMap<(String, String), CacheEntry>, +} + +struct Inner { + resolver: Arc<dyn Resolver>, + closed: AtomicBool, + state: Mutex<State>, + idle: Condvar, +} + +/// Native owner for the active resolver chain. Spotify availability resolutions +/// retain Go's region-scoped TTLs; arbitrary URL lookups are only coalesced. +pub struct PlatformResolverService { + inner: Arc<Inner>, +} + +impl PlatformResolverService { + pub fn new(resolver: Arc<dyn Resolver>) -> Self { + Self { + inner: Arc::new(Inner { + resolver, + closed: AtomicBool::new(false), + state: Mutex::new(State { + region: "US".into(), + generation: 0, + active: 0, + flights: BTreeMap::new(), + cache: BTreeMap::new(), + }), + idle: Condvar::new(), + }), + } + } + + pub fn set_region(&self, region: &str) -> Result<(), ResolverError> { + let region = spotiflac_core::matching::uppercase(region.trim()); + let mut state = self.inner.state.lock().unwrap(); + self.inner.check(&|| Ok(()))?; + state.region = if region.len() == 2 && region.bytes().all(|byte| byte.is_ascii_uppercase()) + { + region + } else { + "US".into() + }; + Ok(()) + } + + pub fn clear_cache(&self) -> Result<usize, ResolverError> { + let mut state = self.inner.state.lock().unwrap(); + self.inner.check(&|| Ok(()))?; + state.generation = state.generation.wrapping_add(1); + let count = state.cache.len(); + state.cache.clear(); + Ok(count) + } + + pub fn resolve_url(&self, url: &str, hint: &Metadata, check: &Check<'_>) -> Outcome { + self.lookup(url, hint, false, check) + } + + pub fn resolve_spotify(&self, id: &str, check: &Check<'_>) -> Outcome { + self.inner.check(check)?; + let id = id.trim(); + if id.is_empty() { + return Err(ResolverError::Failed("spotify track ID is empty".into())); + } + self.lookup( + &format!("https://open.spotify.com/track/{id}"), + &Metadata::default(), + true, + check, + ) + } + + fn lookup(&self, url: &str, hint: &Metadata, cached: bool, check: &Check<'_>) -> Outcome { + self.inner.check(check)?; + if url.len() > 64 << 10 || hint.title.len() + hint.artist.len() > 64 << 10 { + return Err(ResolverError::Failed("resolver input exceeds limit".into())); + } + let (key, flight, start) = { + let mut state = self.inner.state.lock().unwrap(); + self.inner.check(check)?; + if cached { + let cache_key = (state.region.clone(), url.to_owned()); + if let Some(entry) = state.cache.get(&cache_key) + && Instant::now() <= entry.expires + { + return entry.value.clone().map_err(|_| { + ResolverError::Failed("track availability unavailable (cached)".into()) + }); + } + state.cache.remove(&cache_key); + } + let key = Key { + generation: state.generation, + region: state.region.clone(), + url: url.into(), + title: hint.title.clone(), + artist: hint.artist.clone(), + cached, + }; + let existing = state + .flights + .get(&key) + .filter(|flight| flight.waiters.load(Ordering::Acquire) > 0) + .cloned(); + let (flight, start) = if let Some(flight) = existing { + (flight, false) + } else { + if state.active >= MAX_FLIGHTS { + return Err(ResolverError::Busy); + } + let flight = Arc::new(Flight::default()); + state.flights.insert(key.clone(), Arc::clone(&flight)); + state.active += 1; + (flight, true) + }; + flight.waiters.fetch_add(1, Ordering::AcqRel); + (key, flight, start) + }; + let _waiter = Waiter(Arc::clone(&flight)); + if start { + let inner = Arc::clone(&self.inner); + let worker_key = key.clone(); + let worker_flight = Arc::clone(&flight); + let spawned = std::thread::Builder::new() + .name("platform-resolver".into()) + .spawn(move || { + let check = || { + if inner.closed.load(Ordering::Acquire) { + Err("platform resolver closed".into()) + } else if worker_flight.waiters.load(Ordering::Acquire) == 0 { + Err("platform resolution cancelled".into()) + } else { + Ok(()) + } + }; + let result = catch_unwind(AssertUnwindSafe(|| { + inner.resolver.resolve( + &worker_key.url, + &Metadata { + title: worker_key.title.clone(), + artist: worker_key.artist.clone(), + }, + &check, + ) + })) + .unwrap_or_else(|_| { + Err(ResolverError::Failed("platform resolver panicked".into())) + }); + // A resolver callback cannot publish success after losing all + // waiters, even if it forgot to check immediately before return. + let result = check_active(&check).and(result); + inner.finish(&worker_key, &worker_flight, result); + }); + if let Err(error) = spawned { + self.inner + .finish(&key, &flight, Err(ResolverError::Failed(error.to_string()))); + } + } + loop { + self.inner.check(check)?; + let result = flight.result.lock().unwrap(); + if let Some(result) = result.as_ref() { + self.inner.check(check)?; + return result.clone(); + } + drop(flight.ready.wait_timeout(result, POLL).unwrap()); + } + } + + pub fn shutdown(&self) { + self.inner.closed.store(true, Ordering::Release); + let mut state = self.inner.state.lock().unwrap(); + while state.active != 0 { + state = self.inner.idle.wait(state).unwrap(); + } + state.cache.clear(); + } +} + +impl Drop for PlatformResolverService { + fn drop(&mut self) { + self.shutdown(); + } +} + +impl TrackResolver for PlatformResolverService { + fn deezer_id_from_spotify(&self, id: &str, check: &Check<'_>) -> Result<String, LyricsError> { + let resolution = self + .resolve_spotify(id, check) + .map_err(|error| match error { + ResolverError::Cancelled(message) => LyricsError::Cancelled(message), + ResolverError::Closed => LyricsError::Cancelled(error.to_string()), + _ => LyricsError::Other(error.to_string()), + })?; + let link = resolution + .links + .get("deezer") + .ok_or_else(|| LyricsError::Other("track not found on Deezer".into()))?; + let mut id = link.rsplit('/').next().unwrap_or_default(); + if let Some(index) = id.find('?').filter(|index| *index > 0) { + id = &id[..index]; + } + if id.is_empty() { + return Err(LyricsError::Other("track not found on Deezer".into())); + } + Ok(id.into()) + } +} + +impl Inner { + fn check(&self, check: &Check<'_>) -> Result<(), ResolverError> { + if self.closed.load(Ordering::Acquire) { + Err(ResolverError::Closed) + } else { + check_active(check) + } + } + + fn finish(&self, key: &Key, flight: &Arc<Flight>, result: Outcome) { + let mut state = self.state.lock().unwrap(); + if key.cached + && key.generation == state.generation + && !self.closed.load(Ordering::Acquire) + && !matches!( + result, + Err(ResolverError::Cancelled(_) | ResolverError::Closed | ResolverError::Busy) + ) + { + let cache_key = (key.region.clone(), key.url.clone()); + if !state.cache.contains_key(&cache_key) + && state.cache.len() >= MAX_CACHE + && let Some(oldest) = state + .cache + .iter() + .min_by_key(|(_, entry)| entry.expires) + .map(|(key, _)| key.clone()) + { + state.cache.remove(&oldest); + } + let ttl = if result.is_ok() { + POSITIVE_TTL + } else { + NEGATIVE_TTL + }; + state.cache.insert( + cache_key, + CacheEntry { + value: result.clone(), + expires: Instant::now() + ttl, + }, + ); + } + *flight.result.lock().unwrap() = Some(result); + flight.ready.notify_all(); + if state + .flights + .get(key) + .is_some_and(|current| Arc::ptr_eq(current, flight)) + { + state.flights.remove(key); + } + state.active -= 1; + self.idle.notify_all(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Default)] + struct Probe { + calls: AtomicUsize, + release: AtomicBool, + failure: AtomicBool, + } + + impl Resolver for Probe { + fn resolve(&self, _: &str, _: &Metadata, check: &Check<'_>) -> Outcome { + self.calls.fetch_add(1, Ordering::AcqRel); + while !self.release.load(Ordering::Acquire) { + check_active(check)?; + std::thread::sleep(Duration::from_millis(2)); + } + if self.failure.load(Ordering::Acquire) { + return Err(ResolverError::Failed("fixture unavailable".into())); + } + Ok(Resolution { + links: BTreeMap::from([( + "deezer".into(), + "https://www.deezer.com/track/42?source=example".into(), + )]), + ..Resolution::default() + }) + } + } + + fn until(predicate: impl Fn() -> bool) { + let started = Instant::now(); + while !predicate() { + assert!( + started.elapsed() < Duration::from_secs(3), + "resolver test wait timed out" + ); + std::thread::sleep(Duration::from_millis(2)); + } + } + + #[test] + fn coalesced_waiters_cancel_independently_and_cache_by_region() { + let probe = Arc::new(Probe::default()); + let service = Arc::new(PlatformResolverService::new(probe.clone())); + let cancelled = Arc::new(AtomicBool::new(false)); + let first = { + let service = service.clone(); + let cancelled = cancelled.clone(); + std::thread::spawn(move || { + service.resolve_spotify("example", &|| { + if cancelled.load(Ordering::Acquire) { + Err("first cancelled".into()) + } else { + Ok(()) + } + }) + }) + }; + until(|| probe.calls.load(Ordering::Acquire) == 1); + let second = { + let service = service.clone(); + std::thread::spawn(move || service.deezer_id_from_spotify("example", &|| Ok(()))) + }; + until(|| { + service + .inner + .state + .lock() + .unwrap() + .flights + .values() + .next() + .unwrap() + .waiters + .load(Ordering::Acquire) + == 2 + }); + cancelled.store(true, Ordering::Release); + assert_eq!( + first.join().unwrap(), + Err(ResolverError::Cancelled("first cancelled".into())) + ); + probe.release.store(true, Ordering::Release); + assert_eq!(second.join().unwrap().unwrap(), "42"); + assert_eq!( + service + .deezer_id_from_spotify(" example ", &|| Ok(())) + .unwrap(), + "42" + ); + assert_eq!(probe.calls.load(Ordering::Acquire), 1); + service.set_region("id").unwrap(); + service.resolve_spotify("example", &|| Ok(())).unwrap(); + service.set_region("not a region").unwrap(); + service.resolve_spotify("example", &|| Ok(())).unwrap(); + assert_eq!(probe.calls.load(Ordering::Acquire), 2); + assert_eq!(service.clear_cache().unwrap(), 2); + } + + #[test] + fn last_waiter_and_shutdown_cancel_workers_and_reject_retained_handles() { + let probe = Arc::new(Probe::default()); + let service = Arc::new(PlatformResolverService::new(probe.clone())); + let started = Instant::now(); + assert!(matches!( + service.resolve_spotify( + "cancel", + &|| if started.elapsed() >= Duration::from_millis(40) { + Err("cancel".into()) + } else { + Ok(()) + } + ), + Err(ResolverError::Cancelled(_)) + )); + until(|| service.inner.state.lock().unwrap().active == 0); + assert!(service.inner.state.lock().unwrap().cache.is_empty()); + let waiting = { + let service = service.clone(); + std::thread::spawn(move || service.resolve_spotify("shutdown", &|| Ok(()))) + }; + until(|| probe.calls.load(Ordering::Acquire) == 2); + service.shutdown(); + assert_eq!(waiting.join().unwrap(), Err(ResolverError::Closed)); + assert_eq!( + service.resolve_spotify("", &|| Ok(())), + Err(ResolverError::Closed) + ); + assert_eq!(service.set_region("ID"), Err(ResolverError::Closed)); + assert_eq!(service.clear_cache(), Err(ResolverError::Closed)); + assert_eq!(service.inner.state.lock().unwrap().active, 0); + } + + #[test] + fn cache_expiry_failures_and_generation_changes_preserve_fresh_state() { + let probe = Arc::new(Probe::default()); + let service = Arc::new(PlatformResolverService::new(probe.clone())); + let old = { + let service = service.clone(); + std::thread::spawn(move || service.resolve_spotify("old", &|| Ok(()))) + }; + until(|| probe.calls.load(Ordering::Acquire) == 1); + service.clear_cache().unwrap(); + probe.release.store(true, Ordering::Release); + old.join().unwrap().unwrap(); + assert!(service.inner.state.lock().unwrap().cache.is_empty()); + service.resolve_spotify("old", &|| Ok(())).unwrap(); + probe.failure.store(true, Ordering::Release); + assert!(service.resolve_spotify("failed", &|| Ok(())).is_err()); + assert_eq!(probe.calls.load(Ordering::Acquire), 3); + assert!(service.resolve_spotify("failed", &|| Ok(())).is_err()); + assert_eq!(probe.calls.load(Ordering::Acquire), 3); + { + let mut state = service.inner.state.lock().unwrap(); + let positive = state + .cache + .values() + .find(|entry| entry.value.is_ok()) + .unwrap(); + let negative = state + .cache + .values() + .find(|entry| entry.value.is_err()) + .unwrap(); + assert!( + positive.expires.duration_since(negative.expires) > Duration::from_secs(24 * 60) + ); + for entry in state.cache.values_mut() { + entry.expires = Instant::now() - Duration::from_secs(1); + } + } + probe.failure.store(false, Ordering::Release); + service.resolve_spotify("failed", &|| Ok(())).unwrap(); + assert_eq!(probe.calls.load(Ordering::Acquire), 4); + } + + #[test] + fn distinct_lookup_and_cache_counts_are_bounded() { + let probe = Arc::new(Probe::default()); + let service = Arc::new(PlatformResolverService::new(probe.clone())); + let mut waiters = Vec::new(); + for index in 0..MAX_FLIGHTS { + let service = service.clone(); + waiters.push(std::thread::spawn(move || { + service.resolve_spotify(&index.to_string(), &|| Ok(())) + })); + } + until(|| probe.calls.load(Ordering::Acquire) == MAX_FLIGHTS); + assert_eq!( + service.resolve_spotify("over-limit", &|| Ok(())), + Err(ResolverError::Busy) + ); + probe.release.store(true, Ordering::Release); + for waiter in waiters { + waiter.join().unwrap().unwrap(); + } + service.clear_cache().unwrap(); + for index in 0..MAX_CACHE + 3 { + service + .resolve_spotify(&index.to_string(), &|| Ok(())) + .unwrap(); + } + assert_eq!(service.inner.state.lock().unwrap().cache.len(), MAX_CACHE); + assert!( + !service + .inner + .state + .lock() + .unwrap() + .cache + .contains_key(&("US".into(), "https://open.spotify.com/track/0".into())) + ); + } +} diff --git a/rust_backend/crates/providers/src/resolver/urls.rs b/rust_backend/crates/providers/src/resolver/urls.rs new file mode 100644 index 00000000..fd606af6 --- /dev/null +++ b/rust_backend/crates/providers/src/resolver/urls.rs @@ -0,0 +1,178 @@ +use spotiflac_network::url::UrlParts; + +pub fn canonical(platform: &str) -> &'static str { + match platform + .trim() + .to_lowercase() + .replace(['-', '_', ' '], "") + .as_str() + { + "spotify" => "spotify", + "deezer" => "deezer", + "tidal" => "tidal", + "qobuz" => "qobuz", + "soundcloud" => "soundcloud", + "bandcamp" => "bandcamp", + "apple" | "applemusic" => "appleMusic", + "amazon" | "amazonmusic" => "amazonMusic", + "youtube" => "youtube", + "youtubemusic" => "youtubeMusic", + _ => "", + } +} + +pub fn platform(value: &str) -> &'static str { + let Some(url) = UrlParts::parse(value.trim()) else { + return ""; + }; + match url.hostname.to_ascii_lowercase().as_str() { + "open.spotify.com" => "spotify", + "deezer.com" | "www.deezer.com" => "deezer", + "tidal.com" | "www.tidal.com" | "listen.tidal.com" => "tidal", + "open.qobuz.com" | "play.qobuz.com" | "www.qobuz.com" => "qobuz", + "music.apple.com" | "geo.music.apple.com" => "appleMusic", + "music.amazon.com" => "amazonMusic", + "music.youtube.com" => "youtubeMusic", + "youtube.com" | "www.youtube.com" | "youtu.be" => "youtube", + "soundcloud.com" | "www.soundcloud.com" | "m.soundcloud.com" => "soundcloud", + host if host == "bandcamp.com" || host.ends_with(".bandcamp.com") => "bandcamp", + _ => "", + } +} + +pub fn direct(provider: &str, value: &str) -> String { + let value = value.trim(); + let provider = canonical(provider); + let Some(url) = UrlParts::parse(value) else { + return String::new(); + }; + if provider.is_empty() + || url.scheme != "https" + || platform(value) != provider + || url.escaped_path().to_ascii_lowercase().contains("/search") + { + return String::new(); + } + // Preserve Go's host case, optional port, escaped path, query ordering and + // fragment. Url::parse would normalize dot segments and default ports. + let Some((_, rest)) = value.split_once("://") else { + return String::new(); + }; + let authority = rest.split(['/', '?', '#']).next().unwrap_or_default(); + let authority = if let Some((user, host)) = authority.rsplit_once('@') { + let encoded = |value: &str| escape_bytes(&unescape(value), b";$&=+,", false); + let user = if let Some((name, password)) = user.split_once(':') { + format!("{}:{}", encoded(name), encoded(password)) + } else { + encoded(user) + }; + format!("{user}@{host}") + } else { + authority.into() + }; + let mut output = format!("https://{authority}{}", url.escaped_path()); + if url.force_query || !url.raw_query.is_empty() { + output.push('?'); + output.push_str(&url.raw_query); + } + if !url.fragment.is_empty() { + output.push('#'); + if url + .fragment + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-._~!$&'()*+,;=:@/?%[]".contains(&byte)) + { + output.push_str(&url.fragment); + } else { + output.push_str(&escape_bytes( + &unescape(&url.fragment), + b"!$&()*+,;=:@/?", + false, + )); + } + } + output +} + +fn escape(value: &str, extra: &[u8], spaces: bool) -> String { + escape_bytes(value.as_bytes(), extra, spaces) +} + +fn escape_bytes(value: &[u8], extra: &[u8], spaces: bool) -> String { + use std::fmt::Write; + let mut output = String::new(); + for &byte in value { + if byte.is_ascii_alphanumeric() || b"-._~".contains(&byte) || extra.contains(&byte) { + output.push(char::from(byte)); + } else if spaces && byte == b' ' { + output.push('+'); + } else { + let _ = write!(output, "%{byte:02X}"); + } + } + output +} + +fn unescape(value: &str) -> Vec<u8> { + let mut bytes = value.bytes(); + let mut output = Vec::new(); + while let Some(byte) = bytes.next() { + if byte == b'%' { + let high = char::from(bytes.next().expect("validated URL escape")) + .to_digit(16) + .expect("validated URL escape"); + let low = char::from(bytes.next().expect("validated URL escape")) + .to_digit(16) + .expect("validated URL escape"); + output.push((high * 16 + low) as u8); + } else { + output.push(byte); + } + } + output +} + +pub fn path_escape(value: &str) -> String { + escape(value, b"$&+:=@", false) +} +pub fn query_escape(value: &str) -> String { + escape(value, b"", true) +} + +pub fn from_id(provider: &str, kind: &str, id: &str) -> Result<String, String> { + let provider = canonical(provider); + let id = id.trim(); + if provider.is_empty() || id.is_empty() { + return Err("invalid platform or entity ID".into()); + } + let kind = kind.trim().to_lowercase(); + let kind = if kind == "song" { "track" } else { &kind }; + if !matches!(kind, "track" | "album" | "artist") { + return Err(format!("unsupported entity type {kind:?}")); + } + let base = match provider { + "spotify" => "https://open.spotify.com", + "deezer" => "https://www.deezer.com", + "tidal" => "https://tidal.com/browse", + "qobuz" => "https://open.qobuz.com", + "amazonMusic" => { + return Ok(format!( + "https://music.amazon.com/{kind}s/{}", + path_escape(id) + )); + } + "youtube" | "youtubeMusic" => { + if kind != "track" { + return Err(format!("unsupported {provider} entity type {kind:?}")); + } + let host = if provider == "youtube" { + "www.youtube.com" + } else { + "music.youtube.com" + }; + return Ok(format!("https://{host}/watch?v={}", query_escape(id))); + } + _ => return Err(format!("cannot build a direct {provider} URL from an ID")), + }; + Ok(format!("{base}/{kind}/{}", path_escape(id))) +} diff --git a/rust_backend/rust-toolchain.toml b/rust_backend/rust-toolchain.toml new file mode 100644 index 00000000..c3f67b67 --- /dev/null +++ b/rust_backend/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.98.1" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/scripts/build_ios.sh b/scripts/build_ios.sh index da1b40c6..b759a5fb 100644 --- a/scripts/build_ios.sh +++ b/scripts/build_ios.sh @@ -1,45 +1,7 @@ #!/usr/bin/env bash -# Build the Go backend XCFramework used by SpotiFLAC Mobile on iOS. +# Build the Rust backend XCFramework used by SpotiFLAC Mobile on iOS. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_DIR="$(dirname "$SCRIPT_DIR")" -GO_BACKEND_DIR="$PROJECT_DIR/go_backend" -OUTPUT_DIR="$PROJECT_DIR/ios/Frameworks" - -echo "SpotiFLAC Mobile iOS backend build" -echo "Project: $PROJECT_DIR" -echo "Output: $OUTPUT_DIR" - -if [[ "$(uname)" != "Darwin" ]]; then - echo "Error: this script must run on macOS with Xcode installed." >&2 - exit 1 -fi - -if ! command -v go >/dev/null 2>&1; then - echo "Error: Go is not installed." >&2 - exit 1 -fi - -cd "$GO_BACKEND_DIR" -go mod download - -# Running inside the module installs the x/mobile revision pinned by go.mod. -go install golang.org/x/mobile/cmd/gomobile -export PATH="$(go env GOPATH)/bin:$PATH" -gomobile init - -mkdir -p "$OUTPUT_DIR" -gomobile bind \ - -target=ios \ - -tags ios \ - -o "$OUTPUT_DIR/Gobackend.xcframework" \ - . - -if [[ ! -d "$OUTPUT_DIR/Gobackend.xcframework" ]]; then - echo "Error: Gobackend.xcframework was not created." >&2 - exit 1 -fi - -echo "Built $OUTPUT_DIR/Gobackend.xcframework" +exec bash "$SCRIPT_DIR/build_rust_backend.sh" ios diff --git a/scripts/build_rust_backend.sh b/scripts/build_rust_backend.sh new file mode 100644 index 00000000..9edfaf2f --- /dev/null +++ b/scripts/build_rust_backend.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Stage Rust libraries and UniFFI bindings without changing the app's backend. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +RUST_BACKEND_DIR="$PROJECT_DIR/rust_backend" +PLATFORM="${1:-host}" + +case "$PLATFORM" in + host|android|ios) ;; + *) echo "Usage: bash scripts/build_rust_backend.sh [host|android|ios]" >&2; exit 1 ;; +esac + +if [[ "$PLATFORM" == "android" ]]; then + ANDROID_ABIS_RAW="${SPOTIFLAC_RUST_ANDROID_ABIS-arm64-v8a,armeabi-v7a}" + case "$ANDROID_ABIS_RAW" in + arm64-v8a|armeabi-v7a|arm64-v8a,armeabi-v7a|armeabi-v7a,arm64-v8a) ;; + *) + echo "Error: SPOTIFLAC_RUST_ANDROID_ABIS must contain arm64-v8a and/or armeabi-v7a, comma-separated without spaces or duplicates." >&2 + exit 1 + ;; + esac + IFS=',' read -r -a ANDROID_ABIS <<< "$ANDROID_ABIS_RAW" +fi + +cd "$RUST_BACKEND_DIR" +export CARGO_TARGET_DIR="$RUST_BACKEND_DIR/target" +# AES 0.8 requires this opt-in on ARM64. Runtime detection retains the software +# fallback on CPUs without AES; do not force target-feature=+aes on Android. +export RUSTFLAGS="${RUSTFLAGS:-} --cfg aes_armv8" + +case "$(uname -s)" in + Darwin) + HOST_LIBRARY="$CARGO_TARGET_DIR/release/libspotiflac_mobile.dylib" + # Keep cc-rs (QuickJS) and Rust compatible with the Swift host executable. + export MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-11.0}" + ;; + Linux) HOST_LIBRARY="$CARGO_TARGET_DIR/release/libspotiflac_mobile.so" ;; + *) echo "Error: use macOS or Linux for this build script." >&2; exit 1 ;; +esac + +# Build separately from bindgen so CLI and cargo-metadata features are not +# enabled in the shipped library through Cargo's workspace feature unification. +cargo build --locked --release -p spotiflac-mobile +for language in kotlin swift; do + cargo run --locked -p spotiflac-bindgen -- generate \ + --library "$HOST_LIBRARY" \ + --config crates/mobile/uniffi.toml \ + --language "$language" \ + --out-dir "$CARGO_TARGET_DIR/bindings/$language" \ + --no-format +done + +if [[ "$PLATFORM" == "android" ]]; then + if [[ -z "${ANDROID_NDK_HOME:-}" ]]; then + echo "Error: set ANDROID_NDK_HOME to Android NDK 29.0.14206865." >&2 + exit 1 + fi + case "$(uname -s)" in + Darwin) NDK_HOST="darwin-x86_64" ;; + Linux) NDK_HOST="linux-x86_64" ;; + esac + NDK_BIN="$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$NDK_HOST/bin" + NDK_SYSROOT="$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$NDK_HOST/sysroot" + + ANDROID_TARGETS=() + ANDROID_LINKERS=() + for ABI in "${ANDROID_ABIS[@]}"; do + case "$ABI" in + arm64-v8a) + ANDROID_TARGETS+=(aarch64-linux-android) + ANDROID_LINKERS+=("$NDK_BIN/aarch64-linux-android24-clang") + # QuickJS is compiled C. Configure cc-rs and libclang for the same + # target/API as Rust; the host SDK cannot supply Android headers or + # ABI layouts. + export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$NDK_BIN/aarch64-linux-android24-clang" + export CC_aarch64_linux_android="$CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER" + export AR_aarch64_linux_android="$NDK_BIN/llvm-ar" + export BINDGEN_EXTRA_CLANG_ARGS_aarch64_linux_android="--sysroot=\"$NDK_SYSROOT\" --target=aarch64-linux-android24" + ;; + armeabi-v7a) + ANDROID_TARGETS+=(armv7-linux-androideabi) + ANDROID_LINKERS+=("$NDK_BIN/armv7a-linux-androideabi24-clang") + export CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="$NDK_BIN/armv7a-linux-androideabi24-clang" + export CC_armv7_linux_androideabi="$CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER" + export AR_armv7_linux_androideabi="$NDK_BIN/llvm-ar" + export BINDGEN_EXTRA_CLANG_ARGS_armv7_linux_androideabi="--sysroot=\"$NDK_SYSROOT\" --target=armv7a-linux-androideabi24" + ;; + esac + done + + for linker in "${ANDROID_LINKERS[@]}"; do + if [[ ! -x "$linker" ]]; then + echo "Error: Android API 24 linker was not found: $linker" >&2 + exit 1 + fi + done + + rustup target add "${ANDROID_TARGETS[@]}" + # Align ELF segments for Android devices using 16 KB pages. + export RUSTFLAGS="${RUSTFLAGS:-} -C link-arg=-Wl,-z,max-page-size=16384" + for target in "${ANDROID_TARGETS[@]}"; do + cargo build --locked --release -p spotiflac-mobile --target "$target" + case "$target" in + aarch64-linux-android) ABI="arm64-v8a" ;; + armv7-linux-androideabi) ABI="armeabi-v7a" ;; + esac + mkdir -p "$CARGO_TARGET_DIR/android/jniLibs/$ABI" + cp "$CARGO_TARGET_DIR/$target/release/libspotiflac_mobile.so" \ + "$CARGO_TARGET_DIR/android/jniLibs/$ABI/" + done +elif [[ "$PLATFORM" == "ios" ]]; then + if [[ "$(uname -s)" != "Darwin" ]]; then + echo "Error: iOS builds require macOS and Xcode." >&2 + exit 1 + fi + rustup target add aarch64-apple-ios aarch64-apple-ios-sim x86_64-apple-ios + export IPHONEOS_DEPLOYMENT_TARGET=16.0 + export BINDGEN_EXTRA_CLANG_ARGS_aarch64_apple_ios="-isysroot \"$(xcrun --sdk iphoneos --show-sdk-path)\" --target=arm64-apple-ios16.0" + export BINDGEN_EXTRA_CLANG_ARGS_aarch64_apple_ios_sim="-isysroot \"$(xcrun --sdk iphonesimulator --show-sdk-path)\" --target=arm64-apple-ios16.0-simulator" + export BINDGEN_EXTRA_CLANG_ARGS_x86_64_apple_ios="-isysroot \"$(xcrun --sdk iphonesimulator --show-sdk-path)\" --target=x86_64-apple-ios16.0-simulator" + for target in aarch64-apple-ios aarch64-apple-ios-sim x86_64-apple-ios; do + cargo build --locked --release -p spotiflac-mobile --target "$target" + done + HEADERS_DIR="$CARGO_TARGET_DIR/ios/headers" + FRAMEWORK_DIR="$CARGO_TARGET_DIR/ios/SpotiFLACBackendFFI.xcframework" + SIMULATOR_LIBRARY="$CARGO_TARGET_DIR/ios/simulator/libspotiflac_mobile.a" + mkdir -p "$HEADERS_DIR" "$(dirname "$SIMULATOR_LIBRARY")" + cp "$CARGO_TARGET_DIR/bindings/swift/SpotiFLACBackendFFI.h" "$HEADERS_DIR/" + cp "$CARGO_TARGET_DIR/bindings/swift/SpotiFLACBackendFFI.modulemap" "$HEADERS_DIR/module.modulemap" + # Flutter/CocoaPods may request both simulator architectures, even on an + # ARM64 Mac. Supply one universal simulator slice. + xcrun lipo -create \ + "$CARGO_TARGET_DIR/aarch64-apple-ios-sim/release/libspotiflac_mobile.a" \ + "$CARGO_TARGET_DIR/x86_64-apple-ios/release/libspotiflac_mobile.a" \ + -output "$SIMULATOR_LIBRARY" + # xcodebuild refuses to overwrite an existing generated XCFramework. + if [[ -d "$FRAMEWORK_DIR" ]]; then + rm -rf "$FRAMEWORK_DIR" + fi + xcodebuild -create-xcframework \ + -library "$CARGO_TARGET_DIR/aarch64-apple-ios/release/libspotiflac_mobile.a" -headers "$HEADERS_DIR" \ + -library "$SIMULATOR_LIBRARY" -headers "$HEADERS_DIR" \ + -output "$FRAMEWORK_DIR" + # CocoaPods does not traverse nested symlinks, including a relocated target/. + # Stage one pod root that Podfile can resolve before scanning its files. + cp "$CARGO_TARGET_DIR/bindings/swift/SpotiFLACBackend.swift" "$CARGO_TARGET_DIR/ios/" + cp "$RUST_BACKEND_DIR/SpotiFLACBackend.podspec" "$CARGO_TARGET_DIR/ios/" + cp "$PROJECT_DIR/LICENSE" "$CARGO_TARGET_DIR/ios/" +fi + +echo "Rust $PLATFORM artifacts are staged in $CARGO_TARGET_DIR." diff --git a/scripts/check_backend_apk.py b/scripts/check_backend_apk.py new file mode 100644 index 00000000..181888ac --- /dev/null +++ b/scripts/check_backend_apk.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Check that a release APK contains the selected native backend.""" + +import argparse +import hashlib +import struct +import sys +import zipfile +from pathlib import Path +from typing import Iterable, Sequence, Tuple + + +ABI_LAYOUT = { + "arm64-v8a": (2, 183), + "armeabi-v7a": (1, 40), +} +CORE_LIBRARIES = ("libapp.so", "libflutter.so") + + +class AuditError(Exception): + pass + + +def parse_abis(raw: str) -> Tuple[str, ...]: + abis = tuple(part.strip() for part in raw.split(",") if part.strip()) + if not abis: + raise AuditError("--abis must contain at least one ABI") + unknown = sorted(set(abis) - set(ABI_LAYOUT)) + if unknown: + raise AuditError("unsupported ABI(s): " + ", ".join(unknown)) + if len(set(abis)) != len(abis): + raise AuditError("--abis contains a duplicate ABI") + return abis + + +def artifact_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def required_entry(infos: Sequence[zipfile.ZipInfo], path: str) -> zipfile.ZipInfo: + matches = [info for info in infos if info.filename == path and not info.is_dir()] + if not matches: + raise AuditError("missing APK entry: " + path) + if len(matches) > 1: + raise AuditError("duplicate APK entry: " + path) + return matches[0] + + +def check_abi_directories(names: Iterable[str], expected: Sequence[str]) -> None: + actual = set() + for name in names: + parts = name.split("/") + if len(parts) >= 3 and parts[0] == "lib" and parts[1]: + actual.add(parts[1]) + unexpected = sorted(actual - set(expected)) + if unexpected: + raise AuditError("unexpected lib ABI directory(s): " + ", ".join(unexpected)) + + +def check_elf(data: bytes, path: str, abi: str) -> None: + expected_class, expected_machine = ABI_LAYOUT[abi] + if len(data) < 20 or data[:4] != b"\x7fELF": + raise AuditError(path + ": invalid ELF header") + if data[4] != expected_class: + raise AuditError(f"{path}: ELF class {data[4]} does not match {abi}") + if data[5] != 1: + raise AuditError(path + ": Android ARM libraries must be little-endian") + machine = struct.unpack_from("<H", data, 18)[0] + if machine != expected_machine: + raise AuditError(f"{path}: e_machine {machine} does not match {abi}") + + +def check_backend_markers( + zf: zipfile.ZipFile, + infos: Sequence[zipfile.ZipInfo], + backend: str, +) -> None: + names = [info.filename for info in infos] + if backend == "rust": + if any(name.rsplit("/", 1)[-1] == "libgojni.so" for name in names): + raise AuditError("Rust APK contains forbidden libgojni.so") + go_resource = next( + (name for name in names if name.startswith(("gobackend/", "go/"))), None + ) + if go_resource is not None: + raise AuditError(f"Rust APK contains forbidden Go resource: {go_resource}") + for info in infos: + if info.is_dir() or not info.filename.endswith(".dex"): + continue + data = zf.read(info) + marker = next( + (marker for marker in (b"Lgobackend/", b"Lgo/") if marker in data), None + ) + if marker is not None: + raise AuditError(f"Rust APK DEX {info.filename} contains {marker.decode()}") + elif any( + name.rsplit("/", 1)[-1] == "libspotiflac_mobile.so" for name in names + ): + raise AuditError("Go APK contains forbidden libspotiflac_mobile.so") + + +def audit(path: Path, backend: str, abis: Sequence[str]) -> str: + if not path.is_file(): + raise AuditError("APK is not a regular file: " + str(path)) + digest = artifact_sha256(path) + try: + with zipfile.ZipFile(path, "r") as zf: + infos = zf.infolist() + names = [info.filename for info in infos] + if "assets/flutter_assets/kernel_blob.bin" in names: + raise AuditError("debug APK contains assets/flutter_assets/kernel_blob.bin") + check_abi_directories(names, abis) + check_backend_markers(zf, infos, backend) + backend_library = ( + "libspotiflac_mobile.so" if backend == "rust" else "libgojni.so" + ) + libraries = CORE_LIBRARIES + (backend_library,) + if backend == "rust": + libraries += ("libjnidispatch.so",) + for abi in abis: + for library in libraries: + entry_path = f"lib/{abi}/{library}" + entry = required_entry(infos, entry_path) + with zf.open(entry, "r") as stream: + check_elf(stream.read(20), entry_path, abi) + except zipfile.BadZipFile as exc: + raise AuditError("invalid APK/ZIP: " + str(exc)) from exc + return digest + + +def make_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("apk", type=Path, help="release APK to audit") + parser.add_argument("--backend", choices=("rust", "go"), required=True) + parser.add_argument("--abis", required=True, metavar="ABI[,ABI...]", help="expected APK ABIs") + return parser + + +def main(argv: Sequence[str] = None) -> int: + args = make_parser().parse_args(argv) + try: + abis = parse_abis(args.abis) + digest = audit(args.apk, args.backend, abis) + except (AuditError, OSError) as exc: + print("error: " + str(exc), file=sys.stderr) + return 1 + print(f"OK sha256={digest} backend={args.backend} abis={','.join(abis)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_backend_ios.py b/scripts/check_backend_ios.py new file mode 100644 index 00000000..5ac9f4ff --- /dev/null +++ b/scripts/check_backend_ios.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Audit an iOS application bundle for the selected native backend.""" + +import argparse +import hashlib +import plistlib +import re +import subprocess +import sys +from pathlib import Path +from typing import Dict, List, Sequence, Set, Tuple + + +MACHO_MAGICS = { + b"\xfe\xed\xfa\xce", + b"\xce\xfa\xed\xfe", + b"\xfe\xed\xfa\xcf", + b"\xcf\xfa\xed\xfe", + b"\xca\xfe\xba\xbe", + b"\xbe\xba\xfe\xca", + b"\xca\xfe\xba\xbf", + b"\xbf\xba\xfe\xca", +} +VALID_ARCHES = {"arm64", "x86_64"} +PLATFORM_NAMES = {"ios": "IOS", "ios-simulator": "IOSSIMULATOR"} +GO_SECTIONS = ("__gopclntab", "__go_buildinfo") +GO_SYMBOLS = ("_GobackendSetAppVersion", "_crosscall2") +RUST_SYMBOL = "_uniffi_spotiflac_mobile_" + + +class AuditError(Exception): + pass + + +def parse_archs(raw: str) -> Tuple[str, ...]: + archs = tuple(part.strip() for part in raw.split(",")) + if not archs or any(not arch for arch in archs): + raise AuditError("--archs must be a comma-separated list") + if len(set(archs)) != len(archs): + raise AuditError("--archs contains a duplicate architecture") + unknown = sorted(set(archs) - VALID_ARCHES) + if unknown: + raise AuditError("unsupported architecture(s): " + ", ".join(unknown)) + return archs + + +def run_xcrun(*args: str) -> str: + try: + completed = subprocess.run( + ["xcrun", *args], + capture_output=True, + check=False, + errors="replace", + text=True, + timeout=30, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise AuditError("xcrun " + " ".join(args) + " failed: " + str(exc)) from exc + if completed.returncode: + detail = " ".join(completed.stderr.split())[:240] + raise AuditError( + "xcrun " + " ".join(args) + " failed" + (": " + detail if detail else "") + ) + return completed.stdout + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def read_main_executable(app: Path) -> Path: + plist_path = app / "Info.plist" + if not plist_path.is_file(): + raise AuditError("missing Info.plist") + try: + with plist_path.open("rb") as stream: + info = plistlib.load(stream) + except (OSError, ValueError) as exc: + raise AuditError("invalid Info.plist: " + str(exc)) from exc + name = info.get("CFBundleExecutable") if isinstance(info, dict) else None + if not isinstance(name, str) or not name or name in (".", "..") or any( + char in name for char in ("\x00", "/", "\\") + ): + raise AuditError("CFBundleExecutable must be a simple filename") + executable = app / name + if executable.is_symlink() or not executable.is_file(): + raise AuditError("missing main executable: " + name) + return executable + + +def check_main_arches(executable: Path, expected: Sequence[str], platform: str) -> None: + actual = tuple(run_xcrun("lipo", "-archs", str(executable)).split()) + if set(actual) != set(expected) or len(actual) != len(expected): + raise AuditError("main executable architectures are " + ",".join(actual) + + "; expected " + ",".join(expected)) + + output = run_xcrun("vtool", "-show-build", str(executable)) + architecture = re.compile(r"\(architecture\s+([^)]*)\):") + platform_line = re.compile(r"^\s*platform\s+(\S+)") + observed: Dict[str, Set[str]] = {} + current = None + for line in output.splitlines(): + match = architecture.search(line) + if match: + current = match.group(1).strip() + match = platform_line.match(line) + if match: + if current is None and len(expected) == 1: + current = expected[0] + if current in expected: + observed.setdefault(current, set()).add(match.group(1)) + wanted = PLATFORM_NAMES[platform] + for arch in expected: + values = observed.get(arch, set()) + if values != {wanted}: + found = ",".join(sorted(values)) or "missing" + raise AuditError( + f"{arch} main slice platform is {found}; expected {wanted}" + ) + + +def is_macho(path: Path) -> bool: + try: + with path.open("rb") as stream: + return stream.read(4) in MACHO_MAGICS + except OSError as exc: + raise AuditError("cannot read bundle file " + str(path)) from exc + + +def defined_symbol(nm_output: str, marker: str) -> bool: + for line in nm_output.splitlines(): + fields = line.split() + if (len(fields) == 3 and re.fullmatch(r"[0-9a-fA-F]+", fields[0]) + and marker in fields[2] and fields[1] not in {"U", "u"}): + return True + return False + + +def audit(app: Path, backend: str, archs: Sequence[str], platform: str, release: bool) -> str: + if not app.is_dir(): + raise AuditError(".app is not a directory: " + str(app)) + executable = read_main_executable(app) + if not is_macho(executable): + raise AuditError("main executable is not a Mach-O file") + check_main_arches(executable, archs, platform) + digest = sha256(executable) + try: + paths = sorted(app.rglob("*"), key=lambda path: path.as_posix()) + except OSError as exc: + raise AuditError("cannot enumerate app bundle: " + str(exc)) from exc + + macho: List[Tuple[str, str, str]] = [] + rust_framework_path = False + for path in paths: + relative = path.relative_to(app) + parts = relative.parts + if "Gobackend.framework" in parts and backend == "rust": + raise AuditError("Rust app contains Gobackend.framework: " + relative.as_posix()) + rust_framework_path |= any( + part == "SpotiFLACBackend" or part.startswith("SpotiFLACBackend.") + or part == "SpotiFLACBackendFFI" or part.startswith("SpotiFLACBackendFFI.") + for part in parts + ) + if release and path.name in {"kernel_blob.bin", "Runner.debug.dylib"}: + raise AuditError("release app contains forbidden " + path.name) + if path.is_symlink() or not path.is_file() or not is_macho(path): + continue + otool = run_xcrun("otool", "-l", str(path)) + nm = run_xcrun("nm", "-gU", str(path)) + macho.append((relative.as_posix(), otool, nm)) + + go_section = any(section in otool for _, otool, _ in macho for section in GO_SECTIONS) + go_symbol = any( + defined_symbol(nm, symbol) for _, _, nm in macho for symbol in GO_SYMBOLS + ) + rust_symbol = any(RUST_SYMBOL in nm for _, _, nm in macho) + rust_defined_symbol = any(defined_symbol(nm, RUST_SYMBOL) for _, _, nm in macho) + if backend == "rust": + if not rust_defined_symbol: + raise AuditError("Rust app has no defined " + RUST_SYMBOL + " symbol") + if go_section or go_symbol: + raise AuditError("Rust app contains Go sections or defined Go symbols") + else: + if rust_symbol or rust_framework_path: + raise AuditError("Go app contains Rust symbols or framework paths") + if not (go_section or go_symbol): + raise AuditError("Go app has no Go section or defined symbol evidence") + return digest + + +def make_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("app", type=Path, help=".app directory to audit") + parser.add_argument("--backend", choices=("rust", "go"), required=True) + parser.add_argument("--archs", required=True, metavar="ARCH[,ARCH...]") + parser.add_argument("--platform", choices=tuple(PLATFORM_NAMES), required=True) + parser.add_argument("--release", action="store_true", help="apply release artifact checks") + return parser + + +def main(argv: Sequence[str] = None) -> int: + args = make_parser().parse_args(argv) + try: + archs = parse_archs(args.archs) + digest = audit(args.app, args.backend, archs, args.platform, args.release) + except (AuditError, OSError) as exc: + print("error: " + str(exc), file=sys.stderr) + return 1 + print(f"OK sha256={digest} backend={args.backend} archs={','.join(archs)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_rust_backend.sh b/scripts/check_rust_backend.sh new file mode 100644 index 00000000..d50c91c9 --- /dev/null +++ b/scripts/check_rust_backend.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Check the production Rust workspace without local migration archives. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: check_rust_backend.sh + + Run Rust formatting, Clippy, and unit tests. + --help Show this help text. +EOF +} + +if (( $# > 1 )); then + echo "Error: unexpected arguments: $*" >&2 + usage >&2 + exit 2 +fi +if (( $# == 1 )); then + case "$1" in + --help|-h) + usage + exit 0 + ;; + *) + echo "Error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +cd "$PROJECT_DIR/rust_backend" +# Exercise the same runtime-detected AES backend as the release build. +export RUSTFLAGS="${RUSTFLAGS:-} --cfg aes_armv8" +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo test --workspace --locked diff --git a/test/extension_initialization_test.dart b/test/extension_initialization_test.dart new file mode 100644 index 00000000..e20145e4 --- /dev/null +++ b/test/extension_initialization_test.dart @@ -0,0 +1,528 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/services.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:spotiflac_android/models/settings.dart'; +import 'package:spotiflac_android/providers/extension_provider.dart'; +import 'package:spotiflac_android/providers/repo_provider.dart'; +import 'package:spotiflac_android/providers/settings_provider.dart'; +import 'package:spotiflac_android/services/extension_storage_service.dart'; +import 'package:spotiflac_android/services/platform_bridge.dart'; + +class _DelayedSettingsNotifier extends SettingsNotifier { + final _ready = Completer<void>(); + int _loads = 0; + + @override + AppSettings build() => const AppSettings(); + + @override + Future<void> ensureLoaded() async { + _loads++; + await _ready.future; + } + + void complete() { + if (!_ready.isCompleted) _ready.complete(); + } +} + +class _RetryingSettingsNotifier extends SettingsNotifier { + int _syncAttempts = 0; + + @override + AppSettings build() => const AppSettings(); + + @override + Future<void> syncLyricsSettingsToBackend({AppSettings? settings}) async { + _syncAttempts++; + if (_syncAttempts == 1) { + throw StateError('Settings load failed'); + } + } +} + +class _FailingSettingsSyncNotifier extends SettingsNotifier { + int _syncs = 0; + + @override + Future<void> syncLyricsSettingsToBackend({AppSettings? settings}) async { + if (++_syncs == 1) throw StateError('Settings sync unavailable'); + await super.syncLyricsSettingsToBackend(settings: settings); + } +} + +class _DelayedExtensionNotifier extends ExtensionNotifier { + final _started = Completer<void>(); + final _ready = Completer<void>(); + int _initializations = 0; + bool _failInitialization = false; + + @override + Future<void> initialize( + String extensionsDir, + String dataDir, { + required String masterKey, + }) async { + _initializations++; + expect(await Directory(extensionsDir).exists(), isTrue); + expect(await Directory(dataDir).exists(), isTrue); + expect(base64Decode(masterKey), hasLength(32)); + if (!_started.isCompleted) _started.complete(); + await _ready.future; + state = state.copyWith( + isInitialized: !_failInitialization, + error: _failInitialization ? 'Native initialization failed' : null, + ); + } + + @override + Future<void> refreshExtensions() async {} +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + const paths = MethodChannel('plugins.flutter.io/path_provider'); + const secure = MethodChannel('plugins.it_nomads.com/flutter_secure_storage'); + const backend = MethodChannel('com.zarz.spotiflac/backend'); + late Directory root; + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + + setUpAll(() async { + root = await Directory.systemTemp.createTemp('extension-initialization-'); + messenger.setMockMethodCallHandler(paths, (call) async { + return call.method == 'getApplicationSupportDirectory' + ? '${root.path}/support' + : '${root.path}/documents'; + }); + messenger.setMockMethodCallHandler(secure, (call) async { + if (call.method == 'read') return base64Encode(List.filled(32, 1)); + return null; + }); + }); + tearDownAll(() async { + messenger.setMockMethodCallHandler(paths, null); + messenger.setMockMethodCallHandler(secure, null); + messenger.setMockMethodCallHandler(backend, null); + await root.delete(recursive: true); + }); + + test( + 'storage preparation retries a transient failure and coalesces callers', + () async { + var supportReads = 0; + messenger.setMockMethodCallHandler(paths, (call) async { + if (call.method == 'getApplicationSupportDirectory') { + supportReads++; + if (supportReads == 1) { + throw PlatformException(code: 'temporarily_unavailable'); + } + return '${root.path}/support'; + } + return '${root.path}/documents'; + }); + await expectLater( + ExtensionStorageService.prepare(), + throwsA(isA<PlatformException>()), + ); + final first = ExtensionStorageService.prepare(); + final second = ExtensionStorageService.prepare(); + expect(identical(first, second), isTrue); + final storage = await first; + expect(supportReads, 2); + expect(storage.extensionsDir, '${root.path}/support/extensions'); + expect(storage.dataDir, '${root.path}/support/extension_data'); + }, + ); + + test( + 'delayed settings load gates startup, install, upgrade, and repository init', + () async { + SharedPreferences.setMockInitialValues({}); + final settings = _DelayedSettingsNotifier(); + final notifier = _DelayedExtensionNotifier(); + final container = ProviderContainer( + overrides: [ + extensionProvider.overrideWith(() => notifier), + settingsProvider.overrideWith(() => settings), + ], + ); + addTearDown(container.dispose); + container.read(extensionProvider); + final calls = <String>[]; + messenger.setMockMethodCallHandler(backend, (call) async { + calls.add(call.method); + return { + 'name': 'example-extension', + 'display_name': 'Example', + 'version': '1.0.0', + }; + }); + + final startup = notifier.ensureInitialized(); + final install = notifier.installExtension( + '/cache/example-extension.sflx', + ); + final upgrade = notifier.upgradeExtension( + '/cache/example-extension.sflx', + ); + final repository = container + .read(repoProvider.notifier) + .initialize('/cache/store'); + await Future<void>.delayed(Duration.zero); + + expect(settings._loads, 1); + expect(notifier._initializations, 0); + expect(notifier._started.isCompleted, isFalse); + expect(calls, isEmpty); + + settings.complete(); + await notifier._started.future; + expect(notifier._initializations, 1); + expect(calls, isEmpty); + + notifier._ready.complete(); + await startup; + expect(await install, isTrue); + expect(await upgrade, isTrue); + await repository; + expect(container.read(repoProvider).isInitialized, isTrue); + expect(settings._loads, 1); + expect(notifier._initializations, 1); + expect( + calls, + containsAll([ + 'loadExtensionFromPath', + 'upgradeExtension', + 'initExtensionRepo', + ]), + ); + }, + ); + + test('startup, install and upgrade wait for one initialization', () async { + SharedPreferences.setMockInitialValues({}); + final notifier = _DelayedExtensionNotifier(); + final container = ProviderContainer( + overrides: [extensionProvider.overrideWith(() => notifier)], + ); + addTearDown(container.dispose); + container.read(extensionProvider); + final calls = <String>[]; + messenger.setMockMethodCallHandler(backend, (call) async { + if (call.method == 'setLoggingEnabled') return null; + expect(container.read(extensionProvider).isInitialized, isTrue); + calls.add(call.method); + return { + 'name': 'example-extension', + 'display_name': 'Example', + 'version': '1.0.0', + }; + }); + + final startup = notifier.ensureInitialized(); + await notifier._started.future; + final install = notifier.installExtension('/cache/example-extension.sflx'); + final batch = notifier.installExtensions(['/cache/example-b.sflx']); + final check = notifier.checkExtensionUpgrade( + '/cache/example-extension.sflx', + ); + final upgrade = notifier.upgradeExtension('/cache/example-extension.sflx'); + await Future<void>.delayed(Duration.zero); + expect(calls, isEmpty); + notifier._ready.complete(); + await startup; + expect(await install, isTrue); + expect((await batch).installed, 1); + expect((await check)['error'], isNull); + expect(await upgrade, isTrue); + expect(notifier._initializations, 1); + expect( + calls, + containsAll([ + 'loadExtensionFromPath', + 'checkExtensionUpgrade', + 'upgradeExtension', + ]), + ); + }); + + test( + 'failed initialization prevents installation and can be retried', + () async { + SharedPreferences.setMockInitialValues({}); + final notifier = _DelayedExtensionNotifier().._failInitialization = true; + notifier._ready.complete(); + final container = ProviderContainer( + overrides: [extensionProvider.overrideWith(() => notifier)], + ); + addTearDown(container.dispose); + container.read(extensionProvider); + var installs = 0; + messenger.setMockMethodCallHandler(backend, (call) async { + if (call.method == 'setLoggingEnabled') return null; + expect(call.method, 'loadExtensionFromPath'); + installs++; + return {'name': 'example-extension'}; + }); + expect(await notifier.installExtension('/cache/example.sflx'), isFalse); + expect(installs, 0); + expect( + container.read(extensionProvider).error, + contains('Native initialization failed'), + ); + notifier._failInitialization = false; + expect(await notifier.installExtension('/cache/example.sflx'), isTrue); + expect(installs, 1); + expect(notifier._initializations, 2); + }, + ); + + test('repository waits for a successful owner and retries failure', () async { + SharedPreferences.setMockInitialValues({}); + final notifier = _DelayedExtensionNotifier().._failInitialization = true; + notifier._ready.complete(); + final container = ProviderContainer( + overrides: [extensionProvider.overrideWith(() => notifier)], + ); + addTearDown(container.dispose); + var repositoryInitializations = 0; + messenger.setMockMethodCallHandler(backend, (call) async { + if (call.method == 'setLoggingEnabled') return null; + expect(call.method, 'initExtensionRepo'); + expect(container.read(extensionProvider).isInitialized, isTrue); + repositoryInitializations++; + return null; + }); + final repository = container.read(repoProvider.notifier); + await repository.initialize('/cache/store'); + expect(repositoryInitializations, 0); + expect(container.read(repoProvider).isInitialized, isFalse); + expect( + container.read(repoProvider).error, + contains('Native initialization failed'), + ); + notifier._failInitialization = false; + await repository.initialize('/cache/store'); + expect(repositoryInitializations, 1); + expect(container.read(repoProvider).isInitialized, isTrue); + expect(container.read(repoProvider).error, isNull); + }); + + for (final failureMethod in const [ + 'loadExtensionsFromDir', + 'getInstalledExtensions', + ]) { + test('extension refresh failure remains visible and can be retried ' + '($failureMethod)', () async { + SharedPreferences.setMockInitialValues({}); + final container = ProviderContainer(); + addTearDown(container.dispose); + final notifier = container.read(extensionProvider.notifier); + final extensionsDir = Directory('${root.path}/production-$failureMethod'); + await extensionsDir.create(recursive: true); + + var targetAttempts = 0; + var loadAttempts = 0; + var refreshAttempts = 0; + final expectedError = failureMethod == 'loadExtensionsFromDir' + ? 'transient_load' + : 'transient_refresh'; + messenger.setMockMethodCallHandler(backend, (call) async { + switch (call.method) { + case 'loadExtensionsFromDir': + loadAttempts++; + if (failureMethod == call.method && ++targetAttempts == 1) { + throw PlatformException(code: 'transient_load'); + } + return {'loaded': <String>[], 'errors': <Map<String, dynamic>>[]}; + case 'getInstalledExtensions': + refreshAttempts++; + if (failureMethod == call.method && ++targetAttempts == 1) { + throw PlatformException(code: 'transient_refresh'); + } + return <Map<String, dynamic>>[]; + default: + throw StateError('Unexpected backend method: ${call.method}'); + } + }); + + await notifier.loadExtensions(extensionsDir.path); + expect(notifier.state.isInitialized, isFalse); + expect(notifier.state.error, contains(expectedError)); + expect(targetAttempts, 1); + + await notifier.loadExtensions(extensionsDir.path); + expect(notifier.state.isInitialized, isFalse); + expect(notifier.state.error, isNull); + expect(loadAttempts, 2); + expect( + refreshAttempts, + failureMethod == 'getInstalledExtensions' ? 2 : 1, + ); + expect(targetAttempts, 2); + }); + } + + test( + 'settings load failure prevents initialization and a retry works', + () async { + SharedPreferences.setMockInitialValues({}); + final settings = _RetryingSettingsNotifier(); + final notifier = _DelayedExtensionNotifier().._ready.complete(); + final container = ProviderContainer( + overrides: [ + extensionProvider.overrideWith(() => notifier), + settingsProvider.overrideWith(() => settings), + ], + ); + addTearDown(container.dispose); + container.read(extensionProvider); + var installs = 0; + messenger.setMockMethodCallHandler(backend, (call) async { + if (call.method == 'loadExtensionFromPath') installs++; + return {'name': 'example-extension'}; + }); + + await expectLater( + notifier.ensureInitialized(), + throwsA( + isA<StateError>().having( + (error) => error.message, + 'message', + 'Settings load failed', + ), + ), + ); + expect(settings._syncAttempts, 1); + expect(notifier._initializations, 0); + expect(container.read(extensionProvider).isInitialized, isFalse); + expect(installs, 0); + + expect( + await notifier.installExtension('/cache/example-extension.sflx'), + isTrue, + ); + expect(settings._syncAttempts, 2); + expect(notifier._initializations, 1); + expect(installs, 1); + }, + ); + + test( + 'SettingsNotifier coalesces load and restores persisted lyrics settings', + () async { + const providers = [ + 'extension:example.provider.primary', + 'extension:example.provider.fallback', + ]; + const persisted = AppSettings( + lyricsProviders: providers, + lyricsIncludeTranslationNetease: true, + lyricsIncludeRomanizationNetease: true, + lyricsMultiPersonWordByWord: true, + lyricsAppleElrcWordSync: true, + musixmatchLanguage: 'id', + ); + SharedPreferences.setMockInitialValues({ + 'app_settings': jsonEncode(persisted.toJson()), + }); + final container = ProviderContainer(); + addTearDown(container.dispose); + + final notifier = container.read(settingsProvider.notifier); + final first = notifier.ensureLoaded(); + final second = notifier.ensureLoaded(); + expect(identical(first, second), isTrue); + + await first; + final loaded = container.read(settingsProvider); + expect(loaded.lyricsProviders, providers); + expect(loaded.lyricsIncludeTranslationNetease, isTrue); + expect(loaded.lyricsIncludeRomanizationNetease, isTrue); + expect(loaded.lyricsMultiPersonWordByWord, isTrue); + expect(loaded.lyricsAppleElrcWordSync, isTrue); + expect(loaded.musixmatchLanguage, 'id'); + expect(loaded.lyricsFetchOptions, { + 'include_translation_netease': true, + 'include_romanization_netease': true, + 'multi_person_word_by_word': true, + 'apple_elrc_word_sync': true, + 'musixmatch_language': 'id', + }); + }, + ); + + test('real settings loader retries a failed shared future', () async { + SharedPreferences.setMockInitialValues({}); + final notifier = _FailingSettingsSyncNotifier(); + final container = ProviderContainer( + overrides: [settingsProvider.overrideWith(() => notifier)], + ); + addTearDown(container.dispose); + container.read(settingsProvider); + final failed = notifier.ensureLoaded(); + expect(identical(failed, notifier.ensureLoaded()), isTrue); + await expectLater( + failed, + throwsA( + isA<StateError>().having( + (error) => error.message, + 'message', + 'Settings sync unavailable', + ), + ), + ); + final retry = notifier.ensureLoaded(); + expect(identical(failed, retry), isFalse); + expect(identical(retry, notifier.ensureLoaded()), isTrue); + await retry; + expect(notifier._syncs, 2); + expect(identical(retry, notifier.ensureLoaded()), isTrue); + }); + + test( + 'init channel encodes nondefault lyrics providers and options', + () async { + late Map<String, dynamic> arguments; + messenger.setMockMethodCallHandler(backend, (call) async { + expect(call.method, 'initExtensionSystem'); + arguments = Map<String, dynamic>.from(call.arguments as Map); + return null; + }); + const providers = [ + 'extension:example.provider.primary', + 'extension:example.provider.fallback', + ]; + const options = <String, dynamic>{ + 'include_translation_netease': true, + 'musixmatch_language': 'id', + }; + final masterKey = base64Encode(List.filled(32, 7)); + + await PlatformBridge.initExtensionSystem( + '/tmp/example-extensions', + '/tmp/example-extension-data', + masterKey: masterKey, + lyricsProviders: providers, + lyricsFetchOptions: options, + allowedDirectories: const ['/tmp/example-output'], + ); + + expect(arguments['extensions_dir'], '/tmp/example-extensions'); + expect(arguments['data_dir'], '/tmp/example-extension-data'); + expect(arguments['master_key'], masterKey); + expect(arguments['allowed_directories'], ['/tmp/example-output']); + expect( + jsonDecode(arguments['lyrics_providers_json'] as String), + providers, + ); + expect(jsonDecode(arguments['lyrics_options_json'] as String), options); + }, + ); +} diff --git a/test/ios_container_paths_test.dart b/test/ios_container_paths_test.dart new file mode 100644 index 00000000..c65ed373 --- /dev/null +++ b/test/ios_container_paths_test.dart @@ -0,0 +1,103 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/utils/ios_container_paths.dart'; + +void main() { + group('rebaseIosSandboxPath', () { + const oldContainer = + '/var/mobile/Containers/Data/Application/' + '11111111-1111-4111-8111-111111111111'; + const newDocuments = + '/var/mobile/Containers/Data/Application/' + '22222222-2222-4222-8222-222222222222/Documents'; + const newContainer = + '/var/mobile/Containers/Data/Application/' + '22222222-2222-4222-8222-222222222222'; + + test('rebases supported device roots and preserves suffixes', () { + for (final root in const [ + 'Documents', + 'Library/Application Support', + 'Library/Caches', + ]) { + for (final suffix in const [ + '', + '/Music/曲 name #track01.flac', + '/Music/曲 name.flac#track01', + ]) { + final oldPath = '$oldContainer/$root$suffix'; + expect( + rebaseIosSandboxPath(oldPath, newDocuments), + '$newContainer/$root$suffix', + ); + } + } + const privatePath = + '/private/var/mobile/Containers/Data/Application/' + '11111111-1111-4111-8111-111111111111/Documents/曲.flac'; + expect( + rebaseIosSandboxPath(privatePath, newDocuments), + '$newContainer/Documents/曲.flac', + ); + }); + + test('rebases simulator containers', () { + const oldPath = + '/Users/test/Library/Developer/CoreSimulator/Devices/' + '33333333-3333-4333-8333-333333333333/data/Containers/Data/' + 'Application/55555555-5555-4555-8555-555555555555/' + 'Documents/音楽 folder/song #track01.m4a'; + const newDocuments = + '/Users/test/Library/Developer/CoreSimulator/Devices/' + '44444444-4444-4444-8444-444444444444/data/Containers/Data/' + 'Application/66666666-6666-4666-8666-666666666666/' + 'Documents'; + final newContainer = newDocuments.substring( + 0, + newDocuments.length - '/Documents'.length, + ); + expect( + rebaseIosSandboxPath(oldPath, newDocuments), + '$newContainer/Documents/音楽 folder/song #track01.m4a', + ); + }); + + test('is idempotent for the current container', () { + const currentPath = + '$newContainer/Documents/音楽 folder/song #track01.flac'; + expect(rebaseIosSandboxPath(currentPath, newDocuments), currentPath); + }); + + test('leaves roots, traversal, URIs, and external paths unchanged', () { + const unchanged = [ + oldContainer, + '$oldContainer/', + '$oldContainer/tmp/song.flac', + '$oldContainer/Documents/../Library/Caches/cover.jpg', + '$oldContainer/Documents-extra/song.flac', + '/outside$oldContainer/Documents/song.flac', + '/tmp/song.flac', + 'content://downloads/song.flac', + 'file:///var/mobile/Containers/Data/Application/OLD/song.flac', + '/data/user/0/com.example/files/song.flac', + '/private/var/mobile/Library/Mobile Documents/com~apple~CloudDocs/song.flac', + 'Documents/song.flac', + ]; + for (final path in unchanged) { + expect(rebaseIosSandboxPath(path, newDocuments), path); + } + }); + + test('requires a current iOS Documents directory', () { + const oldPath = '$oldContainer/Documents/song.flac'; + for (final documents in const [ + '', + '/Documents', + '/data/user/0/com.example/files/Documents', + '$newContainer/Documents/subfolder', + '$newContainer/Library/Application Support', + ]) { + expect(rebaseIosSandboxPath(oldPath, documents), oldPath); + } + }); + }); +} diff --git a/test/log_selection_test.dart b/test/log_selection_test.dart index fec7dc0a..ae1de4de 100644 --- a/test/log_selection_test.dart +++ b/test/log_selection_test.dart @@ -27,7 +27,7 @@ void main() { expect( formatLogEntries(entries), '[10:20:30.040] [INFO] [Search] first\n' - '[10:20:31.050] [ERROR] [Go] [Download] second | failed', + '[10:20:31.050] [ERROR] [Native] [Download] second | failed', ); }); diff --git a/test/models_and_utils_test.dart b/test/models_and_utils_test.dart index 83ac3bb6..e9cf9690 100644 --- a/test/models_and_utils_test.dart +++ b/test/models_and_utils_test.dart @@ -298,16 +298,23 @@ void main() { ).firstMatch(appDelegateSource); expect(expirationHandler, isNotNull); expect( - expirationHandler! + expirationHandler!.group(0)!, + contains('self?.coreBackend.cancelActiveDownloads()'), + ); + expect( + expirationHandler .group(0)! - .indexOf('GobackendCancelAllActiveDownloads()'), + .indexOf('self?.coreBackend.cancelActiveDownloads()'), lessThan( expirationHandler.group(0)!.indexOf('iosBackgroundDownloadExpired'), ), ); expect( - queueProviderSource, - contains('pauseQueue(persistAcrossRestarts: false)'), + RegExp( + r'pauseQueue\(\s*persistAcrossRestarts:\s*false,\s*' + r'nativeCancelledItemIds:\s*cancelledItemIds,\s*\)', + ).hasMatch(queueProviderSource), + true, ); expect(queueProviderSource, contains('_iosBackgroundExecutionExpired')); expect(queueProviderSource, contains('final requeueItemIds =')); @@ -366,13 +373,16 @@ void main() { () { expect( RegExp( - r'Gobackend\.waitForAllDownloadProgressDelta\(', + r'coreBackend\.openDownloadProgress\(', ).allMatches(workerSnapshotSource), hasLength(1), ); + expect(workerSnapshotSource, isNot(contains('Gobackend.'))); expect( - workerSnapshotSource, - isNot(contains('Gobackend.getAllDownloadProgress()')), + RegExp( + r'\.waitDelta\(sinceSeq, 5_000L\)', + ).allMatches(workerSnapshotSource), + hasLength(1), ); expect(workerSnapshotSource, contains('"item_deltas"')); expect( diff --git a/test/platform_bridge_post_processing_test.dart b/test/platform_bridge_post_processing_test.dart new file mode 100644 index 00000000..de03b62a --- /dev/null +++ b/test/platform_bridge_post_processing_test.dart @@ -0,0 +1,93 @@ +import 'dart:convert'; + +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/services/platform_bridge.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + const channel = MethodChannel('com.zarz.spotiflac/backend'); + + tearDown(() { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, null); + }); + + test( + 'native replacement is exposed to the download queue as file_path', + () async { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, (call) async { + expect(call.method, 'runPostProcessingV2'); + expect(call.arguments, { + 'input': jsonEncode({ + 'item_id': 'queue-item', + 'path': '/music/original.flac', + }), + 'metadata': jsonEncode({'title': 'Example'}), + }); + return jsonEncode({ + 'success': true, + 'new_file_path': '/music/converted.m4a', + }); + }); + final result = await PlatformBridge.runPostProcessingV2( + '/music/original.flac', + metadata: {'title': 'Example'}, + itemId: 'queue-item', + ); + expect(result['file_path'], '/music/converted.m4a'); + expect(result['new_file_path'], '/music/converted.m4a'); + }, + ); + + test( + 'SAF keeps the destination URI instead of exposing a staging path', + () async { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, (call) async { + expect(call.arguments, { + 'input': jsonEncode({'uri': 'content://example/document/track'}), + 'metadata': '', + }); + return jsonEncode({ + 'success': true, + 'new_file_path': '/cache/processed.flac', + 'new_file_uri': 'content://example/document/track', + }); + }); + final result = await PlatformBridge.runPostProcessingV2( + 'content://example/document/track', + ); + expect(result['file_path'], 'content://example/document/track'); + }, + ); + + test( + 'legacy results, no-op hooks and failures retain their contracts', + () async { + for (final response in <Map<String, dynamic>>[ + {'success': true, 'file_path': '/music/legacy.flac'}, + {'success': true, 'new_file_path': ''}, + {'success': false, 'error': 'processing failed'}, + ]) { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler( + channel, + (_) async => jsonEncode(response), + ); + final result = await PlatformBridge.runPostProcessingV2( + '/music/original.flac', + ); + if (response['success'] == true) { + expect( + result['file_path'], + response['file_path'] ?? '/music/original.flac', + ); + } else { + expect(result, response); + } + } + }, + ); +}