perf(extensions): streamline prepared download work

This commit is contained in:
zarzet
2026-08-29 23:47:21 +07:00
parent f21f47b2bd
commit bb1be4fdb4
8 changed files with 490 additions and 15 deletions
+39 -2
View File
@@ -28,6 +28,7 @@ func attemptExtensionDownload(
) (resp *DownloadResponse, cancelledOuter bool) {
req.DownloadProvider = strings.TrimSpace(providerLabel)
req.ProviderTrackID = strings.TrimSpace(trackID)
preparedContext = extensionPreparedDownloadContext(req, preparedContext)
outputPath := buildOutputPathForExtension(req, ext)
if shouldReuseExistingOutput(req, outputPath) {
result := DownloadResult{FilePath: outputPath}
@@ -217,6 +218,7 @@ func attemptVerifiedResumeBeforeMetadata(
req.QobuzID,
req.DurationMS,
req.ItemID,
extensionAvailabilityTrackContext(req),
)
if shouldAbortCancelledFallback(req.ItemID, err) {
return nil, true
@@ -296,6 +298,41 @@ func attemptVerifiedResumeBeforeMetadata(
return nil, false
}
func extensionAvailabilityTrackContext(req DownloadRequest) map[string]any {
return map[string]any{
"id": req.ProviderTrackID,
"name": req.TrackName,
"artists": req.ArtistName,
"album_name": req.AlbumName,
"album_artist": req.AlbumArtist,
"cover_url": req.CoverURL,
"release_date": req.ReleaseDate,
"track_number": req.TrackNumber,
"total_tracks": req.TotalTracks,
"disc_number": req.DiscNumber,
"total_discs": req.TotalDiscs,
"duration_ms": req.DurationMS,
"isrc": req.ISRC,
"genre": req.Genre,
"label": req.Label,
"copyright": req.Copyright,
"composer": req.Composer,
"comment": req.Comment,
"explicit": req.Explicit,
"album_type": req.AlbumType,
"upc": req.UPC,
}
}
func extensionPreparedDownloadContext(req DownloadRequest, prepared map[string]any) map[string]any {
merged := make(map[string]any, len(prepared)+1)
for key, value := range prepared {
merged[key] = value
}
merged["host_track"] = extensionAvailabilityTrackContext(req)
return merged
}
func buildOutputStorageFailureResponse(
providerID string,
err error,
@@ -407,7 +444,7 @@ func DownloadWithExtensionFallback(req DownloadRequest) (*DownloadResponse, erro
ext, err := extManager.GetExtension(req.Source)
if err == nil && ext.Enabled && ext.Error == "" && ext.Manifest.IsDownloadProvider() {
provider := newExtensionProviderWrapper(ext)
availability, availErr := provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID)
availability, availErr := provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID, extensionAvailabilityTrackContext(req))
if shouldAbortCancelledFallback(req.ItemID, availErr) {
return nil, ErrDownloadCancelled
}
@@ -631,7 +668,7 @@ func DownloadWithExtensionFallback(req DownloadRequest) (*DownloadResponse, erro
provider := newExtensionProviderWrapper(ext)
availability, err := provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID)
availability, err := provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID, extensionAvailabilityTrackContext(req))
if shouldAbortCancelledFallback(req.ItemID, err) {
return nil, ErrDownloadCancelled
}
@@ -103,3 +103,32 @@ func TestOverlaySourceExtensionTrackIdentityFillsMissingValues(t *testing.T) {
t.Fatalf("missing identity fields were not enriched: %#v", req)
}
}
func TestExtensionPreparedDownloadContextMergesHostMetadata(t *testing.T) {
providerContext := map[string]any{
"token": "opaque-provider-value",
"host_track": "provider-key-must-not-win",
}
req := DownloadRequest{
ProviderTrackID: "provider-track-1",
TrackName: "Song",
ArtistName: "Artist",
AlbumName: "Album",
ISRC: "ISRC123",
}
merged := extensionPreparedDownloadContext(req, providerContext)
if merged["token"] != "opaque-provider-value" {
t.Fatalf("provider context was not preserved: %#v", merged)
}
hostTrack, ok := merged["host_track"].(map[string]any)
if !ok {
t.Fatalf("host_track = %#v", merged["host_track"])
}
if hostTrack["id"] != req.ProviderTrackID || hostTrack["name"] != req.TrackName || hostTrack["isrc"] != req.ISRC {
t.Fatalf("host metadata was not propagated: %#v", hostTrack)
}
if providerContext["host_track"] != "provider-key-must-not-win" {
t.Fatalf("input provider context was mutated: %#v", providerContext)
}
}
+8 -6
View File
@@ -273,12 +273,14 @@ func (m *extensionManager) loadExtensionFromFileLocked(filePath string) (*loaded
}
var supportedRuntimeFeatures = map[string]int{
"signedSession": 3,
"sessionRefresh": 1,
"sessionGrant": 1,
"globalAction": 1,
"webviewAuth": 1,
"downloadSegments": 1,
"signedSession": 3,
"sessionRefresh": 1,
"sessionGrant": 1,
"globalAction": 1,
"webviewAuth": 1,
"downloadSegments": 1,
"patternedFileTransform": 1,
"preparedContext": 1,
}
// validateManifestGates enforces minAppVersion and requiredRuntimeFeatures
@@ -1,10 +1,65 @@
package gobackend
import (
"errors"
"os"
"path/filepath"
"testing"
"time"
)
func TestExtensionDownloadCancellationInterruptsBusyJavaScript(t *testing.T) {
ext := newTestLoadedExtension(t, ExtensionTypeDownloadProvider)
if err := os.WriteFile(filepath.Join(ext.SourceDir, "index.js"), []byte(`
registerExtension({
download: function() {
while (true) {}
}
});
`), 0600); err != nil {
t.Fatalf("write busy extension: %v", err)
}
provider := newExtensionProviderWrapper(ext)
const itemID = "busy-download-cancel"
outputPath := filepath.Join(t.TempDir(), "busy.flac")
done := make(chan error, 1)
go func() {
_, err := provider.DownloadPrepared(
"track",
"best",
outputPath,
itemID,
nil,
nil,
)
done <- err
}()
deadline := time.Now().Add(time.Second)
for {
downloadCancels.mu.Lock()
entry := downloadCancels.entries[itemID]
ready := entry != nil && entry.refs > 0
downloadCancels.mu.Unlock()
if ready {
break
}
if time.Now().After(deadline) {
t.Fatal("download cancellation context was not initialized")
}
time.Sleep(time.Millisecond)
}
cancelDownload(itemID)
select {
case err := <-done:
if !errors.Is(err, ErrDownloadCancelled) {
t.Fatalf("DownloadPrepared error = %v, want ErrDownloadCancelled", err)
}
case <-time.After(time.Second):
t.Fatal("busy extension did not stop after cancellation")
}
}
func TestExtensionProviderWrapperFullSurface(t *testing.T) {
ext := newTestLoadedExtension(t, ExtensionTypeMetadataProvider, ExtensionTypeDownloadProvider, ExtensionTypeLyricsProvider)
provider := newExtensionProviderWrapper(ext)
+15 -7
View File
@@ -58,6 +58,7 @@ type extCallOpts struct {
// any ProviderID stamping.
func callExtension[T any](p *extensionProviderWrapper, opts extCallOpts, parse func(perf *extensionCallPerf, result goja.Value) (T, error)) (T, error) {
var zero T
ctx := context.Background()
perf := newExtensionCallPerf(p.extension.ID, opts.perfName)
defer perf.finish()
@@ -73,14 +74,13 @@ func callExtension[T any](p *extensionProviderWrapper, opts extCallOpts, parse f
p.extension.runtime.setActiveDownloadItemID(opts.itemID)
defer p.extension.runtime.clearActiveDownloadItemID()
}
initDownloadCancel(opts.itemID)
ctx = initDownloadCancel(opts.itemID)
defer clearDownloadCancel(opts.itemID)
if isDownloadCancelled(opts.itemID) {
return zero, ErrDownloadCancelled
}
}
ctx := context.Background()
if opts.requestID != "" {
if p.extension.runtime != nil {
p.extension.runtime.setActiveRequestID(opts.requestID)
@@ -456,12 +456,13 @@ func (p *extensionProviderWrapper) EnrichTrackForItemID(track *ExtTrackMetadata,
}
perf.recordInit(time.Since(initStartedAt))
defer p.extension.VMMu.Unlock()
downloadCtx := context.Background()
if itemID != "" {
if p.extension.runtime != nil {
p.extension.runtime.setActiveDownloadItemID(itemID)
defer p.extension.runtime.clearActiveDownloadItemID()
}
initDownloadCancel(itemID)
downloadCtx = initDownloadCancel(itemID)
defer clearDownloadCancel(itemID)
if isDownloadCancelled(itemID) {
return track, ErrDownloadCancelled
@@ -469,7 +470,7 @@ func (p *extensionProviderWrapper) EnrichTrackForItemID(track *ExtTrackMetadata,
}
jsStartedAt := time.Now()
result, err := runGojaCallWithTimeoutAndRecover(p.vm, func() (goja.Value, error) {
result, err := runGojaCallWithTimeoutContextAndRecover(downloadCtx, p.vm, func() (goja.Value, error) {
return invokeExtensionMethod(p.vm, "enrichTrack", extensionTrackInput(track))
}, DefaultJSTimeout)
perf.recordJS(time.Since(jsStartedAt))
@@ -505,7 +506,7 @@ func (p *extensionProviderWrapper) EnrichTrackForItemID(track *ExtTrackMetadata,
return &enrichedTrack, nil
}
func (p *extensionProviderWrapper) CheckAvailabilityForItemID(isrc, trackName, artistName, spotifyID, deezerID, tidalID, qobuzID string, durationMS int, itemID string) (*ExtAvailabilityResult, error) {
func (p *extensionProviderWrapper) CheckAvailabilityForItemID(isrc, trackName, artistName, spotifyID, deezerID, tidalID, qobuzID string, durationMS int, itemID string, trackContexts ...map[string]any) (*ExtAvailabilityResult, error) {
if !p.extension.Manifest.IsDownloadProvider() {
return nil, fmt.Errorf("extension '%s' is not a download provider", p.extension.ID)
}
@@ -520,6 +521,9 @@ func (p *extensionProviderWrapper) CheckAvailabilityForItemID(isrc, trackName, a
"qobuz_id": qobuzID,
"duration_ms": durationMS,
}
if len(trackContexts) > 0 && len(trackContexts[0]) > 0 {
availabilityOptions["track"] = trackContexts[0]
}
return callExtension(p, extCallOpts{
perfName: "checkAvailability",
@@ -620,8 +624,9 @@ func (p *extensionProviderWrapper) DownloadPrepared(
runtime.setActiveDownloadItemID(itemID)
defer runtime.clearActiveDownloadItemID()
}
downloadCtx := context.Background()
if itemID != "" {
initDownloadCancel(itemID)
downloadCtx = initDownloadCancel(itemID)
defer clearDownloadCancel(itemID)
SetItemPreparing(itemID)
}
@@ -653,7 +658,7 @@ func (p *extensionProviderWrapper) DownloadPrepared(
if len(preparedContext) > 0 {
downloadOptions["preparedContext"] = preparedContext
}
result, err := runGojaCallWithTimeoutAndRecover(vm, func() (goja.Value, error) {
result, err := runGojaCallWithTimeoutContextAndRecover(downloadCtx, vm, func() (goja.Value, error) {
return invokeExtensionMethod(
vm,
"download",
@@ -670,6 +675,9 @@ func (p *extensionProviderWrapper) DownloadPrepared(
cleanupSafe = !IsRuntimeUnsafeError(err)
unsafeDone = runtimeCompletion(err)
if err != nil {
if itemID != "" && isDownloadCancelled(itemID) {
return nil, ErrDownloadCancelled
}
errMsg := err.Error()
errType := "script_error"
if IsTimeoutError(err) {
+1
View File
@@ -730,6 +730,7 @@ func (r *extensionRuntime) RegisterAPIs(vm *goja.Runtime) {
fileObj.Set("copy", r.fileCopy)
fileObj.Set("move", r.fileMove)
fileObj.Set("getSize", r.fileGetSize)
fileObj.Set("transformPatternedBlocks", r.fileTransformPatternedBlocks)
vm.Set("file", fileObj)
ffmpegObj := vm.NewObject()
@@ -184,6 +184,104 @@ func TestExtensionRuntime_BlockCipherCBCSupportsAES(t *testing.T) {
}
}
func TestExtensionRuntime_FileTransformPatternedBlocks(t *testing.T) {
vm := newBinaryTestRuntime(t, true)
result, err := vm.RunString(`
(function() {
var options = {
algorithm: "blowfish",
mode: "cbc",
key: "0123456789ABCDEFF0E1D2C3B4A59687",
keyEncoding: "hex",
iv: "0001020304050607",
ivEncoding: "hex",
inputEncoding: "hex",
outputEncoding: "hex",
padding: "none"
};
var plainSegments = [
"00112233445566778899aabbccddeeff",
"102132435465768798a9bacbdcedfe0f",
"2031425364758697a8b9cadbecfd0e1f",
"30415263748596a7b8c9daebfc0d1e2f"
];
var encrypted = "";
for (var i = 0; i < plainSegments.length; i++) {
if (i % 3 === 0) {
var enc = utils.encryptBlockCipher(plainSegments[i], options);
if (!enc.success) throw new Error(enc.error);
encrypted += enc.data;
} else {
encrypted += plainSegments[i];
}
}
var partialTail = "a1b2c3d4e5";
encrypted += partialTail;
var write = file.writeBytes("encrypted.bin", encrypted, {
encoding: "hex", truncate: true
});
if (!write.success) throw new Error(write.error);
var callbacks = 0;
var transformed = file.transformPatternedBlocks(
"encrypted.bin",
"decrypted.bin",
{
operation: "decrypt",
algorithm: "blowfish",
mode: "cbc",
key: options.key,
keyEncoding: "hex",
iv: options.iv,
ivEncoding: "hex",
padding: "none",
segmentSize: 16,
transformEvery: 3,
transformOffset: 0,
bufferSize: 32
},
function(processed, total) {
if (processed > total) throw new Error("invalid progress");
callbacks++;
}
);
if (!transformed.success) throw new Error(transformed.error);
var output = file.readBytes("decrypted.bin", {encoding: "hex"});
if (!output.success) throw new Error(output.error);
return JSON.stringify({
output: output.data,
expected: plainSegments.join("") + partialTail,
processed: transformed.bytes_processed,
segments: transformed.segments_processed,
transformedSegments: transformed.segments_transformed,
callbacks: callbacks
});
})()
`)
if err != nil {
t.Fatalf("patterned file transform failed: %v", err)
}
decoded := decodeJSONResult[struct {
Output string `json:"output"`
Expected string `json:"expected"`
Processed int64 `json:"processed"`
Segments int64 `json:"segments"`
TransformedSegments int64 `json:"transformedSegments"`
Callbacks int `json:"callbacks"`
}](t, result)
if decoded.Output != decoded.Expected {
t.Fatalf("output = %q, want %q", decoded.Output, decoded.Expected)
}
if decoded.Processed != 69 || decoded.Segments != 5 || decoded.TransformedSegments != 2 {
t.Fatalf("unexpected transform stats: %+v", decoded)
}
if decoded.Callbacks != 3 {
t.Fatalf("callbacks = %d, want 3 buffered updates", decoded.Callbacks)
}
}
func TestExtensionRuntime_BlockCipherCTRSupportsAES(t *testing.T) {
vm := newBinaryTestRuntime(t, false)
@@ -0,0 +1,245 @@
package gobackend
import (
"context"
"crypto/cipher"
"io"
"os"
"path/filepath"
"strings"
"github.com/dop251/goja"
)
const (
defaultPatternedTransformBufferSize = int64(1 << 20)
maxPatternedTransformBufferSize = int64(16 << 20)
maxPatternedTransformSegmentSize = int64(16 << 20)
)
// fileTransformPatternedBlocks streams one file into another while applying an
// independent block-cipher transform to selected fixed-size segments. It keeps
// provider-specific layout knowledge in the extension: the host only receives
// a generic period/offset declaration and cipher parameters.
//
// JS signature:
//
// file.transformPatternedBlocks(inputPath, outputPath, {
// operation: "decrypt", algorithm: "blowfish", mode: "cbc",
// key: "...", keyEncoding: "hex", iv: "...", ivEncoding: "hex",
// segmentSize: 2048, transformEvery: 3, transformOffset: 0,
// bufferSize: 1048576, transformPartial: false
// }, function(processedBytes, totalBytes) {})
func (r *extensionRuntime) fileTransformPatternedBlocks(call goja.FunctionCall) goja.Value {
if len(call.Arguments) < 3 {
return r.jsError("input path, output path, and options are required")
}
inputPath, err := r.validatePath(call.Arguments[0].String())
if err != nil {
return r.jsError("%s", err.Error())
}
outputPath, err := r.validatePath(call.Arguments[1].String())
if err != nil {
return r.jsError("%s", err.Error())
}
options := parseRuntimeOptionsArgument(call, 2)
parsedCipher, err := parseRuntimeBlockCipherOptions(options)
if err != nil {
return r.jsError("%s", err.Error())
}
if parsedCipher.Padding != "none" {
return r.jsError("patterned file transforms only support padding: none")
}
if parsedCipher.Mode != "cbc" && parsedCipher.Mode != "ctr" {
return r.jsError("unsupported block cipher mode: %s", parsedCipher.Mode)
}
operation := strings.ToLower(runtimeOptionString(options, "operation", "decrypt"))
if operation != "decrypt" && operation != "encrypt" {
return r.jsError("operation must be decrypt or encrypt")
}
segmentSize := runtimeOptionInt64(options, "segmentSize", 0)
transformEvery := runtimeOptionInt64(options, "transformEvery", 1)
transformOffset := runtimeOptionInt64(options, "transformOffset", 0)
bufferSize := runtimeOptionInt64(options, "bufferSize", defaultPatternedTransformBufferSize)
transformPartial := runtimeOptionBool(options, "transformPartial", false)
if segmentSize <= 0 || segmentSize > maxPatternedTransformSegmentSize {
return r.jsError("segmentSize must be between 1 and %d bytes", maxPatternedTransformSegmentSize)
}
if transformEvery <= 0 {
return r.jsError("transformEvery must be greater than zero")
}
if transformOffset < 0 || transformOffset >= transformEvery {
return r.jsError("transformOffset must be between 0 and transformEvery - 1")
}
if bufferSize < segmentSize {
bufferSize = segmentSize
}
if bufferSize > maxPatternedTransformBufferSize {
bufferSize = maxPatternedTransformBufferSize
}
bufferSize -= bufferSize % segmentSize
if bufferSize == 0 {
bufferSize = segmentSize
}
block, err := newRuntimeBlockCipher(parsedCipher)
if err != nil {
return r.jsError("%s", err.Error())
}
if len(parsedCipher.IV) != block.BlockSize() {
return r.jsError("iv must be %d bytes for %s", block.BlockSize(), parsedCipher.Algorithm)
}
if parsedCipher.Mode == "cbc" && segmentSize%int64(block.BlockSize()) != 0 {
return r.jsError("segmentSize must be a multiple of %d bytes for CBC", block.BlockSize())
}
var onProgress goja.Callable
if len(call.Arguments) > 3 && !goja.IsUndefined(call.Arguments[3]) && !goja.IsNull(call.Arguments[3]) {
callback, ok := goja.AssertFunction(call.Arguments[3])
if !ok {
return r.jsError("progress callback must be a function")
}
onProgress = callback
}
unlock := lockDownloadOutputPath(outputPath)
defer unlock()
input, err := os.Open(inputPath)
if err != nil {
return r.jsError("failed to open input file: %v", err)
}
info, err := input.Stat()
if err != nil {
input.Close()
return r.jsError("failed to stat input file: %v", err)
}
totalSize := info.Size()
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
input.Close()
return r.jsError("failed to create output directory: %v", err)
}
stagedPath := outputPath + ".transform.partial"
if filepath.Clean(stagedPath) == filepath.Clean(inputPath) {
input.Close()
return r.jsError("input path conflicts with transform staging path")
}
_ = os.Remove(stagedPath)
output, err := os.OpenFile(stagedPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
input.Close()
return r.jsError("failed to create staged output: %v", err)
}
cleanup := func() {
input.Close()
output.Close()
_ = os.Remove(stagedPath)
}
ctx := r.activeOperationContext(context.Background())
buffer := make([]byte, int(bufferSize))
processed := int64(0)
segmentIndex := int64(0)
segmentsTransformed := int64(0)
for {
if err := ctx.Err(); err != nil {
cleanup()
return r.jsError("patterned file transform cancelled: %v", err)
}
readCount, readErr := io.ReadFull(input, buffer)
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
cleanup()
return r.jsError("failed to read input file: %v", readErr)
}
if readCount == 0 {
break
}
chunk := buffer[:readCount]
for offset := 0; offset < readCount; offset += int(segmentSize) {
end := min(offset+int(segmentSize), readCount)
segment := chunk[offset:end]
selected := segmentIndex%transformEvery == transformOffset
fullSegment := len(segment) == int(segmentSize)
if selected && (fullSegment || transformPartial) {
if parsedCipher.Mode == "cbc" && len(segment)%block.BlockSize() != 0 {
cleanup()
return r.jsError("selected segment %d is not a multiple of %d bytes", segmentIndex, block.BlockSize())
}
transformPatternedSegment(block, parsedCipher, operation, segment)
segmentsTransformed++
}
segmentIndex++
}
written, writeErr := output.Write(chunk)
if writeErr != nil || written != len(chunk) {
if writeErr == nil {
writeErr = io.ErrShortWrite
}
cleanup()
return r.jsError("failed to write transformed file: %v", writeErr)
}
processed += int64(written)
if onProgress != nil {
if _, callbackErr := onProgress(
goja.Undefined(),
r.vm.ToValue(processed),
r.vm.ToValue(totalSize),
); callbackErr != nil {
cleanup()
return r.jsError("progress callback failed: %v", callbackErr)
}
}
if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
break
}
}
if err := input.Close(); err != nil {
output.Close()
_ = os.Remove(stagedPath)
return r.jsError("failed to close input file: %v", err)
}
if err := output.Sync(); err != nil {
output.Close()
_ = os.Remove(stagedPath)
return r.jsError("failed to sync transformed file: %v", err)
}
if err := output.Close(); err != nil {
_ = os.Remove(stagedPath)
return r.jsError("failed to close transformed file: %v", err)
}
if err := os.Rename(stagedPath, outputPath); err != nil {
_ = os.Remove(stagedPath)
return r.jsError("failed to publish transformed file: %v", err)
}
return r.jsSuccess(map[string]any{
"path": outputPath,
"bytes_processed": processed,
"segments_processed": segmentIndex,
"segments_transformed": segmentsTransformed,
})
}
func transformPatternedSegment(
block cipher.Block,
options *runtimeBlockCipherOptions,
operation string,
segment []byte,
) {
if options.Mode == "ctr" {
cipher.NewCTR(block, options.IV).XORKeyStream(segment, segment)
return
}
if operation == "encrypt" {
cipher.NewCBCEncrypter(block, options.IV).CryptBlocks(segment, segment)
return
}
cipher.NewCBCDecrypter(block, options.IV).CryptBlocks(segment, segment)
}