fix(extensions): bound stream resolution without timing out active transfers

This commit is contained in:
zarzet
2026-09-06 14:44:54 +07:00
parent d725e11abc
commit 8daae638f6
12 changed files with 512 additions and 19 deletions
+14
View File
@@ -171,6 +171,20 @@ storage, and file access.
## Downloading files
Each `download(trackID, quality, outputPath, onProgress, options)` call has a
cumulative 60-second stream-resolution allowance. It covers metadata/ticket
requests, URL resolution, retry waits, and refreshes, including HTTP headers
and the first audio byte. Native audio reads after the first byte and bounded
native FFmpeg conversion pause this clock; transfer stall limits and the
overall download timeout still apply. Progress callbacks do not reset it.
`options.resolutionTimeoutMs` reports the initial allowance, while
`utils.getResolutionRemainingMs()` reports the remaining allowance. Check for
the function before using it on older hosts. Stop with `error_type: "timeout"`
if a server's required retry delay cannot fit; do not shorten `Retry-After`
to force another request. The host also enforces the deadline for extensions
that do not check it themselves.
Extensions with `permissions.file: true` can stream a remote file into their
allowed output path:
+10 -1
View File
@@ -635,6 +635,12 @@ func (p *extensionProviderWrapper) DownloadPrepared(
SetItemPreparing(itemID)
}
if runtime != nil {
var finishResolution func()
downloadCtx, finishResolution = runtime.beginResolutionBudget(downloadCtx, extensionResolutionTimeout)
defer finishResolution()
}
progressCallback := vm.ToValue(func(call goja.FunctionCall) goja.Value {
if len(call.Arguments) > 0 {
percent := int(call.Arguments[0].ToInteger())
@@ -658,7 +664,7 @@ func (p *extensionProviderWrapper) DownloadPrepared(
}
jsStartedAt := time.Now()
downloadOptions := map[string]any{}
downloadOptions := map[string]any{"resolutionTimeoutMs": extensionResolutionTimeout.Milliseconds()}
if len(preparedContext) > 0 {
downloadOptions["preparedContext"] = preparedContext
}
@@ -686,6 +692,9 @@ func (p *extensionProviderWrapper) DownloadPrepared(
errType := "script_error"
if IsTimeoutError(err) {
errMsg = "download timeout: extension took too long to complete"
if context.Cause(downloadCtx) == context.DeadlineExceeded {
errMsg = "stream resolution timeout: extension took too long to resolve an audio stream"
}
errType = "timeout"
}
return &ExtDownloadResult{
+164
View File
@@ -0,0 +1,164 @@
package gobackend
import (
"context"
"io"
"net/http"
"sync"
"time"
"github.com/dop251/goja"
)
const extensionResolutionTimeout = 60 * time.Second
// resolutionBudget counts total resolver time across URL refreshes and retries.
// Only native transfers with received bytes and bounded native conversion work
// pause it; progress/status callbacks cannot reset the allowance.
type resolutionBudget struct {
ctx context.Context
cancel context.CancelCauseFunc
mu sync.Mutex
remaining time.Duration
started time.Time
timer *time.Timer
generation uint64
pauses int
stopped bool
}
func newResolutionBudget(parent context.Context, allowance time.Duration) *resolutionBudget {
ctx, cancel := context.WithCancelCause(parent)
b := &resolutionBudget{ctx: ctx, cancel: cancel, remaining: allowance}
b.mu.Lock()
b.armLocked()
b.mu.Unlock()
return b
}
func (b *resolutionBudget) armLocked() {
b.started = time.Now()
b.generation++
generation := b.generation
b.timer = time.AfterFunc(b.remaining, func() {
b.mu.Lock()
defer b.mu.Unlock()
if b.stopped || b.pauses > 0 || b.generation != generation {
return
}
b.remaining = 0
b.cancel(context.DeadlineExceeded)
})
}
func (b *resolutionBudget) pause() func() {
b.mu.Lock()
if b.stopped || b.ctx.Err() != nil {
b.mu.Unlock()
return func() {}
}
if b.pauses == 0 {
b.timer.Stop()
b.generation++
b.remaining -= time.Since(b.started)
if b.remaining <= 0 {
b.remaining = 0
b.cancel(context.DeadlineExceeded)
b.mu.Unlock()
return func() {}
}
}
b.pauses++
b.mu.Unlock()
var once sync.Once
return func() {
once.Do(func() {
b.mu.Lock()
defer b.mu.Unlock()
b.pauses--
if b.pauses == 0 && !b.stopped && b.ctx.Err() == nil {
b.armLocked()
}
})
}
}
func (b *resolutionBudget) remainingTime() time.Duration {
b.mu.Lock()
defer b.mu.Unlock()
remaining := b.remaining
if b.pauses == 0 && !b.stopped {
remaining -= time.Since(b.started)
}
if remaining < 0 || b.ctx.Err() != nil {
return 0
}
return remaining
}
func (b *resolutionBudget) stop() {
b.mu.Lock()
defer b.mu.Unlock()
b.stopped = true
b.generation++
b.timer.Stop()
b.cancel(context.Canceled)
}
func (r *extensionRuntime) currentResolutionBudget() *resolutionBudget {
r.resolutionMu.RLock()
defer r.resolutionMu.RUnlock()
return r.resolutionBudget
}
func (r *extensionRuntime) beginResolutionBudget(ctx context.Context, allowance time.Duration) (context.Context, func()) {
b := newResolutionBudget(ctx, allowance)
r.resolutionMu.Lock()
r.resolutionBudget = b
r.resolutionMu.Unlock()
return b.ctx, func() {
b.stop()
r.resolutionMu.Lock()
if r.resolutionBudget == b {
r.resolutionBudget = nil
}
r.resolutionMu.Unlock()
}
}
func (r *extensionRuntime) getResolutionRemainingMs(goja.FunctionCall) goja.Value {
if b := r.currentResolutionBudget(); b != nil {
return r.vm.ToValue(b.remainingTime().Milliseconds())
}
return r.vm.ToValue(extensionResolutionTimeout.Milliseconds())
}
// Wrapping only native audio transfers keeps API/ticket bodies on the resolver
// clock. Headers and the first byte still consume the resolution allowance.
func (r *extensionRuntime) trackResolutionTransfer(resp *http.Response) {
if resp == nil || resp.Body == nil || (resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent) {
return
}
if b := r.currentResolutionBudget(); b != nil {
resp.Body = &resolutionTransferBody{ReadCloser: resp.Body, budget: b}
}
}
type resolutionTransferBody struct {
io.ReadCloser
budget *resolutionBudget
receivedBytes bool // accessed only by the body's reader
}
func (b *resolutionTransferBody) Read(p []byte) (int, error) {
// Pause only the native read, so JS progress callbacks (which can invoke
// more resolvers) and retry waits continue spending the same allowance.
if b.receivedBytes {
defer b.budget.pause()()
}
n, err := b.ReadCloser.Read(p)
if n > 0 {
b.receivedBytes = true
}
return n, err
}
@@ -0,0 +1,277 @@
package gobackend
import (
"context"
"errors"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/dop251/goja"
)
func runResolutionScript(t *testing.T, r *extensionRuntime, parent context.Context, allowance time.Duration, script string) (goja.Value, error) {
t.Helper()
ctx, finish := r.beginResolutionBudget(parent, allowance)
defer finish()
return RunWithTimeoutContextAndRecover(ctx, r.vm, script, 3*time.Second)
}
func TestResolutionBudgetInterruptsBlockedOperationsAsTimeout(t *testing.T) {
for _, operation := range []string{"http", "sleep", "signed-session", "busy-script"} {
t.Run(operation, func(t *testing.T) {
r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) {
<-req.Context().Done()
return nil, req.Context().Err()
})
r.vm.Set("blockedHTTP", func() goja.Value { return r.doExtensionHTTP("GET", "https://cdn.example.com/api", nil, false, nil) })
r.vm.Set("sleep", r.sleep)
r.vm.Set("signedSessionWait", func() goja.Value {
ctx, cancel := r.signedSessionExchangeContext()
defer cancel()
<-ctx.Done()
return r.vm.ToValue(false)
})
script := map[string]string{
"http": "blockedHTTP(); true",
"sleep": "sleep(300000); true",
"signed-session": "signedSessionWait(); true",
"busy-script": "while (true) {}",
}[operation]
started := time.Now()
_, err := runResolutionScript(t, r, context.Background(), 50*time.Millisecond, script)
if !IsTimeoutError(err) || errors.Is(err, ErrExtensionRequestCancelled) || IsRuntimeUnsafeError(err) {
t.Fatalf("expected safe timeout, got %v", err)
}
if time.Since(started) > time.Second {
t.Fatal("blocked operation did not stop promptly")
}
// The interrupt was cleared and the operation context was detached.
value, err := runResolutionScript(t, r, context.Background(), time.Second, "42")
if err != nil || value.ToInteger() != 42 {
t.Fatalf("reuse: %v, %v", value, err)
}
})
}
}
type resolutionSlowBody struct {
ctx context.Context
reads int
}
func (b *resolutionSlowBody) Read(p []byte) (int, error) {
if b.reads == 3 {
return 0, io.EOF
}
if b.reads > 0 {
select {
case <-time.After(90 * time.Millisecond):
case <-b.ctx.Done():
return 0, b.ctx.Err()
}
}
p[0] = 'a'
b.reads++
return 1, nil
}
func (*resolutionSlowBody) Close() error { return nil }
func TestResolutionBudgetAllowsActiveNativeTransfers(t *testing.T) {
for _, kind := range []string{"plain", "chunked", "segments"} {
t.Run(kind, func(t *testing.T) {
r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) {
header := make(http.Header)
header.Set("Content-Length", "3")
status := http.StatusOK
if req.Header.Get("Range") != "" {
status = http.StatusPartialContent
header.Set("Content-Range", "bytes 0-2/3")
}
var body io.ReadCloser = &resolutionSlowBody{ctx: req.Context()}
if req.Method == "HEAD" {
body = io.NopCloser(strings.NewReader(""))
}
return &http.Response{StatusCode: status, Header: header, Body: body, ContentLength: 3, Request: req}, nil
})
r.vm.Set("download", r.fileDownload)
r.vm.Set("segments", r.fileDownloadSegments)
script := `download("https://cdn.example.com/audio", "audio.flac")`
if kind == "chunked" {
script = `download("https://cdn.example.com/audio", "audio.flac", {chunked:true})`
}
if kind == "segments" {
script = `segments(["https://cdn.example.com/audio"], "audio.flac")`
}
started := time.Now()
value, err := runResolutionScript(t, r, context.Background(), 70*time.Millisecond, script)
if err != nil {
t.Fatal(err)
}
if result := value.Export().(map[string]any); result["success"] != true {
t.Fatalf("transfer failed: %#v", result)
}
if time.Since(started) < 180*time.Millisecond {
t.Fatal("transfer did not exceed resolution allowance")
}
data, err := os.ReadFile(filepath.Join(r.dataDir, "audio.flac"))
if err != nil || string(data) != "aaa" {
t.Fatalf("output: %q, %v", data, err)
}
})
}
}
func TestResolutionBudgetDoesNotResetAcrossTransfersAndRefresh(t *testing.T) {
r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: &resolutionSlowBody{ctx: req.Context()}, ContentLength: 3, Request: req}, nil
})
r.vm.Set("download", r.fileDownload)
r.vm.Set("sleep", r.sleep)
_, err := runResolutionScript(t, r, context.Background(), 120*time.Millisecond, `
sleep(70);
var result = download("https://cdn.example.com/audio", "audio.flac");
if (!result.success) throw new Error("transfer failed");
sleep(70); // refresh must spend the remaining allowance, not a new 120ms
true;
`)
if !IsTimeoutError(err) {
t.Fatalf("expected cumulative timeout, got %v", err)
}
}
func TestResolutionBudgetIncludesTransferFirstByteAndProgressCallbacks(t *testing.T) {
for _, mode := range []string{"first-byte", "callback"} {
t.Run(mode, func(t *testing.T) {
r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) {
body := &resolutionSlowBody{ctx: req.Context()}
if mode == "first-byte" {
body.reads = 1
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body, ContentLength: 3, Request: req}, nil
})
r.vm.Set("download", r.fileDownload)
r.vm.Set("sleep", r.sleep)
_, err := runResolutionScript(t, r, context.Background(), 60*time.Millisecond, `download("https://cdn.example.com/audio", "audio.flac", {onProgress:function() { sleep(1000); }})`)
if !IsTimeoutError(err) {
t.Fatalf("expected timeout, got %v", err)
}
})
}
}
func TestResolutionBudgetPreservesUserCancellationDuringTransfer(t *testing.T) {
parent, cancel := context.WithCancel(context.Background())
defer cancel()
r := newFileDownloadTestRuntime(t, func(req *http.Request) (*http.Response, error) {
time.AfterFunc(40*time.Millisecond, cancel)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: &resolutionSlowBody{ctx: req.Context()}, ContentLength: 3, Request: req}, nil
})
r.vm.Set("download", r.fileDownload)
_, err := runResolutionScript(t, r, parent, time.Second, `download("https://cdn.example.com/audio","audio.flac")`)
if !errors.Is(err, ErrExtensionRequestCancelled) || IsTimeoutError(err) {
t.Fatalf("expected cancellation, got %v", err)
}
}
func TestResolutionBudgetConcurrentPausesAndStop(t *testing.T) {
b := newResolutionBudget(context.Background(), time.Second)
var workers sync.WaitGroup
for i := 0; i < 20; i++ {
workers.Add(1)
go func() {
defer workers.Done()
for j := 0; j < 100; j++ {
resume := b.pause()
b.remainingTime()
resume()
resume()
}
}()
}
workers.Wait()
b.stop()
if !errors.Is(context.Cause(b.ctx), context.Canceled) {
t.Fatalf("unexpected cause: %v", context.Cause(b.ctx))
}
}
func TestResolutionBudgetPoolClearsOperationContext(t *testing.T) {
ext := newTestLoadedExtension(t, ExtensionTypeDownloadProvider)
provider := newExtensionProviderWrapper(ext)
for i := 0; i < 2; i++ {
result, err := provider.Download("track-1", "LOSSLESS", filepath.Join(t.TempDir(), "audio.flac"), "", nil)
if err != nil || !result.Success {
t.Fatalf("download: %#v, %v", result, err)
}
ext.isolatedPoolMu.Lock()
if len(ext.isolatedPool) != 1 {
ext.isolatedPoolMu.Unlock()
t.Fatal("healthy runtime was not pooled")
}
r := ext.isolatedPool[0].runtime
ext.isolatedPoolMu.Unlock()
if r.currentResolutionBudget() != nil || r.activeOperationContext(context.Background()).Err() != nil {
t.Fatal("pooled runtime retained expired resolution context")
}
}
}
func TestResolutionBudgetFFmpegWaitExcludesConversionAndHonorsCancellation(t *testing.T) {
for _, cancelled := range []bool{false, true} {
t.Run(map[bool]string{false: "complete", true: "cancel"}[cancelled], func(t *testing.T) {
r := newFileDownloadTestRuntime(t, nil)
r.extensionID = "resolution-ffmpeg-test"
r.vm.Set("convert", r.ffmpegConvert)
parent, cancel := context.WithCancel(context.Background())
defer cancel()
responderDone := make(chan struct{})
go func() {
defer close(responderDone)
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
ffmpegCommandsMu.RLock()
id := ""
for key, command := range ffmpegCommands {
if command.ExtensionID == r.extensionID {
id = key
break
}
}
ffmpegCommandsMu.RUnlock()
if id != "" {
time.Sleep(120 * time.Millisecond)
if cancelled {
cancel()
} else {
SetFFmpegCommandResult(id, true, "converted", "")
}
return
}
time.Sleep(time.Millisecond)
}
}()
value, err := runResolutionScript(t, r, parent, 60*time.Millisecond, `convert("input.flac","output.flac",{codec:"flac"})`)
<-responderDone
if cancelled {
if !errors.Is(err, ErrExtensionRequestCancelled) {
t.Fatalf("expected cancel: %v", err)
}
} else if err != nil || value.Export().(map[string]any)["success"] != true {
t.Fatalf("conversion: %v, %v", value, err)
}
ffmpegCommandsMu.RLock()
defer ffmpegCommandsMu.RUnlock()
for _, command := range ffmpegCommands {
if command.ExtensionID == r.extensionID {
t.Fatal("FFmpeg command leaked")
}
}
})
}
}
+7
View File
@@ -245,6 +245,9 @@ type extensionRuntime struct {
activeDownloadMu sync.RWMutex
activeDownloadItemID string
resolutionMu sync.RWMutex
resolutionBudget *resolutionBudget
activeRequestMu sync.RWMutex
activeRequestID string
@@ -420,6 +423,9 @@ func (r *extensionRuntime) bindDownloadCancelContext(req *http.Request) *http.Re
// cancels it when that response body closes, so that request context must not
// be reused for provider retry delays between requests.
func (r *extensionRuntime) activeOperationContext(fallback context.Context) context.Context {
if budget := r.currentResolutionBudget(); budget != nil {
return budget.ctx
}
itemID := r.getActiveDownloadItemID()
if itemID == "" {
requestID := r.getActiveRequestID()
@@ -791,6 +797,7 @@ func (r *extensionRuntime) RegisterAPIs(vm *goja.Runtime) {
utilsObj.Set("appVersion", r.appVersion)
utilsObj.Set("appUserAgent", r.appUserAgent)
utilsObj.Set("sleep", r.sleep)
utilsObj.Set("getResolutionRemainingMs", r.getResolutionRemainingMs)
utilsObj.Set("isDownloadCancelled", r.isDownloadCancelled)
utilsObj.Set("isRequestCancelled", r.isRequestCancelled)
utilsObj.Set("setDownloadStatus", r.setDownloadStatus)
+1
View File
@@ -265,6 +265,7 @@ func (r *extensionRuntime) fileDownloadChunked(
}
request, watchdog := bindStallWatchdog(request, downloadStallTimeout)
response, responseErr := client.Do(request)
r.trackResolutionTransfer(response)
if responseErr != nil {
stalled := watchdog.stalled.Load()
watchdog.stop()
+11
View File
@@ -1,6 +1,7 @@
package gobackend
import (
"context"
"fmt"
"regexp"
"sync"
@@ -75,6 +76,13 @@ func (r *extensionRuntime) ffmpegExecute(call goja.FunctionCall) goja.Value {
}
func (r *extensionRuntime) executeFFmpegCommand(arguments []string, inputPath, outputPath string) goja.Value {
ctx := r.activeOperationContext(context.Background())
if budget := r.currentResolutionBudget(); budget != nil {
defer budget.pause()()
}
if ctx.Err() != nil {
return r.jsError("FFmpeg command cancelled: %v", context.Cause(ctx))
}
ffmpegCommandsMu.Lock()
ffmpegCommandID++
@@ -106,6 +114,9 @@ func (r *extensionRuntime) executeFFmpegCommand(arguments []string, inputPath, o
delete(ffmpegCommands, cmdID)
ffmpegCommandsMu.Unlock()
return r.vm.ToValue(result)
case <-ctx.Done():
ClearFFmpegCommand(cmdID)
return r.jsError("FFmpeg command cancelled: %v", context.Cause(ctx))
case <-time.After(5 * time.Minute):
ClearFFmpegCommand(cmdID)
return r.jsError("FFmpeg command timed out")
+1
View File
@@ -219,6 +219,7 @@ func (r *extensionRuntime) fetchSegmentToTemp(
}
req, watchdog := bindStallWatchdog(req, downloadStallTimeout)
resp, err := client.Do(req)
r.trackResolutionTransfer(resp)
if err != nil {
stalled := watchdog.stalled.Load()
watchdog.stop()
+1
View File
@@ -395,6 +395,7 @@ func (r *extensionRuntime) reliableFileDownload(
retryContext := req.Context()
req, watchdog := bindStallWatchdog(req, downloadStallTimeout)
resp, requestErr := client.Do(req)
r.trackResolutionTransfer(resp)
if requestErr != nil {
stalled := watchdog.stalled.Load()
watchdog.stop()
+16 -12
View File
@@ -1,6 +1,7 @@
package gobackend
import (
"context"
"crypto/hmac"
"crypto/md5"
"crypto/rand"
@@ -285,24 +286,27 @@ func (r *extensionRuntime) sleep(call goja.FunctionCall) goja.Value {
sleepMs = 5 * 60 * 1000
}
ctx := r.activeOperationContext(context.Background())
timer := time.NewTimer(time.Duration(sleepMs) * time.Millisecond)
defer timer.Stop()
// A pending cancellation sentinel can precede context initialization.
// Preserve its visibility for utility callers outside DownloadPrepared.
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
itemID := r.getActiveDownloadItemID()
deadline := time.Now().Add(time.Duration(sleepMs) * time.Millisecond)
requestID := r.getActiveRequestID()
for {
if itemID != "" && isDownloadCancelled(itemID) {
if (itemID != "" && isDownloadCancelled(itemID)) ||
(requestID != "" && isExtensionRequestCancelled(requestID)) {
return r.vm.ToValue(false)
}
remaining := time.Until(deadline)
if remaining <= 0 {
select {
case <-ctx.Done():
return r.vm.ToValue(false)
case <-timer.C:
return r.vm.ToValue(true)
case <-ticker.C:
}
step := 100 * time.Millisecond
if remaining < step {
step = remaining
}
time.Sleep(step)
}
}
+1 -5
View File
@@ -639,11 +639,7 @@ func (r *extensionRuntime) exchangeSignedSessionGrant(grant string) error {
func (r *extensionRuntime) signedSessionExchangeContext() (context.Context, context.CancelFunc) {
parent := context.Background()
if r != nil {
if itemID := r.getActiveDownloadItemID(); itemID != "" {
parent = downloadCancelContext(itemID)
} else if requestID := r.getActiveRequestID(); requestID != "" {
parent = extensionRequestCancelContext(requestID)
}
parent = r.activeOperationContext(parent)
}
return context.WithTimeout(parent, signedSessionExchangeTimeout)
}
+9 -1
View File
@@ -89,9 +89,17 @@ func runGojaCallWithTimeoutContext(ctx context.Context, vm *goja.Runtime, call f
select {
case res := <-resultCh:
// A host call may return as cancellation fires. Do not let a swallowed
// native timeout become a script error (or even success).
if ctx.Err() != nil {
if errors.Is(context.Cause(ctx), context.Canceled) {
return nil, ErrExtensionRequestCancelled
}
return nil, &JSExecutionError{Message: "execution timeout exceeded", IsTimeout: true}
}
return res.value, res.err
case <-ctx.Done():
cancelled := ctx.Err() == context.Canceled
cancelled := errors.Is(context.Cause(ctx), context.Canceled)
interruptMu.Lock()
interrupted = true
interruptMu.Unlock()