mirror of
https://github.com/zarzet/SpotiFLAC-Mobile.git
synced 2026-09-03 16:50:40 +02:00
perf(extensions): streamline prepared download work
This commit is contained in:
@@ -28,6 +28,7 @@ func attemptExtensionDownload(
|
|||||||
) (resp *DownloadResponse, cancelledOuter bool) {
|
) (resp *DownloadResponse, cancelledOuter bool) {
|
||||||
req.DownloadProvider = strings.TrimSpace(providerLabel)
|
req.DownloadProvider = strings.TrimSpace(providerLabel)
|
||||||
req.ProviderTrackID = strings.TrimSpace(trackID)
|
req.ProviderTrackID = strings.TrimSpace(trackID)
|
||||||
|
preparedContext = extensionPreparedDownloadContext(req, preparedContext)
|
||||||
outputPath := buildOutputPathForExtension(req, ext)
|
outputPath := buildOutputPathForExtension(req, ext)
|
||||||
if shouldReuseExistingOutput(req, outputPath) {
|
if shouldReuseExistingOutput(req, outputPath) {
|
||||||
result := DownloadResult{FilePath: outputPath}
|
result := DownloadResult{FilePath: outputPath}
|
||||||
@@ -217,6 +218,7 @@ func attemptVerifiedResumeBeforeMetadata(
|
|||||||
req.QobuzID,
|
req.QobuzID,
|
||||||
req.DurationMS,
|
req.DurationMS,
|
||||||
req.ItemID,
|
req.ItemID,
|
||||||
|
extensionAvailabilityTrackContext(req),
|
||||||
)
|
)
|
||||||
if shouldAbortCancelledFallback(req.ItemID, err) {
|
if shouldAbortCancelledFallback(req.ItemID, err) {
|
||||||
return nil, true
|
return nil, true
|
||||||
@@ -296,6 +298,41 @@ func attemptVerifiedResumeBeforeMetadata(
|
|||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func extensionAvailabilityTrackContext(req DownloadRequest) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"id": req.ProviderTrackID,
|
||||||
|
"name": req.TrackName,
|
||||||
|
"artists": req.ArtistName,
|
||||||
|
"album_name": req.AlbumName,
|
||||||
|
"album_artist": req.AlbumArtist,
|
||||||
|
"cover_url": req.CoverURL,
|
||||||
|
"release_date": req.ReleaseDate,
|
||||||
|
"track_number": req.TrackNumber,
|
||||||
|
"total_tracks": req.TotalTracks,
|
||||||
|
"disc_number": req.DiscNumber,
|
||||||
|
"total_discs": req.TotalDiscs,
|
||||||
|
"duration_ms": req.DurationMS,
|
||||||
|
"isrc": req.ISRC,
|
||||||
|
"genre": req.Genre,
|
||||||
|
"label": req.Label,
|
||||||
|
"copyright": req.Copyright,
|
||||||
|
"composer": req.Composer,
|
||||||
|
"comment": req.Comment,
|
||||||
|
"explicit": req.Explicit,
|
||||||
|
"album_type": req.AlbumType,
|
||||||
|
"upc": req.UPC,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extensionPreparedDownloadContext(req DownloadRequest, prepared map[string]any) map[string]any {
|
||||||
|
merged := make(map[string]any, len(prepared)+1)
|
||||||
|
for key, value := range prepared {
|
||||||
|
merged[key] = value
|
||||||
|
}
|
||||||
|
merged["host_track"] = extensionAvailabilityTrackContext(req)
|
||||||
|
return merged
|
||||||
|
}
|
||||||
|
|
||||||
func buildOutputStorageFailureResponse(
|
func buildOutputStorageFailureResponse(
|
||||||
providerID string,
|
providerID string,
|
||||||
err error,
|
err error,
|
||||||
@@ -407,7 +444,7 @@ func DownloadWithExtensionFallback(req DownloadRequest) (*DownloadResponse, erro
|
|||||||
ext, err := extManager.GetExtension(req.Source)
|
ext, err := extManager.GetExtension(req.Source)
|
||||||
if err == nil && ext.Enabled && ext.Error == "" && ext.Manifest.IsDownloadProvider() {
|
if err == nil && ext.Enabled && ext.Error == "" && ext.Manifest.IsDownloadProvider() {
|
||||||
provider := newExtensionProviderWrapper(ext)
|
provider := newExtensionProviderWrapper(ext)
|
||||||
availability, availErr := provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID)
|
availability, availErr := provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID, extensionAvailabilityTrackContext(req))
|
||||||
if shouldAbortCancelledFallback(req.ItemID, availErr) {
|
if shouldAbortCancelledFallback(req.ItemID, availErr) {
|
||||||
return nil, ErrDownloadCancelled
|
return nil, ErrDownloadCancelled
|
||||||
}
|
}
|
||||||
@@ -631,7 +668,7 @@ func DownloadWithExtensionFallback(req DownloadRequest) (*DownloadResponse, erro
|
|||||||
|
|
||||||
provider := newExtensionProviderWrapper(ext)
|
provider := newExtensionProviderWrapper(ext)
|
||||||
|
|
||||||
availability, err := provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID)
|
availability, err := provider.CheckAvailabilityForItemID(req.ISRC, req.TrackName, req.ArtistName, req.SpotifyID, req.DeezerID, req.TidalID, req.QobuzID, req.DurationMS, req.ItemID, extensionAvailabilityTrackContext(req))
|
||||||
if shouldAbortCancelledFallback(req.ItemID, err) {
|
if shouldAbortCancelledFallback(req.ItemID, err) {
|
||||||
return nil, ErrDownloadCancelled
|
return nil, ErrDownloadCancelled
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -103,3 +103,32 @@ func TestOverlaySourceExtensionTrackIdentityFillsMissingValues(t *testing.T) {
|
|||||||
t.Fatalf("missing identity fields were not enriched: %#v", req)
|
t.Fatalf("missing identity fields were not enriched: %#v", req)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtensionPreparedDownloadContextMergesHostMetadata(t *testing.T) {
|
||||||
|
providerContext := map[string]any{
|
||||||
|
"token": "opaque-provider-value",
|
||||||
|
"host_track": "provider-key-must-not-win",
|
||||||
|
}
|
||||||
|
req := DownloadRequest{
|
||||||
|
ProviderTrackID: "provider-track-1",
|
||||||
|
TrackName: "Song",
|
||||||
|
ArtistName: "Artist",
|
||||||
|
AlbumName: "Album",
|
||||||
|
ISRC: "ISRC123",
|
||||||
|
}
|
||||||
|
|
||||||
|
merged := extensionPreparedDownloadContext(req, providerContext)
|
||||||
|
if merged["token"] != "opaque-provider-value" {
|
||||||
|
t.Fatalf("provider context was not preserved: %#v", merged)
|
||||||
|
}
|
||||||
|
hostTrack, ok := merged["host_track"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("host_track = %#v", merged["host_track"])
|
||||||
|
}
|
||||||
|
if hostTrack["id"] != req.ProviderTrackID || hostTrack["name"] != req.TrackName || hostTrack["isrc"] != req.ISRC {
|
||||||
|
t.Fatalf("host metadata was not propagated: %#v", hostTrack)
|
||||||
|
}
|
||||||
|
if providerContext["host_track"] != "provider-key-must-not-win" {
|
||||||
|
t.Fatalf("input provider context was mutated: %#v", providerContext)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -273,12 +273,14 @@ func (m *extensionManager) loadExtensionFromFileLocked(filePath string) (*loaded
|
|||||||
}
|
}
|
||||||
|
|
||||||
var supportedRuntimeFeatures = map[string]int{
|
var supportedRuntimeFeatures = map[string]int{
|
||||||
"signedSession": 3,
|
"signedSession": 3,
|
||||||
"sessionRefresh": 1,
|
"sessionRefresh": 1,
|
||||||
"sessionGrant": 1,
|
"sessionGrant": 1,
|
||||||
"globalAction": 1,
|
"globalAction": 1,
|
||||||
"webviewAuth": 1,
|
"webviewAuth": 1,
|
||||||
"downloadSegments": 1,
|
"downloadSegments": 1,
|
||||||
|
"patternedFileTransform": 1,
|
||||||
|
"preparedContext": 1,
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateManifestGates enforces minAppVersion and requiredRuntimeFeatures
|
// validateManifestGates enforces minAppVersion and requiredRuntimeFeatures
|
||||||
|
|||||||
@@ -1,10 +1,65 @@
|
|||||||
package gobackend
|
package gobackend
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestExtensionDownloadCancellationInterruptsBusyJavaScript(t *testing.T) {
|
||||||
|
ext := newTestLoadedExtension(t, ExtensionTypeDownloadProvider)
|
||||||
|
if err := os.WriteFile(filepath.Join(ext.SourceDir, "index.js"), []byte(`
|
||||||
|
registerExtension({
|
||||||
|
download: function() {
|
||||||
|
while (true) {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
`), 0600); err != nil {
|
||||||
|
t.Fatalf("write busy extension: %v", err)
|
||||||
|
}
|
||||||
|
provider := newExtensionProviderWrapper(ext)
|
||||||
|
const itemID = "busy-download-cancel"
|
||||||
|
outputPath := filepath.Join(t.TempDir(), "busy.flac")
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := provider.DownloadPrepared(
|
||||||
|
"track",
|
||||||
|
"best",
|
||||||
|
outputPath,
|
||||||
|
itemID,
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
deadline := time.Now().Add(time.Second)
|
||||||
|
for {
|
||||||
|
downloadCancels.mu.Lock()
|
||||||
|
entry := downloadCancels.entries[itemID]
|
||||||
|
ready := entry != nil && entry.refs > 0
|
||||||
|
downloadCancels.mu.Unlock()
|
||||||
|
if ready {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("download cancellation context was not initialized")
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
cancelDownload(itemID)
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if !errors.Is(err, ErrDownloadCancelled) {
|
||||||
|
t.Fatalf("DownloadPrepared error = %v, want ErrDownloadCancelled", err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("busy extension did not stop after cancellation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExtensionProviderWrapperFullSurface(t *testing.T) {
|
func TestExtensionProviderWrapperFullSurface(t *testing.T) {
|
||||||
ext := newTestLoadedExtension(t, ExtensionTypeMetadataProvider, ExtensionTypeDownloadProvider, ExtensionTypeLyricsProvider)
|
ext := newTestLoadedExtension(t, ExtensionTypeMetadataProvider, ExtensionTypeDownloadProvider, ExtensionTypeLyricsProvider)
|
||||||
provider := newExtensionProviderWrapper(ext)
|
provider := newExtensionProviderWrapper(ext)
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ type extCallOpts struct {
|
|||||||
// any ProviderID stamping.
|
// any ProviderID stamping.
|
||||||
func callExtension[T any](p *extensionProviderWrapper, opts extCallOpts, parse func(perf *extensionCallPerf, result goja.Value) (T, error)) (T, error) {
|
func callExtension[T any](p *extensionProviderWrapper, opts extCallOpts, parse func(perf *extensionCallPerf, result goja.Value) (T, error)) (T, error) {
|
||||||
var zero T
|
var zero T
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
perf := newExtensionCallPerf(p.extension.ID, opts.perfName)
|
perf := newExtensionCallPerf(p.extension.ID, opts.perfName)
|
||||||
defer perf.finish()
|
defer perf.finish()
|
||||||
@@ -73,14 +74,13 @@ func callExtension[T any](p *extensionProviderWrapper, opts extCallOpts, parse f
|
|||||||
p.extension.runtime.setActiveDownloadItemID(opts.itemID)
|
p.extension.runtime.setActiveDownloadItemID(opts.itemID)
|
||||||
defer p.extension.runtime.clearActiveDownloadItemID()
|
defer p.extension.runtime.clearActiveDownloadItemID()
|
||||||
}
|
}
|
||||||
initDownloadCancel(opts.itemID)
|
ctx = initDownloadCancel(opts.itemID)
|
||||||
defer clearDownloadCancel(opts.itemID)
|
defer clearDownloadCancel(opts.itemID)
|
||||||
if isDownloadCancelled(opts.itemID) {
|
if isDownloadCancelled(opts.itemID) {
|
||||||
return zero, ErrDownloadCancelled
|
return zero, ErrDownloadCancelled
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
if opts.requestID != "" {
|
if opts.requestID != "" {
|
||||||
if p.extension.runtime != nil {
|
if p.extension.runtime != nil {
|
||||||
p.extension.runtime.setActiveRequestID(opts.requestID)
|
p.extension.runtime.setActiveRequestID(opts.requestID)
|
||||||
@@ -456,12 +456,13 @@ func (p *extensionProviderWrapper) EnrichTrackForItemID(track *ExtTrackMetadata,
|
|||||||
}
|
}
|
||||||
perf.recordInit(time.Since(initStartedAt))
|
perf.recordInit(time.Since(initStartedAt))
|
||||||
defer p.extension.VMMu.Unlock()
|
defer p.extension.VMMu.Unlock()
|
||||||
|
downloadCtx := context.Background()
|
||||||
if itemID != "" {
|
if itemID != "" {
|
||||||
if p.extension.runtime != nil {
|
if p.extension.runtime != nil {
|
||||||
p.extension.runtime.setActiveDownloadItemID(itemID)
|
p.extension.runtime.setActiveDownloadItemID(itemID)
|
||||||
defer p.extension.runtime.clearActiveDownloadItemID()
|
defer p.extension.runtime.clearActiveDownloadItemID()
|
||||||
}
|
}
|
||||||
initDownloadCancel(itemID)
|
downloadCtx = initDownloadCancel(itemID)
|
||||||
defer clearDownloadCancel(itemID)
|
defer clearDownloadCancel(itemID)
|
||||||
if isDownloadCancelled(itemID) {
|
if isDownloadCancelled(itemID) {
|
||||||
return track, ErrDownloadCancelled
|
return track, ErrDownloadCancelled
|
||||||
@@ -469,7 +470,7 @@ func (p *extensionProviderWrapper) EnrichTrackForItemID(track *ExtTrackMetadata,
|
|||||||
}
|
}
|
||||||
|
|
||||||
jsStartedAt := time.Now()
|
jsStartedAt := time.Now()
|
||||||
result, err := runGojaCallWithTimeoutAndRecover(p.vm, func() (goja.Value, error) {
|
result, err := runGojaCallWithTimeoutContextAndRecover(downloadCtx, p.vm, func() (goja.Value, error) {
|
||||||
return invokeExtensionMethod(p.vm, "enrichTrack", extensionTrackInput(track))
|
return invokeExtensionMethod(p.vm, "enrichTrack", extensionTrackInput(track))
|
||||||
}, DefaultJSTimeout)
|
}, DefaultJSTimeout)
|
||||||
perf.recordJS(time.Since(jsStartedAt))
|
perf.recordJS(time.Since(jsStartedAt))
|
||||||
@@ -505,7 +506,7 @@ func (p *extensionProviderWrapper) EnrichTrackForItemID(track *ExtTrackMetadata,
|
|||||||
return &enrichedTrack, nil
|
return &enrichedTrack, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *extensionProviderWrapper) CheckAvailabilityForItemID(isrc, trackName, artistName, spotifyID, deezerID, tidalID, qobuzID string, durationMS int, itemID string) (*ExtAvailabilityResult, error) {
|
func (p *extensionProviderWrapper) CheckAvailabilityForItemID(isrc, trackName, artistName, spotifyID, deezerID, tidalID, qobuzID string, durationMS int, itemID string, trackContexts ...map[string]any) (*ExtAvailabilityResult, error) {
|
||||||
if !p.extension.Manifest.IsDownloadProvider() {
|
if !p.extension.Manifest.IsDownloadProvider() {
|
||||||
return nil, fmt.Errorf("extension '%s' is not a download provider", p.extension.ID)
|
return nil, fmt.Errorf("extension '%s' is not a download provider", p.extension.ID)
|
||||||
}
|
}
|
||||||
@@ -520,6 +521,9 @@ func (p *extensionProviderWrapper) CheckAvailabilityForItemID(isrc, trackName, a
|
|||||||
"qobuz_id": qobuzID,
|
"qobuz_id": qobuzID,
|
||||||
"duration_ms": durationMS,
|
"duration_ms": durationMS,
|
||||||
}
|
}
|
||||||
|
if len(trackContexts) > 0 && len(trackContexts[0]) > 0 {
|
||||||
|
availabilityOptions["track"] = trackContexts[0]
|
||||||
|
}
|
||||||
|
|
||||||
return callExtension(p, extCallOpts{
|
return callExtension(p, extCallOpts{
|
||||||
perfName: "checkAvailability",
|
perfName: "checkAvailability",
|
||||||
@@ -620,8 +624,9 @@ func (p *extensionProviderWrapper) DownloadPrepared(
|
|||||||
runtime.setActiveDownloadItemID(itemID)
|
runtime.setActiveDownloadItemID(itemID)
|
||||||
defer runtime.clearActiveDownloadItemID()
|
defer runtime.clearActiveDownloadItemID()
|
||||||
}
|
}
|
||||||
|
downloadCtx := context.Background()
|
||||||
if itemID != "" {
|
if itemID != "" {
|
||||||
initDownloadCancel(itemID)
|
downloadCtx = initDownloadCancel(itemID)
|
||||||
defer clearDownloadCancel(itemID)
|
defer clearDownloadCancel(itemID)
|
||||||
SetItemPreparing(itemID)
|
SetItemPreparing(itemID)
|
||||||
}
|
}
|
||||||
@@ -653,7 +658,7 @@ func (p *extensionProviderWrapper) DownloadPrepared(
|
|||||||
if len(preparedContext) > 0 {
|
if len(preparedContext) > 0 {
|
||||||
downloadOptions["preparedContext"] = preparedContext
|
downloadOptions["preparedContext"] = preparedContext
|
||||||
}
|
}
|
||||||
result, err := runGojaCallWithTimeoutAndRecover(vm, func() (goja.Value, error) {
|
result, err := runGojaCallWithTimeoutContextAndRecover(downloadCtx, vm, func() (goja.Value, error) {
|
||||||
return invokeExtensionMethod(
|
return invokeExtensionMethod(
|
||||||
vm,
|
vm,
|
||||||
"download",
|
"download",
|
||||||
@@ -670,6 +675,9 @@ func (p *extensionProviderWrapper) DownloadPrepared(
|
|||||||
cleanupSafe = !IsRuntimeUnsafeError(err)
|
cleanupSafe = !IsRuntimeUnsafeError(err)
|
||||||
unsafeDone = runtimeCompletion(err)
|
unsafeDone = runtimeCompletion(err)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if itemID != "" && isDownloadCancelled(itemID) {
|
||||||
|
return nil, ErrDownloadCancelled
|
||||||
|
}
|
||||||
errMsg := err.Error()
|
errMsg := err.Error()
|
||||||
errType := "script_error"
|
errType := "script_error"
|
||||||
if IsTimeoutError(err) {
|
if IsTimeoutError(err) {
|
||||||
|
|||||||
@@ -730,6 +730,7 @@ func (r *extensionRuntime) RegisterAPIs(vm *goja.Runtime) {
|
|||||||
fileObj.Set("copy", r.fileCopy)
|
fileObj.Set("copy", r.fileCopy)
|
||||||
fileObj.Set("move", r.fileMove)
|
fileObj.Set("move", r.fileMove)
|
||||||
fileObj.Set("getSize", r.fileGetSize)
|
fileObj.Set("getSize", r.fileGetSize)
|
||||||
|
fileObj.Set("transformPatternedBlocks", r.fileTransformPatternedBlocks)
|
||||||
vm.Set("file", fileObj)
|
vm.Set("file", fileObj)
|
||||||
|
|
||||||
ffmpegObj := vm.NewObject()
|
ffmpegObj := vm.NewObject()
|
||||||
|
|||||||
@@ -184,6 +184,104 @@ func TestExtensionRuntime_BlockCipherCBCSupportsAES(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtensionRuntime_FileTransformPatternedBlocks(t *testing.T) {
|
||||||
|
vm := newBinaryTestRuntime(t, true)
|
||||||
|
|
||||||
|
result, err := vm.RunString(`
|
||||||
|
(function() {
|
||||||
|
var options = {
|
||||||
|
algorithm: "blowfish",
|
||||||
|
mode: "cbc",
|
||||||
|
key: "0123456789ABCDEFF0E1D2C3B4A59687",
|
||||||
|
keyEncoding: "hex",
|
||||||
|
iv: "0001020304050607",
|
||||||
|
ivEncoding: "hex",
|
||||||
|
inputEncoding: "hex",
|
||||||
|
outputEncoding: "hex",
|
||||||
|
padding: "none"
|
||||||
|
};
|
||||||
|
var plainSegments = [
|
||||||
|
"00112233445566778899aabbccddeeff",
|
||||||
|
"102132435465768798a9bacbdcedfe0f",
|
||||||
|
"2031425364758697a8b9cadbecfd0e1f",
|
||||||
|
"30415263748596a7b8c9daebfc0d1e2f"
|
||||||
|
];
|
||||||
|
var encrypted = "";
|
||||||
|
for (var i = 0; i < plainSegments.length; i++) {
|
||||||
|
if (i % 3 === 0) {
|
||||||
|
var enc = utils.encryptBlockCipher(plainSegments[i], options);
|
||||||
|
if (!enc.success) throw new Error(enc.error);
|
||||||
|
encrypted += enc.data;
|
||||||
|
} else {
|
||||||
|
encrypted += plainSegments[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var partialTail = "a1b2c3d4e5";
|
||||||
|
encrypted += partialTail;
|
||||||
|
var write = file.writeBytes("encrypted.bin", encrypted, {
|
||||||
|
encoding: "hex", truncate: true
|
||||||
|
});
|
||||||
|
if (!write.success) throw new Error(write.error);
|
||||||
|
|
||||||
|
var callbacks = 0;
|
||||||
|
var transformed = file.transformPatternedBlocks(
|
||||||
|
"encrypted.bin",
|
||||||
|
"decrypted.bin",
|
||||||
|
{
|
||||||
|
operation: "decrypt",
|
||||||
|
algorithm: "blowfish",
|
||||||
|
mode: "cbc",
|
||||||
|
key: options.key,
|
||||||
|
keyEncoding: "hex",
|
||||||
|
iv: options.iv,
|
||||||
|
ivEncoding: "hex",
|
||||||
|
padding: "none",
|
||||||
|
segmentSize: 16,
|
||||||
|
transformEvery: 3,
|
||||||
|
transformOffset: 0,
|
||||||
|
bufferSize: 32
|
||||||
|
},
|
||||||
|
function(processed, total) {
|
||||||
|
if (processed > total) throw new Error("invalid progress");
|
||||||
|
callbacks++;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if (!transformed.success) throw new Error(transformed.error);
|
||||||
|
var output = file.readBytes("decrypted.bin", {encoding: "hex"});
|
||||||
|
if (!output.success) throw new Error(output.error);
|
||||||
|
return JSON.stringify({
|
||||||
|
output: output.data,
|
||||||
|
expected: plainSegments.join("") + partialTail,
|
||||||
|
processed: transformed.bytes_processed,
|
||||||
|
segments: transformed.segments_processed,
|
||||||
|
transformedSegments: transformed.segments_transformed,
|
||||||
|
callbacks: callbacks
|
||||||
|
});
|
||||||
|
})()
|
||||||
|
`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("patterned file transform failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
decoded := decodeJSONResult[struct {
|
||||||
|
Output string `json:"output"`
|
||||||
|
Expected string `json:"expected"`
|
||||||
|
Processed int64 `json:"processed"`
|
||||||
|
Segments int64 `json:"segments"`
|
||||||
|
TransformedSegments int64 `json:"transformedSegments"`
|
||||||
|
Callbacks int `json:"callbacks"`
|
||||||
|
}](t, result)
|
||||||
|
if decoded.Output != decoded.Expected {
|
||||||
|
t.Fatalf("output = %q, want %q", decoded.Output, decoded.Expected)
|
||||||
|
}
|
||||||
|
if decoded.Processed != 69 || decoded.Segments != 5 || decoded.TransformedSegments != 2 {
|
||||||
|
t.Fatalf("unexpected transform stats: %+v", decoded)
|
||||||
|
}
|
||||||
|
if decoded.Callbacks != 3 {
|
||||||
|
t.Fatalf("callbacks = %d, want 3 buffered updates", decoded.Callbacks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExtensionRuntime_BlockCipherCTRSupportsAES(t *testing.T) {
|
func TestExtensionRuntime_BlockCipherCTRSupportsAES(t *testing.T) {
|
||||||
vm := newBinaryTestRuntime(t, false)
|
vm := newBinaryTestRuntime(t, false)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,245 @@
|
|||||||
|
package gobackend
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/cipher"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/dop251/goja"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultPatternedTransformBufferSize = int64(1 << 20)
|
||||||
|
maxPatternedTransformBufferSize = int64(16 << 20)
|
||||||
|
maxPatternedTransformSegmentSize = int64(16 << 20)
|
||||||
|
)
|
||||||
|
|
||||||
|
// fileTransformPatternedBlocks streams one file into another while applying an
|
||||||
|
// independent block-cipher transform to selected fixed-size segments. It keeps
|
||||||
|
// provider-specific layout knowledge in the extension: the host only receives
|
||||||
|
// a generic period/offset declaration and cipher parameters.
|
||||||
|
//
|
||||||
|
// JS signature:
|
||||||
|
//
|
||||||
|
// file.transformPatternedBlocks(inputPath, outputPath, {
|
||||||
|
// operation: "decrypt", algorithm: "blowfish", mode: "cbc",
|
||||||
|
// key: "...", keyEncoding: "hex", iv: "...", ivEncoding: "hex",
|
||||||
|
// segmentSize: 2048, transformEvery: 3, transformOffset: 0,
|
||||||
|
// bufferSize: 1048576, transformPartial: false
|
||||||
|
// }, function(processedBytes, totalBytes) {})
|
||||||
|
func (r *extensionRuntime) fileTransformPatternedBlocks(call goja.FunctionCall) goja.Value {
|
||||||
|
if len(call.Arguments) < 3 {
|
||||||
|
return r.jsError("input path, output path, and options are required")
|
||||||
|
}
|
||||||
|
|
||||||
|
inputPath, err := r.validatePath(call.Arguments[0].String())
|
||||||
|
if err != nil {
|
||||||
|
return r.jsError("%s", err.Error())
|
||||||
|
}
|
||||||
|
outputPath, err := r.validatePath(call.Arguments[1].String())
|
||||||
|
if err != nil {
|
||||||
|
return r.jsError("%s", err.Error())
|
||||||
|
}
|
||||||
|
options := parseRuntimeOptionsArgument(call, 2)
|
||||||
|
parsedCipher, err := parseRuntimeBlockCipherOptions(options)
|
||||||
|
if err != nil {
|
||||||
|
return r.jsError("%s", err.Error())
|
||||||
|
}
|
||||||
|
if parsedCipher.Padding != "none" {
|
||||||
|
return r.jsError("patterned file transforms only support padding: none")
|
||||||
|
}
|
||||||
|
if parsedCipher.Mode != "cbc" && parsedCipher.Mode != "ctr" {
|
||||||
|
return r.jsError("unsupported block cipher mode: %s", parsedCipher.Mode)
|
||||||
|
}
|
||||||
|
operation := strings.ToLower(runtimeOptionString(options, "operation", "decrypt"))
|
||||||
|
if operation != "decrypt" && operation != "encrypt" {
|
||||||
|
return r.jsError("operation must be decrypt or encrypt")
|
||||||
|
}
|
||||||
|
|
||||||
|
segmentSize := runtimeOptionInt64(options, "segmentSize", 0)
|
||||||
|
transformEvery := runtimeOptionInt64(options, "transformEvery", 1)
|
||||||
|
transformOffset := runtimeOptionInt64(options, "transformOffset", 0)
|
||||||
|
bufferSize := runtimeOptionInt64(options, "bufferSize", defaultPatternedTransformBufferSize)
|
||||||
|
transformPartial := runtimeOptionBool(options, "transformPartial", false)
|
||||||
|
if segmentSize <= 0 || segmentSize > maxPatternedTransformSegmentSize {
|
||||||
|
return r.jsError("segmentSize must be between 1 and %d bytes", maxPatternedTransformSegmentSize)
|
||||||
|
}
|
||||||
|
if transformEvery <= 0 {
|
||||||
|
return r.jsError("transformEvery must be greater than zero")
|
||||||
|
}
|
||||||
|
if transformOffset < 0 || transformOffset >= transformEvery {
|
||||||
|
return r.jsError("transformOffset must be between 0 and transformEvery - 1")
|
||||||
|
}
|
||||||
|
if bufferSize < segmentSize {
|
||||||
|
bufferSize = segmentSize
|
||||||
|
}
|
||||||
|
if bufferSize > maxPatternedTransformBufferSize {
|
||||||
|
bufferSize = maxPatternedTransformBufferSize
|
||||||
|
}
|
||||||
|
bufferSize -= bufferSize % segmentSize
|
||||||
|
if bufferSize == 0 {
|
||||||
|
bufferSize = segmentSize
|
||||||
|
}
|
||||||
|
|
||||||
|
block, err := newRuntimeBlockCipher(parsedCipher)
|
||||||
|
if err != nil {
|
||||||
|
return r.jsError("%s", err.Error())
|
||||||
|
}
|
||||||
|
if len(parsedCipher.IV) != block.BlockSize() {
|
||||||
|
return r.jsError("iv must be %d bytes for %s", block.BlockSize(), parsedCipher.Algorithm)
|
||||||
|
}
|
||||||
|
if parsedCipher.Mode == "cbc" && segmentSize%int64(block.BlockSize()) != 0 {
|
||||||
|
return r.jsError("segmentSize must be a multiple of %d bytes for CBC", block.BlockSize())
|
||||||
|
}
|
||||||
|
|
||||||
|
var onProgress goja.Callable
|
||||||
|
if len(call.Arguments) > 3 && !goja.IsUndefined(call.Arguments[3]) && !goja.IsNull(call.Arguments[3]) {
|
||||||
|
callback, ok := goja.AssertFunction(call.Arguments[3])
|
||||||
|
if !ok {
|
||||||
|
return r.jsError("progress callback must be a function")
|
||||||
|
}
|
||||||
|
onProgress = callback
|
||||||
|
}
|
||||||
|
|
||||||
|
unlock := lockDownloadOutputPath(outputPath)
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
|
input, err := os.Open(inputPath)
|
||||||
|
if err != nil {
|
||||||
|
return r.jsError("failed to open input file: %v", err)
|
||||||
|
}
|
||||||
|
info, err := input.Stat()
|
||||||
|
if err != nil {
|
||||||
|
input.Close()
|
||||||
|
return r.jsError("failed to stat input file: %v", err)
|
||||||
|
}
|
||||||
|
totalSize := info.Size()
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
|
||||||
|
input.Close()
|
||||||
|
return r.jsError("failed to create output directory: %v", err)
|
||||||
|
}
|
||||||
|
stagedPath := outputPath + ".transform.partial"
|
||||||
|
if filepath.Clean(stagedPath) == filepath.Clean(inputPath) {
|
||||||
|
input.Close()
|
||||||
|
return r.jsError("input path conflicts with transform staging path")
|
||||||
|
}
|
||||||
|
_ = os.Remove(stagedPath)
|
||||||
|
output, err := os.OpenFile(stagedPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
input.Close()
|
||||||
|
return r.jsError("failed to create staged output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup := func() {
|
||||||
|
input.Close()
|
||||||
|
output.Close()
|
||||||
|
_ = os.Remove(stagedPath)
|
||||||
|
}
|
||||||
|
ctx := r.activeOperationContext(context.Background())
|
||||||
|
buffer := make([]byte, int(bufferSize))
|
||||||
|
processed := int64(0)
|
||||||
|
segmentIndex := int64(0)
|
||||||
|
segmentsTransformed := int64(0)
|
||||||
|
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
cleanup()
|
||||||
|
return r.jsError("patterned file transform cancelled: %v", err)
|
||||||
|
}
|
||||||
|
readCount, readErr := io.ReadFull(input, buffer)
|
||||||
|
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
|
||||||
|
cleanup()
|
||||||
|
return r.jsError("failed to read input file: %v", readErr)
|
||||||
|
}
|
||||||
|
if readCount == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
chunk := buffer[:readCount]
|
||||||
|
for offset := 0; offset < readCount; offset += int(segmentSize) {
|
||||||
|
end := min(offset+int(segmentSize), readCount)
|
||||||
|
segment := chunk[offset:end]
|
||||||
|
selected := segmentIndex%transformEvery == transformOffset
|
||||||
|
fullSegment := len(segment) == int(segmentSize)
|
||||||
|
if selected && (fullSegment || transformPartial) {
|
||||||
|
if parsedCipher.Mode == "cbc" && len(segment)%block.BlockSize() != 0 {
|
||||||
|
cleanup()
|
||||||
|
return r.jsError("selected segment %d is not a multiple of %d bytes", segmentIndex, block.BlockSize())
|
||||||
|
}
|
||||||
|
transformPatternedSegment(block, parsedCipher, operation, segment)
|
||||||
|
segmentsTransformed++
|
||||||
|
}
|
||||||
|
segmentIndex++
|
||||||
|
}
|
||||||
|
|
||||||
|
written, writeErr := output.Write(chunk)
|
||||||
|
if writeErr != nil || written != len(chunk) {
|
||||||
|
if writeErr == nil {
|
||||||
|
writeErr = io.ErrShortWrite
|
||||||
|
}
|
||||||
|
cleanup()
|
||||||
|
return r.jsError("failed to write transformed file: %v", writeErr)
|
||||||
|
}
|
||||||
|
processed += int64(written)
|
||||||
|
if onProgress != nil {
|
||||||
|
if _, callbackErr := onProgress(
|
||||||
|
goja.Undefined(),
|
||||||
|
r.vm.ToValue(processed),
|
||||||
|
r.vm.ToValue(totalSize),
|
||||||
|
); callbackErr != nil {
|
||||||
|
cleanup()
|
||||||
|
return r.jsError("progress callback failed: %v", callbackErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := input.Close(); err != nil {
|
||||||
|
output.Close()
|
||||||
|
_ = os.Remove(stagedPath)
|
||||||
|
return r.jsError("failed to close input file: %v", err)
|
||||||
|
}
|
||||||
|
if err := output.Sync(); err != nil {
|
||||||
|
output.Close()
|
||||||
|
_ = os.Remove(stagedPath)
|
||||||
|
return r.jsError("failed to sync transformed file: %v", err)
|
||||||
|
}
|
||||||
|
if err := output.Close(); err != nil {
|
||||||
|
_ = os.Remove(stagedPath)
|
||||||
|
return r.jsError("failed to close transformed file: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(stagedPath, outputPath); err != nil {
|
||||||
|
_ = os.Remove(stagedPath)
|
||||||
|
return r.jsError("failed to publish transformed file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r.jsSuccess(map[string]any{
|
||||||
|
"path": outputPath,
|
||||||
|
"bytes_processed": processed,
|
||||||
|
"segments_processed": segmentIndex,
|
||||||
|
"segments_transformed": segmentsTransformed,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func transformPatternedSegment(
|
||||||
|
block cipher.Block,
|
||||||
|
options *runtimeBlockCipherOptions,
|
||||||
|
operation string,
|
||||||
|
segment []byte,
|
||||||
|
) {
|
||||||
|
if options.Mode == "ctr" {
|
||||||
|
cipher.NewCTR(block, options.IV).XORKeyStream(segment, segment)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if operation == "encrypt" {
|
||||||
|
cipher.NewCBCEncrypter(block, options.IV).CryptBlocks(segment, segment)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cipher.NewCBCDecrypter(block, options.IV).CryptBlocks(segment, segment)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user