perf(library): stream scans and optimize queue queries

go_backend/exports_library.go

@@ -8,6 +8,10 @@ func ScanLibraryFolderJSON(folderPath string) (string, error) {
 	return ScanLibraryFolder(folderPath)
 }
 
+func ScanLibraryFolderToNDJSONFileJSON(folderPath, outputPath string) (int, error) {
+	return ScanLibraryFolderToNDJSONFile(folderPath, outputPath)
+}
+
 func ScanLibraryFolderIncrementalJSON(folderPath, existingFilesJSON string) (string, error) {
 	return ScanLibraryFolderIncremental(folderPath, existingFilesJSON)
 }

go_backend/httputil_utls.go

@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"strings"
 	"sync"
+	"time"
 
 	utls "github.com/refraction-networking/utls"
 	"golang.org/x/net/http2"
@@ -25,7 +26,15 @@ type utlsTransport struct {
 	dialer *net.Dialer
 	h2     *http2.Transport
 	mu     sync.Mutex
-	conns  map[string]*http2.ClientConn
+	conns  map[string]pooledHTTP2ClientConn
+}
+
+type pooledHTTP2ClientConn interface {
+	RoundTrip(*http.Request) (*http.Response, error)
+	ReserveNewRequest() bool
+	State() http2.ClientConnState
+	Close() error
+	Shutdown(context.Context) error
 }
 
 func newUTLSTransport() *utlsTransport {
@@ -35,7 +44,7 @@ func newUTLSTransport() *utlsTransport {
 			KeepAlive: 30 * Second,
 		},
 		h2:    &http2.Transport{},
-		conns: make(map[string]*http2.ClientConn),
+		conns: make(map[string]pooledHTTP2ClientConn),
 	}
 }
 
@@ -52,6 +61,9 @@ func (t *utlsTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 		if err == nil {
 			return resp, nil
 		}
+		if req.Context().Err() != nil {
+			return nil, err
+		}
 		// A pooled conn can be silently dead after a network switch. Drop it
 		// and, when the request is safely repeatable, fall through to a fresh
 		// dial instead of failing where the old dial-per-request code would
@@ -85,8 +97,8 @@ func (t *utlsTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 		tlsConn.Close()
 		return nil, err
 	}
-	cc = t.storeConn(addr, cc)
-	return cc.RoundTrip(req)
+	pooled := t.storeConn(addr, cc)
+	return pooled.RoundTrip(req)
 }
 
 // rewindRequestBody returns a request whose body can be sent again after a
@@ -132,37 +144,99 @@ func (t *utlsTransport) dial(ctx context.Context, host, addr string) (*utls.UCon
 	return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol, nil
 }
 
-func (t *utlsTransport) cachedConn(addr string) *http2.ClientConn {
+func (t *utlsTransport) cachedConn(addr string) pooledHTTP2ClientConn {
 	t.mu.Lock()
-	defer t.mu.Unlock()
-	if cc := t.conns[addr]; cc != nil && cc.CanTakeNewRequest() {
+	cc := t.conns[addr]
+	if cc != nil && cc.ReserveNewRequest() {
+		t.mu.Unlock()
 		return cc
 	}
+	if cc != nil {
+		delete(t.conns, addr)
+	}
+	t.mu.Unlock()
+	if cc != nil {
+		retirePooledHTTP2Conn(cc)
+	}
 	return nil
 }
 
-func (t *utlsTransport) invalidate(addr string, cc *http2.ClientConn) {
+func (t *utlsTransport) invalidate(addr string, cc pooledHTTP2ClientConn) {
 	t.mu.Lock()
+	removed := false
 	if t.conns[addr] == cc {
 		delete(t.conns, addr)
+		removed = true
 	}
 	t.mu.Unlock()
+	if removed {
+		retirePooledHTTP2Conn(cc)
+	}
 }
 
 // storeConn caches cc, but if a concurrent dial already cached a healthy conn for
 // addr it discards the freshly built cc (no in-flight requests) and returns the
 // existing one, avoiding a leaked connection.
-func (t *utlsTransport) storeConn(addr string, cc *http2.ClientConn) *http2.ClientConn {
+func (t *utlsTransport) storeConn(addr string, cc pooledHTTP2ClientConn) pooledHTTP2ClientConn {
 	t.mu.Lock()
-	defer t.mu.Unlock()
-	if existing := t.conns[addr]; existing != nil && existing.CanTakeNewRequest() {
-		cc.Close()
+	if existing := t.conns[addr]; existing != nil && existing.ReserveNewRequest() {
+		t.mu.Unlock()
+		_ = cc.Close()
 		return existing
 	}
+	stale := t.conns[addr]
 	t.conns[addr] = cc
+	_ = cc.ReserveNewRequest()
+	t.mu.Unlock()
+	if stale != nil {
+		retirePooledHTTP2Conn(stale)
+	}
 	return cc
 }
 
+// retirePooledHTTP2Conn prevents new streams while allowing existing streams
+// to finish. The independent watchdog also bounds Shutdown implementations
+// that block before observing their context.
+func pooledHTTP2RetirementTimeout(state http2.ClientConnState) time.Duration {
+	if state.StreamsActive > 0 || state.StreamsPending > 0 || state.StreamsReserved > 0 {
+		return 0
+	}
+	return 5 * Second
+}
+
+func retirePooledHTTP2Conn(conn pooledHTTP2ClientConn) {
+	go func() {
+		retirePooledHTTP2ConnWithTimeout(conn, pooledHTTP2RetirementTimeout(conn.State()))
+	}()
+}
+
+func retirePooledHTTP2ConnWithTimeout(conn pooledHTTP2ClientConn, timeout time.Duration) {
+	go func() {
+		var closeOnce sync.Once
+		forceClose := func() {
+			closeOnce.Do(func() { _ = conn.Close() })
+		}
+		if timeout <= 0 {
+			if err := conn.Shutdown(context.Background()); err != nil {
+				forceClose()
+			}
+			return
+		}
+
+		watchdogDone := make(chan struct{})
+		watchdog := time.AfterFunc(timeout, func() {
+			forceClose()
+			close(watchdogDone)
+		})
+		if err := conn.Shutdown(context.Background()); err != nil {
+			forceClose()
+		}
+		if !watchdog.Stop() {
+			<-watchdogDone
+		}
+	}()
+}
+
 // closeIdleConnections drops every pooled conn so the next request re-dials —
 // needed after a network switch, where pooled conns are silently dead and the
 // first request would otherwise hang on one until its timeout. Conns are shut
@@ -170,10 +244,10 @@ func (t *utlsTransport) storeConn(addr string, cc *http2.ClientConn) *http2.Clie
 func (t *utlsTransport) closeIdleConnections() {
 	t.mu.Lock()
 	conns := t.conns
-	t.conns = make(map[string]*http2.ClientConn)
+	t.conns = make(map[string]pooledHTTP2ClientConn)
 	t.mu.Unlock()
 	for _, cc := range conns {
-		go cc.Shutdown(context.Background())
+		retirePooledHTTP2Conn(cc)
 	}
 }
 

go_backend/httputil_utls_pool_test.go

@@ -0,0 +1,187 @@
+//go:build !ios
+
+package gobackend
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+)
+
+type fakePooledHTTP2Conn struct {
+	healthy             bool
+	streamsActive       int
+	blockShutdown       bool
+	ignoreShutdownCtx   bool
+	closeCount          atomic.Int32
+	shutdownCount       atomic.Int32
+	shutdownOnce        sync.Once
+	shutdownDone        chan struct{}
+	forceCloseUnblocked chan struct{}
+	forceCloseOnce      sync.Once
+}
+
+func newFakePooledHTTP2Conn(healthy bool) *fakePooledHTTP2Conn {
+	return &fakePooledHTTP2Conn{
+		healthy:             healthy,
+		shutdownDone:        make(chan struct{}),
+		forceCloseUnblocked: make(chan struct{}),
+	}
+}
+
+func (c *fakePooledHTTP2Conn) RoundTrip(*http.Request) (*http.Response, error) {
+	return nil, nil
+}
+
+func (c *fakePooledHTTP2Conn) ReserveNewRequest() bool { return c.healthy }
+
+func (c *fakePooledHTTP2Conn) State() http2.ClientConnState {
+	if c.healthy {
+		return http2.ClientConnState{
+			StreamsActive:        c.streamsActive,
+			MaxConcurrentStreams: 100,
+		}
+	}
+	return http2.ClientConnState{Closing: true, StreamsActive: c.streamsActive}
+}
+
+func (c *fakePooledHTTP2Conn) Close() error {
+	c.closeCount.Add(1)
+	c.forceCloseOnce.Do(func() { close(c.forceCloseUnblocked) })
+	return nil
+}
+
+func (c *fakePooledHTTP2Conn) Shutdown(ctx context.Context) error {
+	c.shutdownCount.Add(1)
+	c.shutdownOnce.Do(func() { close(c.shutdownDone) })
+	if c.ignoreShutdownCtx {
+		<-c.forceCloseUnblocked
+		return context.DeadlineExceeded
+	}
+	if c.blockShutdown {
+		<-ctx.Done()
+		return ctx.Err()
+	}
+	return nil
+}
+
+func waitForShutdown(t *testing.T, conn *fakePooledHTTP2Conn) {
+	t.Helper()
+	select {
+	case <-conn.shutdownDone:
+	case <-time.After(time.Second):
+		t.Fatal("connection did not begin graceful shutdown")
+	}
+}
+
+func TestUTLSPoolRetiresStaleCachedConnection(t *testing.T) {
+	transport := newUTLSTransport()
+	stale := newFakePooledHTTP2Conn(false)
+	transport.conns["example:443"] = stale
+
+	if got := transport.cachedConn("example:443"); got != nil {
+		t.Fatalf("cachedConn returned stale connection: %#v", got)
+	}
+	waitForShutdown(t, stale)
+	if stale.closeCount.Load() != 0 {
+		t.Fatalf("gracefully retired connection was force closed")
+	}
+	if _, exists := transport.conns["example:443"]; exists {
+		t.Fatal("stale connection was not removed")
+	}
+}
+
+func TestUTLSPoolStoreClosesDiscardedAndRetiresReplacedConnection(t *testing.T) {
+	transport := newUTLSTransport()
+	healthy := newFakePooledHTTP2Conn(true)
+	transport.conns["example:443"] = healthy
+	fresh := newFakePooledHTTP2Conn(true)
+
+	if got := transport.storeConn("example:443", fresh); got != healthy {
+		t.Fatal("healthy pooled connection was not reused")
+	}
+	if fresh.closeCount.Load() != 1 {
+		t.Fatalf("discarded fresh close count = %d", fresh.closeCount.Load())
+	}
+
+	stale := newFakePooledHTTP2Conn(false)
+	transport.conns["example:443"] = stale
+	replacement := newFakePooledHTTP2Conn(true)
+	if got := transport.storeConn("example:443", replacement); got != replacement {
+		t.Fatal("stale connection was not replaced")
+	}
+	waitForShutdown(t, stale)
+	if stale.closeCount.Load() != 0 {
+		t.Fatalf("replaced connection was force closed")
+	}
+}
+
+func TestUTLSPoolInvalidateRetiresOnlyRequestedConnection(t *testing.T) {
+	transport := newUTLSTransport()
+	current := newFakePooledHTTP2Conn(true)
+	old := newFakePooledHTTP2Conn(false)
+	transport.conns["example:443"] = current
+
+	transport.invalidate("example:443", old)
+	if transport.conns["example:443"] != current {
+		t.Fatal("invalidating an old connection removed the replacement")
+	}
+	if old.shutdownCount.Load() != 0 {
+		t.Fatal("connection already removed from the pool was retired again")
+	}
+
+	transport.invalidate("example:443", current)
+	waitForShutdown(t, current)
+	if _, exists := transport.conns["example:443"]; exists {
+		t.Fatal("invalidated current connection remained in the pool")
+	}
+}
+
+func TestUTLSPoolCloseIdleUsesBoundedShutdown(t *testing.T) {
+	transport := newUTLSTransport()
+	conn := newFakePooledHTTP2Conn(true)
+	transport.conns["example:443"] = conn
+
+	transport.closeIdleConnections()
+	if len(transport.conns) != 0 {
+		t.Fatal("pool was not cleared synchronously")
+	}
+	select {
+	case <-conn.shutdownDone:
+	case <-time.After(time.Second):
+		t.Fatal("pooled connection was not shut down")
+	}
+	if conn.shutdownCount.Load() != 1 {
+		t.Fatalf("shutdown count = %d", conn.shutdownCount.Load())
+	}
+}
+
+func TestUTLSPoolForcesCloseWhenGracefulShutdownTimesOut(t *testing.T) {
+	conn := newFakePooledHTTP2Conn(true)
+	conn.ignoreShutdownCtx = true
+
+	retirePooledHTTP2ConnWithTimeout(conn, 20*time.Millisecond)
+	waitForShutdown(t, conn)
+
+	deadline := time.After(time.Second)
+	for conn.closeCount.Load() == 0 {
+		select {
+		case <-deadline:
+			t.Fatal("timed-out graceful shutdown did not force close")
+		case <-time.After(10 * time.Millisecond):
+		}
+	}
+}
+
+func TestUTLSPoolDoesNotPutADeadlineOnActiveStreams(t *testing.T) {
+	conn := newFakePooledHTTP2Conn(false)
+	conn.streamsActive = 1
+	if timeout := pooledHTTP2RetirementTimeout(conn.State()); timeout != 0 {
+		t.Fatalf("active connection retirement timeout = %v", timeout)
+	}
+}

go_backend/library_scan.go

@@ -1,6 +1,7 @@
 package gobackend
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -122,7 +123,7 @@ func collectLibraryAudioFiles(folderPath string, cancelCh <-chan struct{}) ([]li
 
 	err := filepath.WalkDir(folderPath, func(path string, entry os.DirEntry, err error) error {
 		if err != nil {
-			return nil
+			return fmt.Errorf("walk library path %s: %w", path, err)
 		}
 
 		select {
@@ -145,7 +146,7 @@ func collectLibraryAudioFiles(folderPath string, cancelCh <-chan struct{}) ([]li
 
 		info, err := entry.Info()
 		if err != nil {
-			return nil
+			return fmt.Errorf("read library file info %s: %w", path, err)
 		}
 
 		files = append(files, libraryAudioFileInfo{
@@ -195,7 +196,28 @@ func updateLibraryScanProgress(scannedFiles, totalFiles int, currentPath string)
 }
 
 func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, cancelCh <-chan struct{}, totalFiles int, completed *int) (map[int][]LibraryScanResult, int, error) {
-	resultsByIndex := make(map[int][]LibraryScanResult, len(tasks))
+	return scanLibraryAudioTasksParallelWithSink(
+		tasks,
+		scanTime,
+		cancelCh,
+		totalFiles,
+		completed,
+		nil,
+	)
+}
+
+func scanLibraryAudioTasksParallelWithSink(
+	tasks []libraryScanTask,
+	scanTime string,
+	cancelCh <-chan struct{},
+	totalFiles int,
+	completed *int,
+	sink func([]LibraryScanResult) error,
+) (map[int][]LibraryScanResult, int, error) {
+	var resultsByIndex map[int][]LibraryScanResult
+	if sink == nil {
+		resultsByIndex = make(map[int][]LibraryScanResult, len(tasks))
+	}
 	if len(tasks) == 0 {
 		return resultsByIndex, 0, nil
 	}
@@ -223,7 +245,14 @@ func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, can
 				GoLog("[LibraryScan] Error scanning %s: %v\n", task.info.path, err)
 				continue
 			}
-			resultsByIndex[task.index] = []LibraryScanResult{*result}
+			results := []LibraryScanResult{*result}
+			if sink != nil {
+				if err := sink(results); err != nil {
+					return resultsByIndex, errorCount, err
+				}
+			} else {
+				resultsByIndex[task.index] = results
+			}
 		}
 		return resultsByIndex, errorCount, nil
 	}
@@ -283,6 +312,7 @@ func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, can
 	}()
 
 	errorCount := 0
+	var sinkErr error
 	for taskResult := range resultCh {
 		*completed++
 		updateLibraryScanProgress(*completed, totalFiles, taskResult.path)
@@ -291,7 +321,16 @@ func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, can
 			GoLog("[LibraryScan] Error scanning %s: %v\n", taskResult.path, taskResult.err)
 			continue
 		}
-		resultsByIndex[taskResult.index] = taskResult.results
+		if sink != nil {
+			if sinkErr == nil {
+				sinkErr = sink(taskResult.results)
+			}
+		} else {
+			resultsByIndex[taskResult.index] = taskResult.results
+		}
+	}
+	if sinkErr != nil {
+		return resultsByIndex, errorCount, sinkErr
 	}
 
 	select {
@@ -308,17 +347,21 @@ func SetLibraryCoverCacheDir(cacheDir string) {
 	libraryCoverCacheMu.Unlock()
 }
 
-func ScanLibraryFolder(folderPath string) (string, error) {
+func scanLibraryFolderWithSink(
+	folderPath string,
+	sink func(LibraryScanResult) error,
+	preserveOrder bool,
+) (int, error) {
 	if folderPath == "" {
-		return "[]", fmt.Errorf("folder path is empty")
+		return 0, fmt.Errorf("folder path is empty")
 	}
 
 	info, err := os.Stat(folderPath)
 	if err != nil {
-		return "[]", fmt.Errorf("folder not found: %w", err)
+		return 0, fmt.Errorf("folder not found: %w", err)
 	}
 	if !info.IsDir() {
-		return "[]", fmt.Errorf("path is not a folder: %s", folderPath)
+		return 0, fmt.Errorf("path is not a folder: %s", folderPath)
 	}
 
 	libraryScanProgressMu.Lock()
@@ -335,7 +378,7 @@ func ScanLibraryFolder(folderPath string) (string, error) {
 
 	audioFileInfos, err := collectLibraryAudioFiles(folderPath, cancelCh)
 	if err != nil {
-		return "[]", err
+		return 0, err
 	}
 
 	totalFiles := len(audioFileInfos)
@@ -346,51 +389,61 @@ func ScanLibraryFolder(folderPath string) (string, error) {
 	if totalFiles == 0 {
 		libraryScanProgressMu.Lock()
 		libraryScanProgress.IsComplete = true
+		libraryScanProgress.ProgressPct = 100
 		libraryScanProgressMu.Unlock()
-		return "[]", nil
+		return 0, nil
 	}
 
 	GoLog("[LibraryScan] Found %d audio files to scan\n", totalFiles)
 
-	results := make([]LibraryScanResult, 0, totalFiles)
 	scanTime := time.Now().UTC().Format(time.RFC3339)
 	errorCount := 0
+	emittedCount := 0
+	emitResults := func(results []LibraryScanResult) error {
+		for i := range results {
+			if err := sink(results[i]); err != nil {
+				return err
+			}
+			emittedCount++
+		}
+		return nil
+	}
 
 	cueReferencedAudioFiles := make(map[string]bool)
 	parsedCueFiles := make(map[string]scannedCueFileInfo)
-
 	for _, fileInfo := range audioFileInfos {
 		filePath := fileInfo.path
-		ext := strings.ToLower(filepath.Ext(filePath))
-		if ext == ".cue" {
-			sheet, err := ParseCueFile(filePath)
-			if err == nil && sheet.FileName != "" {
-				audioPath := ResolveCueAudioPath(filePath, sheet.FileName)
-				if audioPath != "" {
-					parsedCueFiles[filePath] = scannedCueFileInfo{
-						sheet:     sheet,
-						audioPath: audioPath,
-					}
-					cueReferencedAudioFiles[audioPath] = true
+		if strings.ToLower(filepath.Ext(filePath)) != ".cue" {
+			continue
+		}
+		sheet, parseErr := ParseCueFile(filePath)
+		if parseErr == nil && sheet.FileName != "" {
+			audioPath := ResolveCueAudioPath(filePath, sheet.FileName)
+			if audioPath != "" {
+				parsedCueFiles[filePath] = scannedCueFileInfo{
+					sheet:     sheet,
+					audioPath: audioPath,
 				}
+				cueReferencedAudioFiles[audioPath] = true
 			}
 		}
 	}
 
-	resultsByIndex := make(map[int][]LibraryScanResult, totalFiles)
 	audioTasks := make([]libraryScanTask, 0, totalFiles)
+	var orderedResults map[int][]LibraryScanResult
+	if preserveOrder {
+		orderedResults = make(map[int][]LibraryScanResult, totalFiles)
+	}
 	completedFiles := 0
-
 	for i, fileInfo := range audioFileInfos {
 		filePath := fileInfo.path
 		select {
 		case <-cancelCh:
-			return "[]", fmt.Errorf("scan cancelled")
+			return emittedCount, fmt.Errorf("scan cancelled")
 		default:
 		}
 
 		ext := strings.ToLower(filepath.Ext(filePath))
-
 		if ext == ".cue" {
 			var cueResults []LibraryScanResult
 			cueInfo, ok := parsedCueFiles[filePath]
@@ -407,16 +460,18 @@ func ScanLibraryFolder(folderPath string) (string, error) {
 			} else {
 				cueResults, err = ScanCueFileForLibrary(filePath, scanTime)
 			}
+			completedFiles++
+			updateLibraryScanProgress(completedFiles, totalFiles, filePath)
 			if err != nil {
 				errorCount++
 				GoLog("[LibraryScan] Error scanning cue %s: %v\n", filePath, err)
-				completedFiles++
-				updateLibraryScanProgress(completedFiles, totalFiles, filePath)
 				continue
 			}
-			resultsByIndex[i] = cueResults
-			completedFiles++
-			updateLibraryScanProgress(completedFiles, totalFiles, filePath)
+			if preserveOrder {
+				orderedResults[i] = cueResults
+			} else if err := emitResults(cueResults); err != nil {
+				return emittedCount, fmt.Errorf("write scan result: %w", err)
+			}
 			GoLog("[LibraryScan] CUE sheet %s: %d tracks\n", filepath.Base(filePath), len(cueResults))
 			continue
 		}
@@ -431,31 +486,53 @@ func ScanLibraryFolder(folderPath string) (string, error) {
 		audioTasks = append(audioTasks, libraryScanTask{index: i, info: fileInfo})
 	}
 
-	audioResults, audioErrors, err := scanLibraryAudioTasksParallel(
+	var audioSink func([]LibraryScanResult) error
+	if !preserveOrder {
+		audioSink = emitResults
+	}
+	audioResults, audioErrors, err := scanLibraryAudioTasksParallelWithSink(
 		audioTasks,
 		scanTime,
 		cancelCh,
 		totalFiles,
 		&completedFiles,
+		audioSink,
 	)
 	if err != nil {
-		return "[]", err
+		return emittedCount, err
 	}
 	errorCount += audioErrors
-	for index, scanResults := range audioResults {
-		resultsByIndex[index] = scanResults
-	}
-
-	for i := range audioFileInfos {
-		results = append(results, resultsByIndex[i]...)
+	if preserveOrder {
+		for index, results := range audioResults {
+			orderedResults[index] = results
+		}
+		for i := range audioFileInfos {
+			if err := emitResults(orderedResults[i]); err != nil {
+				return emittedCount, fmt.Errorf("write scan result: %w", err)
+			}
+		}
 	}
 
 	libraryScanProgressMu.Lock()
 	libraryScanProgress.ErrorCount = errorCount
 	libraryScanProgress.IsComplete = true
+	libraryScanProgress.ScannedFiles = totalFiles
+	libraryScanProgress.ProgressPct = 100
 	libraryScanProgressMu.Unlock()
 
-	GoLog("[LibraryScan] Scan complete: %d tracks found, %d errors\n", len(results), errorCount)
+	GoLog("[LibraryScan] Scan complete: %d tracks found, %d errors\n", emittedCount, errorCount)
+	return emittedCount, nil
+}
+
+func ScanLibraryFolder(folderPath string) (string, error) {
+	results := make([]LibraryScanResult, 0)
+	_, err := scanLibraryFolderWithSink(folderPath, func(result LibraryScanResult) error {
+		results = append(results, result)
+		return nil
+	}, true)
+	if err != nil {
+		return "[]", err
+	}
 
 	jsonBytes, err := json.Marshal(results)
 	if err != nil {
@@ -465,6 +542,43 @@ func ScanLibraryFolder(folderPath string) (string, error) {
 	return string(jsonBytes), nil
 }
 
+// ScanLibraryFolderToNDJSONFile writes one JSON object per line so mobile
+// clients can decode and ingest bounded batches instead of materializing a
+// full-library JSON array in both the Go and Dart heaps.
+func ScanLibraryFolderToNDJSONFile(folderPath, outputPath string) (int, error) {
+	if outputPath == "" {
+		return 0, fmt.Errorf("output path is empty")
+	}
+	file, err := os.Create(outputPath)
+	if err != nil {
+		return 0, fmt.Errorf("create scan output: %w", err)
+	}
+	removeOnError := true
+	defer func() {
+		_ = file.Close()
+		if removeOnError {
+			_ = os.Remove(outputPath)
+		}
+	}()
+
+	writer := bufio.NewWriterSize(file, 64*1024)
+	encoder := json.NewEncoder(writer)
+	count, err := scanLibraryFolderWithSink(folderPath, func(result LibraryScanResult) error {
+		return encoder.Encode(result)
+	}, false)
+	if err != nil {
+		return count, err
+	}
+	if err := writer.Flush(); err != nil {
+		return count, fmt.Errorf("flush scan output: %w", err)
+	}
+	if err := file.Close(); err != nil {
+		return count, fmt.Errorf("close scan output: %w", err)
+	}
+	removeOnError = false
+	return count, nil
+}
+
 func GetLibraryScanProgress() string {
 	libraryScanProgressMu.RLock()
 	defer libraryScanProgressMu.RUnlock()

go_backend/library_scan_supplement_test.go

@@ -1,7 +1,9 @@
 package gobackend
 
 import (
+	"bufio"
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -89,6 +91,35 @@ func TestLibraryScanFullIncrementalAndMetadataFallbacks(t *testing.T) {
 	if !foundTagged {
 		t.Fatalf("tagged APE not found in %#v", results)
 	}
+
+	ndjsonPath := filepath.Join(t.TempDir(), "library.ndjson")
+	streamedCount, err := ScanLibraryFolderToNDJSONFile(dir, ndjsonPath)
+	if err != nil {
+		t.Fatalf("ScanLibraryFolderToNDJSONFile: %v", err)
+	}
+	ndjsonFile, err := os.Open(ndjsonPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ndjsonFile.Close()
+	decodedCount := 0
+	scanner := bufio.NewScanner(ndjsonFile)
+	for scanner.Scan() {
+		var result LibraryScanResult
+		if err := json.Unmarshal(scanner.Bytes(), &result); err != nil {
+			t.Fatalf("decode NDJSON row: %v", err)
+		}
+		if result.FilePath == "" {
+			t.Fatal("NDJSON row has no file path")
+		}
+		decodedCount++
+	}
+	if err := scanner.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if decodedCount != streamedCount || decodedCount != len(results) {
+		t.Fatalf("NDJSON counts = decoded:%d streamed:%d array:%d", decodedCount, streamedCount, len(results))
+	}
 	if progress := GetLibraryScanProgress(); !strings.Contains(progress, `"IsComplete":true`) && !strings.Contains(progress, `"is_complete":true`) {
 		t.Fatalf("progress = %s", progress)
 	}
@@ -161,3 +192,31 @@ func TestLibraryScanFullIncrementalAndMetadataFallbacks(t *testing.T) {
 	CancelLibraryScan()
 	SetLibraryCoverCacheDir("")
 }
+
+func TestScanLibraryFolderPreservesFileOrderWithParallelWorkers(t *testing.T) {
+	dir := t.TempDir()
+	for i := 0; i < 20; i++ {
+		name := fmt.Sprintf("%02d - Track.mp3", i)
+		if err := os.WriteFile(filepath.Join(dir, name), []byte("not really mp3"), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	jsonText, err := ScanLibraryFolder(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var results []LibraryScanResult
+	if err := json.Unmarshal([]byte(jsonText), &results); err != nil {
+		t.Fatal(err)
+	}
+	if len(results) != 20 {
+		t.Fatalf("results = %d", len(results))
+	}
+	for i, result := range results {
+		expected := filepath.Join(dir, fmt.Sprintf("%02d - Track.mp3", i))
+		if result.FilePath != expected {
+			t.Fatalf("result %d path = %q, want %q", i, result.FilePath, expected)
+		}
+	}
+}