SpotiFLAC-Mobile/go_backend/library_scan.go

package gobackend

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

type LibraryScanResult struct {
	ID                   string `json:"id"`
	TrackName            string `json:"trackName"`
	ArtistName           string `json:"artistName"`
	AlbumName            string `json:"albumName"`
	AlbumArtist          string `json:"albumArtist,omitempty"`
	FilePath             string `json:"filePath"`
	CoverPath            string `json:"coverPath,omitempty"`
	ScannedAt            string `json:"scannedAt"`
	FileModTime          int64  `json:"fileModTime,omitempty"` // Unix timestamp in milliseconds
	ISRC                 string `json:"isrc,omitempty"`
	TrackNumber          int    `json:"trackNumber,omitempty"`
	TotalTracks          int    `json:"totalTracks,omitempty"`
	DiscNumber           int    `json:"discNumber,omitempty"`
	TotalDiscs           int    `json:"totalDiscs,omitempty"`
	Duration             int    `json:"duration,omitempty"`
	ReleaseDate          string `json:"releaseDate,omitempty"`
	BitDepth             int    `json:"bitDepth,omitempty"`
	SampleRate           int    `json:"sampleRate,omitempty"`
	Bitrate              int    `json:"bitrate,omitempty"` // average kbps for both lossless and lossy audio
	Genre                string `json:"genre,omitempty"`
	Composer             string `json:"composer,omitempty"`
	Label                string `json:"label,omitempty"`
	Copyright            string `json:"copyright,omitempty"`
	Comment              string `json:"comment,omitempty"`
	AlbumType            string `json:"albumType,omitempty"`
	Explicit             bool   `json:"explicit,omitempty"`
	UPC                  string `json:"upc,omitempty"`
	Format               string `json:"format,omitempty"`
	MetadataFromFilename bool   `json:"metadataFromFilename,omitempty"`
}

type LibraryScanProgress struct {
	TotalFiles   int     `json:"total_files"`
	ScannedFiles int     `json:"scanned_files"`
	CurrentFile  string  `json:"current_file"`
	ErrorCount   int     `json:"error_count"`
	ProgressPct  float64 `json:"progress_pct"`
	IsComplete   bool    `json:"is_complete"`
}

type IncrementalScanResult struct {
	Scanned      []LibraryScanResult `json:"scanned"`      // New or updated files
	DeletedPaths []string            `json:"deletedPaths"` // Files that no longer exist
	SkippedCount int                 `json:"skippedCount"` // Files that were unchanged
	TotalFiles   int                 `json:"totalFiles"`   // Total files in folder
}

var (
	libraryScanProgress   LibraryScanProgress
	libraryScanProgressMu sync.RWMutex
	libraryScanCancel     chan struct{}
	libraryScanCancelMu   sync.Mutex
	libraryCoverCacheDir  string
	libraryCoverCacheMu   sync.RWMutex
)

var supportedAudioFormats = map[string]bool{
	".flac": true,
	".m4a":  true,
	".mp4":  true,
	".aac":  true,
	".mp3":  true,
	".opus": true,
	".ogg":  true,
	".ape":  true,
	".wv":   true,
	".mpc":  true,
	".wav":  true,
	".aiff": true,
	".aif":  true,
	".cue":  true,
}

type libraryAudioFileInfo struct {
	path    string
	modTime int64
	size    int64
}

type scannedCueFileInfo struct {
	sheet     *CueSheet
	audioPath string
}

type libraryScanTask struct {
	index int
	info  libraryAudioFileInfo
}

type libraryScanTaskResult struct {
	index   int
	path    string
	results []LibraryScanResult
	err     error
}

func isLibraryStagingFile(path string) bool {
	name := strings.ToLower(filepath.Base(path))
	if strings.HasSuffix(name, ".partial") {
		return true
	}
	for ext := range supportedAudioFormats {
		if strings.HasSuffix(name, ".partial"+ext) {
			return true
		}
	}
	return false
}

func collectLibraryAudioFiles(folderPath string, cancelCh <-chan struct{}) ([]libraryAudioFileInfo, error) {
	var files []libraryAudioFileInfo

	err := filepath.WalkDir(folderPath, func(path string, entry os.DirEntry, err error) error {
		if err != nil {
			return fmt.Errorf("walk library path %s: %w", path, err)
		}

		select {
		case <-cancelCh:
			return fmt.Errorf("scan cancelled")
		default:
		}

		if entry.IsDir() {
			return nil
		}
		if isLibraryStagingFile(path) {
			return nil
		}

		ext := strings.ToLower(filepath.Ext(path))
		if !supportedAudioFormats[ext] {
			return nil
		}

		info, err := entry.Info()
		if err != nil {
			return fmt.Errorf("read library file info %s: %w", path, err)
		}

		files = append(files, libraryAudioFileInfo{
			path:    path,
			modTime: info.ModTime().UnixMilli(),
			size:    info.Size(),
		})
		return nil
	})

	if err != nil {
		return nil, err
	}

	return files, nil
}

func libraryAudioCoverCacheKey(info libraryAudioFileInfo) string {
	return fmt.Sprintf("%s|%d|%d", info.path, info.size, info.modTime)
}

func libraryScanWorkerCount(taskCount int) int {
	if taskCount < 16 {
		return 1
	}
	workers := runtime.NumCPU()
	if workers > 4 {
		workers = 4
	}
	if workers < 2 {
		workers = 2
	}
	if workers > taskCount {
		workers = taskCount
	}
	return workers
}

func updateLibraryScanProgress(scannedFiles, totalFiles int, currentPath string) {
	libraryScanProgressMu.Lock()
	libraryScanProgress.ScannedFiles = scannedFiles
	libraryScanProgress.CurrentFile = filepath.Base(currentPath)
	if totalFiles > 0 {
		libraryScanProgress.ProgressPct = float64(scannedFiles) / float64(totalFiles) * 100
	}
	libraryScanProgressMu.Unlock()
}

func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, cancelCh <-chan struct{}, totalFiles int, completed *int) (map[int][]LibraryScanResult, int, error) {
	return scanLibraryAudioTasksParallelWithSink(
		tasks,
		scanTime,
		cancelCh,
		totalFiles,
		completed,
		nil,
	)
}

func scanLibraryAudioTasksParallelWithSink(
	tasks []libraryScanTask,
	scanTime string,
	cancelCh <-chan struct{},
	totalFiles int,
	completed *int,
	sink func([]LibraryScanResult) error,
) (map[int][]LibraryScanResult, int, error) {
	var resultsByIndex map[int][]LibraryScanResult
	if sink == nil {
		resultsByIndex = make(map[int][]LibraryScanResult, len(tasks))
	}
	if len(tasks) == 0 {
		return resultsByIndex, 0, nil
	}

	workers := libraryScanWorkerCount(len(tasks))
	if workers <= 1 {
		errorCount := 0
		for _, task := range tasks {
			select {
			case <-cancelCh:
				return resultsByIndex, errorCount, fmt.Errorf("scan cancelled")
			default:
			}
			result, err := scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(
				task.info.path,
				"",
				libraryAudioCoverCacheKey(task.info),
				scanTime,
				task.info.modTime,
			)
			*completed++
			updateLibraryScanProgress(*completed, totalFiles, task.info.path)
			if err != nil {
				errorCount++
				GoLog("[LibraryScan] Error scanning %s: %v\n", task.info.path, err)
				continue
			}
			results := []LibraryScanResult{*result}
			if sink != nil {
				if err := sink(results); err != nil {
					return resultsByIndex, errorCount, err
				}
			} else {
				resultsByIndex[task.index] = results
			}
		}
		return resultsByIndex, errorCount, nil
	}

	taskCh := make(chan libraryScanTask)
	resultCh := make(chan libraryScanTaskResult, workers)
	var wg sync.WaitGroup

	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for task := range taskCh {
				select {
				case <-cancelCh:
					return
				default:
				}
				result, err := scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(
					task.info.path,
					"",
					libraryAudioCoverCacheKey(task.info),
					scanTime,
					task.info.modTime,
				)
				taskResult := libraryScanTaskResult{
					index: task.index,
					path:  task.info.path,
					err:   err,
				}
				if err == nil && result != nil {
					taskResult.results = []LibraryScanResult{*result}
				}
				select {
				case <-cancelCh:
					return
				case resultCh <- taskResult:
				}
			}
		}()
	}

	go func() {
		defer close(taskCh)
		for _, task := range tasks {
			select {
			case <-cancelCh:
				return
			case taskCh <- task:
			}
		}
	}()

	go func() {
		wg.Wait()
		close(resultCh)
	}()

	errorCount := 0
	var sinkErr error
	for taskResult := range resultCh {
		*completed++
		updateLibraryScanProgress(*completed, totalFiles, taskResult.path)
		if taskResult.err != nil {
			errorCount++
			GoLog("[LibraryScan] Error scanning %s: %v\n", taskResult.path, taskResult.err)
			continue
		}
		if sink != nil {
			if sinkErr == nil {
				sinkErr = sink(taskResult.results)
			}
		} else {
			resultsByIndex[taskResult.index] = taskResult.results
		}
	}
	if sinkErr != nil {
		return resultsByIndex, errorCount, sinkErr
	}

	select {
	case <-cancelCh:
		return resultsByIndex, errorCount, fmt.Errorf("scan cancelled")
	default:
	}
	return resultsByIndex, errorCount, nil
}

func SetLibraryCoverCacheDir(cacheDir string) {
	libraryCoverCacheMu.Lock()
	libraryCoverCacheDir = cacheDir
	libraryCoverCacheMu.Unlock()
}

func scanLibraryFolderWithSink(
	folderPath string,
	sink func(LibraryScanResult) error,
	preserveOrder bool,
) (int, error) {
	if folderPath == "" {
		return 0, fmt.Errorf("folder path is empty")
	}

	info, err := os.Stat(folderPath)
	if err != nil {
		return 0, fmt.Errorf("folder not found: %w", err)
	}
	if !info.IsDir() {
		return 0, fmt.Errorf("path is not a folder: %s", folderPath)
	}

	libraryScanProgressMu.Lock()
	libraryScanProgress = LibraryScanProgress{}
	libraryScanProgressMu.Unlock()

	libraryScanCancelMu.Lock()
	if libraryScanCancel != nil {
		close(libraryScanCancel)
	}
	libraryScanCancel = make(chan struct{})
	cancelCh := libraryScanCancel
	libraryScanCancelMu.Unlock()

	audioFileInfos, err := collectLibraryAudioFiles(folderPath, cancelCh)
	if err != nil {
		return 0, err
	}

	totalFiles := len(audioFileInfos)
	libraryScanProgressMu.Lock()
	libraryScanProgress.TotalFiles = totalFiles
	libraryScanProgressMu.Unlock()

	if totalFiles == 0 {
		libraryScanProgressMu.Lock()
		libraryScanProgress.IsComplete = true
		libraryScanProgress.ProgressPct = 100
		libraryScanProgressMu.Unlock()
		return 0, nil
	}

	GoLog("[LibraryScan] Found %d audio files to scan\n", totalFiles)

	scanTime := time.Now().UTC().Format(time.RFC3339)
	errorCount := 0
	emittedCount := 0
	emitResults := func(results []LibraryScanResult) error {
		for i := range results {
			if err := sink(results[i]); err != nil {
				return err
			}
			emittedCount++
		}
		return nil
	}

	cueReferencedAudioFiles := make(map[string]bool)
	parsedCueFiles := make(map[string]scannedCueFileInfo)
	for _, fileInfo := range audioFileInfos {
		filePath := fileInfo.path
		if strings.ToLower(filepath.Ext(filePath)) != ".cue" {
			continue
		}
		sheet, parseErr := ParseCueFile(filePath)
		if parseErr == nil && sheet.FileName != "" {
			audioPath := ResolveCueAudioPath(filePath, sheet.FileName)
			if audioPath != "" {
				parsedCueFiles[filePath] = scannedCueFileInfo{
					sheet:     sheet,
					audioPath: audioPath,
				}
				cueReferencedAudioFiles[audioPath] = true
			}
		}
	}

	audioTasks := make([]libraryScanTask, 0, totalFiles)
	var orderedResults map[int][]LibraryScanResult
	if preserveOrder {
		orderedResults = make(map[int][]LibraryScanResult, totalFiles)
	}
	completedFiles := 0
	for i, fileInfo := range audioFileInfos {
		filePath := fileInfo.path
		select {
		case <-cancelCh:
			return emittedCount, fmt.Errorf("scan cancelled")
		default:
		}

		ext := strings.ToLower(filepath.Ext(filePath))
		if ext == ".cue" {
			var cueResults []LibraryScanResult
			cueInfo, ok := parsedCueFiles[filePath]
			if ok {
				cueResults, err = scanCueSheetForLibrary(
					filePath,
					cueInfo.sheet,
					cueInfo.audioPath,
					"",
					fileInfo.modTime,
					"",
					scanTime,
				)
			} else {
				cueResults, err = ScanCueFileForLibrary(filePath, scanTime)
			}
			completedFiles++
			updateLibraryScanProgress(completedFiles, totalFiles, filePath)
			if err != nil {
				errorCount++
				GoLog("[LibraryScan] Error scanning cue %s: %v\n", filePath, err)
				continue
			}
			if preserveOrder {
				orderedResults[i] = cueResults
			} else if err := emitResults(cueResults); err != nil {
				return emittedCount, fmt.Errorf("write scan result: %w", err)
			}
			GoLog("[LibraryScan] CUE sheet %s: %d tracks\n", filepath.Base(filePath), len(cueResults))
			continue
		}

		if cueReferencedAudioFiles[filePath] {
			completedFiles++
			updateLibraryScanProgress(completedFiles, totalFiles, filePath)
			GoLog("[LibraryScan] Skipping %s (referenced by .cue sheet)\n", filepath.Base(filePath))
			continue
		}

		audioTasks = append(audioTasks, libraryScanTask{index: i, info: fileInfo})
	}

	var audioSink func([]LibraryScanResult) error
	if !preserveOrder {
		audioSink = emitResults
	}
	audioResults, audioErrors, err := scanLibraryAudioTasksParallelWithSink(
		audioTasks,
		scanTime,
		cancelCh,
		totalFiles,
		&completedFiles,
		audioSink,
	)
	if err != nil {
		return emittedCount, err
	}
	errorCount += audioErrors
	if preserveOrder {
		for index, results := range audioResults {
			orderedResults[index] = results
		}
		for i := range audioFileInfos {
			if err := emitResults(orderedResults[i]); err != nil {
				return emittedCount, fmt.Errorf("write scan result: %w", err)
			}
		}
	}

	libraryScanProgressMu.Lock()
	libraryScanProgress.ErrorCount = errorCount
	libraryScanProgress.IsComplete = true
	libraryScanProgress.ScannedFiles = totalFiles
	libraryScanProgress.ProgressPct = 100
	libraryScanProgressMu.Unlock()

	GoLog("[LibraryScan] Scan complete: %d tracks found, %d errors\n", emittedCount, errorCount)
	return emittedCount, nil
}

func ScanLibraryFolder(folderPath string) (string, error) {
	results := make([]LibraryScanResult, 0)
	_, err := scanLibraryFolderWithSink(folderPath, func(result LibraryScanResult) error {
		results = append(results, result)
		return nil
	}, true)
	if err != nil {
		return "[]", err
	}

	jsonBytes, err := json.Marshal(results)
	if err != nil {
		return "[]", fmt.Errorf("failed to marshal results: %w", err)
	}

	return string(jsonBytes), nil
}

// ScanLibraryFolderToNDJSONFile writes one JSON object per line so mobile
// clients can decode and ingest bounded batches instead of materializing a
// full-library JSON array in both the Go and Dart heaps.
func ScanLibraryFolderToNDJSONFile(folderPath, outputPath string) (int, error) {
	if outputPath == "" {
		return 0, fmt.Errorf("output path is empty")
	}
	file, err := os.Create(outputPath)
	if err != nil {
		return 0, fmt.Errorf("create scan output: %w", err)
	}
	removeOnError := true
	defer func() {
		_ = file.Close()
		if removeOnError {
			_ = os.Remove(outputPath)
		}
	}()

	writer := bufio.NewWriterSize(file, 64*1024)
	encoder := json.NewEncoder(writer)
	count, err := scanLibraryFolderWithSink(folderPath, func(result LibraryScanResult) error {
		return encoder.Encode(result)
	}, false)
	if err != nil {
		return count, err
	}
	if err := writer.Flush(); err != nil {
		return count, fmt.Errorf("flush scan output: %w", err)
	}
	if err := file.Close(); err != nil {
		return count, fmt.Errorf("close scan output: %w", err)
	}
	removeOnError = false
	return count, nil
}

func GetLibraryScanProgress() string {
	libraryScanProgressMu.RLock()
	defer libraryScanProgressMu.RUnlock()

	jsonBytes, _ := json.Marshal(libraryScanProgress)
	return string(jsonBytes)
}

func CancelLibraryScan() {
	libraryScanCancelMu.Lock()
	defer libraryScanCancelMu.Unlock()

	if libraryScanCancel != nil {
		close(libraryScanCancel)
		libraryScanCancel = nil
	}
}

func ReadAudioMetadata(filePath string) (string, error) {
	return ReadAudioMetadataWithDisplayName(filePath, "")
}

func ReadAudioMetadataWithDisplayName(filePath, displayNameHint string) (string, error) {
	return ReadAudioMetadataWithDisplayNameAndCoverCacheKey(filePath, displayNameHint, "")
}

func ReadAudioMetadataWithDisplayNameAndCoverCacheKey(filePath, displayNameHint, coverCacheKey string) (string, error) {
	scanTime := time.Now().UTC().Format(time.RFC3339)
	result, err := scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(
		filePath,
		displayNameHint,
		coverCacheKey,
		scanTime,
		0,
	)
	if err != nil {
		return "", err
	}

	jsonBytes, err := json.Marshal(result)
	if err != nil {
		return "", fmt.Errorf("failed to marshal result: %w", err)
	}

	return string(jsonBytes), nil
}