From bd2bd85c9e9dc70e34ec34fb69a0eb94737ae0f3 Mon Sep 17 00:00:00 2001 From: zarzet Date: Mon, 27 Jul 2026 00:21:04 +0700 Subject: [PATCH] refactor(go): split per-format scanners and incremental scan out of library_scan.go --- go_backend/library_scan.go | 665 ------------------------- go_backend/library_scan_formats.go | 392 +++++++++++++++ go_backend/library_scan_incremental.go | 291 +++++++++++ 3 files changed, 683 insertions(+), 665 deletions(-) create mode 100644 go_backend/library_scan_formats.go create mode 100644 go_backend/library_scan_incremental.go diff --git a/go_backend/library_scan.go b/go_backend/library_scan.go index 757b13ae..349ab95a 100644 --- a/go_backend/library_scan.go +++ b/go_backend/library_scan.go @@ -1,13 +1,11 @@ package gobackend import ( - "bufio" "encoding/json" "fmt" "os" "path/filepath" "runtime" - "strconv" "strings" "sync" "time" @@ -467,390 +465,6 @@ func ScanLibraryFolder(folderPath string) (string, error) { return string(jsonBytes), nil } -func scanAudioFileWithKnownModTime(filePath, scanTime string, knownModTime int64) (*LibraryScanResult, error) { - return scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(filePath, "", "", scanTime, knownModTime) -} - -func scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(filePath, displayNameHint, coverCacheKey, scanTime string, knownModTime int64) (*LibraryScanResult, error) { - ext := resolveLibraryAudioExt(filePath, displayNameHint) - - result := &LibraryScanResult{ - ID: generateLibraryID(filePath), - FilePath: filePath, - ScannedAt: scanTime, - Format: strings.TrimPrefix(ext, "."), - } - - if knownModTime > 0 { - result.FileModTime = knownModTime - } else if info, err := os.Stat(filePath); err == nil { - result.FileModTime = info.ModTime().UnixMilli() - } - - libraryCoverCacheMu.RLock() - coverCacheDir := libraryCoverCacheDir - libraryCoverCacheMu.RUnlock() - if ext == ".flac" { - return scanFLACFileWithCoverCache(filePath, result, displayNameHint, coverCacheDir, coverCacheKey) - } - if ext == ".m4a" || ext == ".mp4" || ext == ".aac" { - return scanM4AFileWithCoverCache(filePath, result, displayNameHint, coverCacheDir, coverCacheKey) - } - if coverCacheDir != "" { - coverPath, err := SaveCoverToCacheWithHintAndKey( - filePath, - displayNameHint, - coverCacheDir, - coverCacheKey, - ) - if err == nil && coverPath != "" { - result.CoverPath = coverPath - } - } - - switch ext { - case ".mp3": - return scanMP3File(filePath, result, displayNameHint) - case ".opus", ".ogg": - return scanOggFile(filePath, result, displayNameHint) - case ".ape", ".wv", ".mpc": - return scanAPEFile(filePath, result, displayNameHint) - case ".wav": - return scanWAVFile(filePath, result, displayNameHint) - case ".aiff", ".aif", ".aifc": - return scanAIFFFile(filePath, result, displayNameHint) - default: - return scanFromFilename(filePath, displayNameHint, result) - } -} - -func embeddedCoverMIME(data []byte) string { - if len(data) >= 8 && - data[0] == 0x89 && - data[1] == 0x50 && - data[2] == 0x4e && - data[3] == 0x47 { - return "image/png" - } - return "image/jpeg" -} - -func cacheScannedCover(filePath, cacheDir, coverCacheKey string, coverData []byte) string { - if cacheDir == "" || len(coverData) == 0 { - return "" - } - cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) - path, err := saveLibraryCoverDataToCache(cacheDir, cacheKey, coverData, embeddedCoverMIME(coverData)) - if err != nil { - return "" - } - return path -} - -func resolveLibraryAudioExt(filePath, displayNameHint string) string { - ext := strings.ToLower(filepath.Ext(filePath)) - if ext != "" { - return ext - } - return strings.ToLower(filepath.Ext(displayNameHint)) -} - -func libraryDisplayNameOrPath(filePath, displayNameHint string) string { - if displayNameHint != "" { - return displayNameHint - } - return filePath -} - -func applyDefaultLibraryMetadata(filePath, displayNameHint string, result *LibraryScanResult) { - nameSource := libraryDisplayNameOrPath(filePath, displayNameHint) - if result.TrackName == "" { - result.TrackName = strings.TrimSuffix(filepath.Base(nameSource), filepath.Ext(nameSource)) - } - if result.ArtistName == "" { - result.ArtistName = "Unknown Artist" - } - if result.AlbumName == "" { - result.AlbumName = "Unknown Album" - } -} - -func scanFLACFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - return scanFLACFileWithCoverCache(filePath, result, displayNameHint, "", "") -} - -func scanFLACFileWithCoverCache(filePath string, result *LibraryScanResult, displayNameHint, coverCacheDir, coverCacheKey string) (*LibraryScanResult, error) { - f, err := parseFlacFile(filePath) - if err != nil { - return scanFromFilename(filePath, displayNameHint, result) - } - defer f.Close() - metadata := metadataFromParsedFlac(f) - - result.TrackName = metadata.Title - result.ArtistName = metadata.Artist - result.AlbumName = metadata.Album - result.AlbumArtist = metadata.AlbumArtist - result.ISRC = metadata.ISRC - result.TrackNumber = metadata.TrackNumber - result.TotalTracks = metadata.TotalTracks - result.DiscNumber = metadata.DiscNumber - result.TotalDiscs = metadata.TotalDiscs - result.ReleaseDate = metadata.Date - result.Genre = metadata.Genre - result.Composer = metadata.Composer - result.Label = metadata.Label - result.Copyright = metadata.Copyright - - quality, err := audioQualityFromParsedFlac(f) - if err == nil { - result.BitDepth = quality.BitDepth - result.SampleRate = quality.SampleRate - if quality.SampleRate > 0 && quality.TotalSamples > 0 { - result.Duration = int(quality.TotalSamples / int64(quality.SampleRate)) - } - } - if coverCacheDir != "" { - cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) - if existing := existingLibraryCoverCachePath(coverCacheDir, cacheKey); existing != "" { - result.CoverPath = existing - } else if coverData, coverErr := coverArtFromParsedFlac(f); coverErr == nil { - result.CoverPath = cacheScannedCover(filePath, coverCacheDir, cacheKey, coverData) - } - } - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - - return result, nil -} - -func scanM4AFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - return scanM4AFileWithCoverCache(filePath, result, displayNameHint, "", "") -} - -func scanM4AFileWithCoverCache(filePath string, result *LibraryScanResult, displayNameHint, coverCacheDir, coverCacheKey string) (*LibraryScanResult, error) { - f, err := os.Open(filePath) - if err != nil { - return scanFromFilename(filePath, displayNameHint, result) - } - defer f.Close() - info, err := f.Stat() - if err != nil { - return scanFromFilename(filePath, displayNameHint, result) - } - fileSize := info.Size() - - var metadata *AudioMetadata - ilst, ilstErr := findM4AIlstAtom(f, fileSize) - if ilstErr == nil { - metadata, err = readM4ATagsFromIlst(f, fileSize, ilst) - } else { - err = ilstErr - } - if err != nil { - GoLog("[LibraryScan] M4A read error for %s: %v\n", filePath, err) - } - - if metadata != nil { - applyAudioMetadataToScan(metadata, result) - } - - quality, err := m4aQualityFromFile(f, fileSize) - if err == nil { - result.BitDepth = quality.BitDepth - result.SampleRate = quality.SampleRate - result.Duration = quality.Duration - if quality.Bitrate > 0 { - result.Bitrate = quality.Bitrate - } - if format := libraryFormatForM4ACodec(quality.Codec); format != "" { - result.Format = format - if isLosslessLibraryFormat(format) { - result.Bitrate = 0 - } - } - } - if coverCacheDir != "" { - cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) - if existing := existingLibraryCoverCachePath(coverCacheDir, cacheKey); existing != "" { - result.CoverPath = existing - } else if ilstErr == nil { - if coverData, coverErr := extractCoverFromM4AIlst(f, fileSize, ilst); coverErr == nil { - result.CoverPath = cacheScannedCover(filePath, coverCacheDir, cacheKey, coverData) - } - } - } - - if metadata == nil { - return scanFromFilename(filePath, displayNameHint, result) - } - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - return result, nil -} - -func libraryFormatForM4ACodec(codec string) string { - switch strings.ToLower(strings.TrimSpace(codec)) { - case "flac": - return "flac" - case "alac": - return "alac" - case "eac3", "ec-3": - return "eac3" - case "ac3", "ac-3": - return "ac3" - case "ac4", "ac-4": - return "ac4" - case "aac", "mp4a": - return "m4a" - default: - return "" - } -} - -func isLosslessLibraryFormat(format string) bool { - switch strings.ToLower(strings.TrimSpace(format)) { - case "flac", "alac", "wav", "aiff", "aif", "aifc": - return true - default: - return false - } -} - -func scanMP3File(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - metadata, err := ReadID3Tags(filePath) - if err != nil { - GoLog("[LibraryScan] ID3 read error for %s: %v\n", filePath, err) - return scanFromFilename(filePath, displayNameHint, result) - } - - applyAudioMetadataToScan(metadata, result) - - quality, err := GetMP3Quality(filePath) - if err == nil { - result.SampleRate = quality.SampleRate - result.BitDepth = quality.BitDepth // 0 for lossy - result.Duration = quality.Duration - if quality.Bitrate > 0 { - result.Bitrate = quality.Bitrate / 1000 // convert bps to kbps - } - } - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - - return result, nil -} - -func scanOggFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - metadata, err := ReadOggVorbisComments(filePath) - if err != nil { - GoLog("[LibraryScan] Ogg/Opus read error for %s: %v\n", filePath, err) - return scanFromFilename(filePath, displayNameHint, result) - } - - result.TrackName = metadata.Title - result.ArtistName = metadata.Artist - result.AlbumName = metadata.Album - result.AlbumArtist = metadata.AlbumArtist - result.ISRC = metadata.ISRC - result.TrackNumber = metadata.TrackNumber - result.TotalTracks = metadata.TotalTracks - result.DiscNumber = metadata.DiscNumber - result.TotalDiscs = metadata.TotalDiscs - result.Genre = metadata.Genre - result.ReleaseDate = metadata.Date - result.Composer = metadata.Composer - result.Label = metadata.Label - result.Copyright = metadata.Copyright - - quality, err := GetOggQuality(filePath) - if err == nil { - result.SampleRate = quality.SampleRate - result.BitDepth = quality.BitDepth // 0 for lossy - result.Duration = quality.Duration - if quality.Bitrate > 0 { - result.Bitrate = quality.Bitrate / 1000 // convert bps to kbps - } - } - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - - return result, nil -} - -func scanAPEFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - tag, err := ReadAPETags(filePath) - if err != nil { - GoLog("[LibraryScan] APE tag read error for %s: %v\n", filePath, err) - return scanFromFilename(filePath, displayNameHint, result) - } - - metadata := APETagToAudioMetadata(tag) - if metadata == nil { - return scanFromFilename(filePath, displayNameHint, result) - } - - applyAudioMetadataToScan(metadata, result) - - applyDefaultLibraryMetadata(filePath, displayNameHint, result) - - return result, nil -} - -func scanFromFilename(filePath, displayNameHint string, result *LibraryScanResult) (*LibraryScanResult, error) { - result.MetadataFromFilename = true - nameSource := libraryDisplayNameOrPath(filePath, displayNameHint) - filename := strings.TrimSuffix(filepath.Base(nameSource), filepath.Ext(nameSource)) - - parts := strings.SplitN(filename, " - ", 2) - if len(parts) == 2 { - if len(parts[0]) <= 3 && isNumeric(parts[0]) { - result.TrackName = parts[1] - result.ArtistName = "Unknown Artist" - } else { - result.ArtistName = parts[0] - result.TrackName = parts[1] - } - } else { - if len(filename) > 3 && isNumeric(filename[:2]) { - title := strings.TrimLeft(filename[2:], " .-") - result.TrackName = title - } else { - result.TrackName = filename - } - result.ArtistName = "Unknown Artist" - } - - dir := filepath.Dir(filePath) - result.AlbumName = filepath.Base(dir) - if result.AlbumName == "." || result.AlbumName == "" || result.AlbumName == "fd" || result.AlbumName == "self" { - result.AlbumName = "Unknown Album" - } - - return result, nil -} - -func isNumeric(s string) bool { - for _, c := range s { - if c < '0' || c > '9' { - return false - } - } - return len(s) > 0 -} - -func generateLibraryID(filePath string) string { - return fmt.Sprintf("lib_%x", hashString(filePath)) -} - -func hashString(s string) uint32 { - var hash uint32 = 5381 - for _, c := range s { - hash = ((hash << 5) + hash) + uint32(c) - } - return hash -} - func GetLibraryScanProgress() string { libraryScanProgressMu.RLock() defer libraryScanProgressMu.RUnlock() @@ -897,282 +511,3 @@ func ReadAudioMetadataWithDisplayNameAndCoverCacheKey(filePath, displayNameHint, return string(jsonBytes), nil } - -func loadExistingFilesSnapshot(snapshotPath string) (map[string]int64, error) { - existingFiles := make(map[string]int64) - if snapshotPath == "" { - return existingFiles, nil - } - - file, err := os.Open(snapshotPath) - if err != nil { - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if line == "" { - continue - } - parts := strings.SplitN(line, "\t", 2) - if len(parts) != 2 { - continue - } - modTime, err := strconv.ParseInt(parts[0], 10, 64) - if err != nil { - continue - } - existingFiles[parts[1]] = modTime - } - - if err := scanner.Err(); err != nil { - return nil, err - } - - return existingFiles, nil -} - -func scanLibraryFolderIncrementalWithExistingFiles(folderPath string, existingFiles map[string]int64) (string, error) { - if folderPath == "" { - return "{}", fmt.Errorf("folder path is empty") - } - - info, err := os.Stat(folderPath) - if err != nil { - return "{}", fmt.Errorf("folder not found: %w", err) - } - if !info.IsDir() { - return "{}", fmt.Errorf("path is not a folder: %s", folderPath) - } - - GoLog("[LibraryScan] Incremental scan starting, %d existing files in database\n", len(existingFiles)) - - libraryScanProgressMu.Lock() - libraryScanProgress = LibraryScanProgress{} - libraryScanProgressMu.Unlock() - - libraryScanCancelMu.Lock() - if libraryScanCancel != nil { - close(libraryScanCancel) - } - libraryScanCancel = make(chan struct{}) - cancelCh := libraryScanCancel - libraryScanCancelMu.Unlock() - - currentFiles, err := collectLibraryAudioFiles(folderPath, cancelCh) - if err != nil { - return "{}", err - } - currentPathSet := make(map[string]bool, len(currentFiles)) - for _, fileInfo := range currentFiles { - currentPathSet[fileInfo.path] = true - } - - totalFiles := len(currentFiles) - libraryScanProgressMu.Lock() - libraryScanProgress.TotalFiles = totalFiles - libraryScanProgressMu.Unlock() - - var filesToScan []libraryAudioFileInfo - skippedCount := 0 - existingCueTrackModTimes := make(map[string]int64) - for existingPath, modTime := range existingFiles { - if idx := strings.LastIndex(existingPath, "#track"); idx > 0 { - baseCuePath := existingPath[:idx] - if _, exists := existingCueTrackModTimes[baseCuePath]; !exists { - existingCueTrackModTimes[baseCuePath] = modTime - } - } - } - - for _, f := range currentFiles { - existingModTime, exists := existingFiles[f.path] - if !exists { - if strings.ToLower(filepath.Ext(f.path)) == ".cue" { - if cueTrackModTime, hasCueTracks := existingCueTrackModTimes[f.path]; hasCueTracks { - if f.modTime == cueTrackModTime { - skippedCount++ - } else { - filesToScan = append(filesToScan, f) - } - continue - } - } - filesToScan = append(filesToScan, f) - } else if f.modTime != existingModTime { - filesToScan = append(filesToScan, f) - } else { - skippedCount++ - } - } - - var deletedPaths []string - for existingPath := range existingFiles { - if idx := strings.LastIndex(existingPath, "#track"); idx > 0 { - baseCuePath := existingPath[:idx] - if currentPathSet[baseCuePath] { - continue - } - deletedPaths = append(deletedPaths, existingPath) - } else if !currentPathSet[existingPath] { - deletedPaths = append(deletedPaths, existingPath) - } - } - - GoLog("[LibraryScan] Incremental: %d to scan, %d skipped, %d deleted\n", - len(filesToScan), skippedCount, len(deletedPaths)) - - if len(filesToScan) == 0 { - libraryScanProgressMu.Lock() - libraryScanProgress.ScannedFiles = totalFiles - libraryScanProgress.IsComplete = true - libraryScanProgress.ProgressPct = 100 - libraryScanProgressMu.Unlock() - - result := IncrementalScanResult{ - Scanned: []LibraryScanResult{}, - DeletedPaths: deletedPaths, - SkippedCount: skippedCount, - TotalFiles: totalFiles, - } - jsonBytes, _ := json.Marshal(result) - return string(jsonBytes), nil - } - - results := make([]LibraryScanResult, 0, len(filesToScan)) - scanTime := time.Now().UTC().Format(time.RFC3339) - errorCount := 0 - - cueReferencedAudioFilesInc := make(map[string]bool) - parsedCueFiles := make(map[string]scannedCueFileInfo) - for _, f := range filesToScan { - ext := strings.ToLower(filepath.Ext(f.path)) - if ext == ".cue" { - sheet, err := ParseCueFile(f.path) - if err == nil && sheet.FileName != "" { - audioPath := ResolveCueAudioPath(f.path, sheet.FileName) - if audioPath != "" { - parsedCueFiles[f.path] = scannedCueFileInfo{ - sheet: sheet, - audioPath: audioPath, - } - cueReferencedAudioFilesInc[audioPath] = true - } - } - } - } - - resultsByIndex := make(map[int][]LibraryScanResult, len(filesToScan)) - audioTasks := make([]libraryScanTask, 0, len(filesToScan)) - completedFiles := skippedCount - - for i, f := range filesToScan { - select { - case <-cancelCh: - return "{}", fmt.Errorf("scan cancelled") - default: - } - - ext := strings.ToLower(filepath.Ext(f.path)) - - if ext == ".cue" { - var cueResults []LibraryScanResult - cueInfo, ok := parsedCueFiles[f.path] - if ok { - cueResults, err = scanCueSheetForLibrary( - f.path, - cueInfo.sheet, - cueInfo.audioPath, - "", - f.modTime, - "", - scanTime, - ) - } else { - cueResults, err = ScanCueFileForLibrary(f.path, scanTime) - } - if err != nil { - errorCount++ - GoLog("[LibraryScan] Error scanning cue %s: %v\n", f.path, err) - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, f.path) - continue - } - resultsByIndex[i] = cueResults - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, f.path) - continue - } - - if cueReferencedAudioFilesInc[f.path] { - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, f.path) - continue - } - - audioTasks = append(audioTasks, libraryScanTask{index: i, info: f}) - } - - audioResults, audioErrors, err := scanLibraryAudioTasksParallel( - audioTasks, - scanTime, - cancelCh, - totalFiles, - &completedFiles, - ) - if err != nil { - return "{}", err - } - errorCount += audioErrors - for index, scanResults := range audioResults { - resultsByIndex[index] = scanResults - } - - for i := range filesToScan { - results = append(results, resultsByIndex[i]...) - } - - libraryScanProgressMu.Lock() - libraryScanProgress.ErrorCount = errorCount - libraryScanProgress.IsComplete = true - libraryScanProgress.ScannedFiles = totalFiles - libraryScanProgress.ProgressPct = 100 - libraryScanProgressMu.Unlock() - - GoLog("[LibraryScan] Incremental scan complete: %d scanned, %d skipped, %d deleted, %d errors\n", - len(results), skippedCount, len(deletedPaths), errorCount) - - scanResult := IncrementalScanResult{ - Scanned: results, - DeletedPaths: deletedPaths, - SkippedCount: skippedCount, - TotalFiles: totalFiles, - } - - jsonBytes, err := json.Marshal(scanResult) - if err != nil { - return "{}", fmt.Errorf("failed to marshal results: %w", err) - } - - return string(jsonBytes), nil -} - -func ScanLibraryFolderIncremental(folderPath, existingFilesJSON string) (string, error) { - existingFiles := make(map[string]int64) - if existingFilesJSON != "" && existingFilesJSON != "{}" { - if err := json.Unmarshal([]byte(existingFilesJSON), &existingFiles); err != nil { - GoLog("[LibraryScan] Warning: failed to parse existing files JSON: %v\n", err) - } - } - return scanLibraryFolderIncrementalWithExistingFiles(folderPath, existingFiles) -} - -func ScanLibraryFolderIncrementalFromSnapshot(folderPath, snapshotPath string) (string, error) { - existingFiles, err := loadExistingFilesSnapshot(snapshotPath) - if err != nil { - return "{}", fmt.Errorf("failed to load incremental snapshot: %w", err) - } - return scanLibraryFolderIncrementalWithExistingFiles(folderPath, existingFiles) -} diff --git a/go_backend/library_scan_formats.go b/go_backend/library_scan_formats.go new file mode 100644 index 00000000..f50d3c06 --- /dev/null +++ b/go_backend/library_scan_formats.go @@ -0,0 +1,392 @@ +package gobackend + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func scanAudioFileWithKnownModTime(filePath, scanTime string, knownModTime int64) (*LibraryScanResult, error) { + return scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(filePath, "", "", scanTime, knownModTime) +} + +func scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(filePath, displayNameHint, coverCacheKey, scanTime string, knownModTime int64) (*LibraryScanResult, error) { + ext := resolveLibraryAudioExt(filePath, displayNameHint) + + result := &LibraryScanResult{ + ID: generateLibraryID(filePath), + FilePath: filePath, + ScannedAt: scanTime, + Format: strings.TrimPrefix(ext, "."), + } + + if knownModTime > 0 { + result.FileModTime = knownModTime + } else if info, err := os.Stat(filePath); err == nil { + result.FileModTime = info.ModTime().UnixMilli() + } + + libraryCoverCacheMu.RLock() + coverCacheDir := libraryCoverCacheDir + libraryCoverCacheMu.RUnlock() + if ext == ".flac" { + return scanFLACFileWithCoverCache(filePath, result, displayNameHint, coverCacheDir, coverCacheKey) + } + if ext == ".m4a" || ext == ".mp4" || ext == ".aac" { + return scanM4AFileWithCoverCache(filePath, result, displayNameHint, coverCacheDir, coverCacheKey) + } + if coverCacheDir != "" { + coverPath, err := SaveCoverToCacheWithHintAndKey( + filePath, + displayNameHint, + coverCacheDir, + coverCacheKey, + ) + if err == nil && coverPath != "" { + result.CoverPath = coverPath + } + } + + switch ext { + case ".mp3": + return scanMP3File(filePath, result, displayNameHint) + case ".opus", ".ogg": + return scanOggFile(filePath, result, displayNameHint) + case ".ape", ".wv", ".mpc": + return scanAPEFile(filePath, result, displayNameHint) + case ".wav": + return scanWAVFile(filePath, result, displayNameHint) + case ".aiff", ".aif", ".aifc": + return scanAIFFFile(filePath, result, displayNameHint) + default: + return scanFromFilename(filePath, displayNameHint, result) + } +} + +func embeddedCoverMIME(data []byte) string { + if len(data) >= 8 && + data[0] == 0x89 && + data[1] == 0x50 && + data[2] == 0x4e && + data[3] == 0x47 { + return "image/png" + } + return "image/jpeg" +} + +func cacheScannedCover(filePath, cacheDir, coverCacheKey string, coverData []byte) string { + if cacheDir == "" || len(coverData) == 0 { + return "" + } + cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) + path, err := saveLibraryCoverDataToCache(cacheDir, cacheKey, coverData, embeddedCoverMIME(coverData)) + if err != nil { + return "" + } + return path +} + +func resolveLibraryAudioExt(filePath, displayNameHint string) string { + ext := strings.ToLower(filepath.Ext(filePath)) + if ext != "" { + return ext + } + return strings.ToLower(filepath.Ext(displayNameHint)) +} + +func libraryDisplayNameOrPath(filePath, displayNameHint string) string { + if displayNameHint != "" { + return displayNameHint + } + return filePath +} + +func applyDefaultLibraryMetadata(filePath, displayNameHint string, result *LibraryScanResult) { + nameSource := libraryDisplayNameOrPath(filePath, displayNameHint) + if result.TrackName == "" { + result.TrackName = strings.TrimSuffix(filepath.Base(nameSource), filepath.Ext(nameSource)) + } + if result.ArtistName == "" { + result.ArtistName = "Unknown Artist" + } + if result.AlbumName == "" { + result.AlbumName = "Unknown Album" + } +} + +func scanFLACFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { + return scanFLACFileWithCoverCache(filePath, result, displayNameHint, "", "") +} + +func scanFLACFileWithCoverCache(filePath string, result *LibraryScanResult, displayNameHint, coverCacheDir, coverCacheKey string) (*LibraryScanResult, error) { + f, err := parseFlacFile(filePath) + if err != nil { + return scanFromFilename(filePath, displayNameHint, result) + } + defer f.Close() + metadata := metadataFromParsedFlac(f) + + result.TrackName = metadata.Title + result.ArtistName = metadata.Artist + result.AlbumName = metadata.Album + result.AlbumArtist = metadata.AlbumArtist + result.ISRC = metadata.ISRC + result.TrackNumber = metadata.TrackNumber + result.TotalTracks = metadata.TotalTracks + result.DiscNumber = metadata.DiscNumber + result.TotalDiscs = metadata.TotalDiscs + result.ReleaseDate = metadata.Date + result.Genre = metadata.Genre + result.Composer = metadata.Composer + result.Label = metadata.Label + result.Copyright = metadata.Copyright + + quality, err := audioQualityFromParsedFlac(f) + if err == nil { + result.BitDepth = quality.BitDepth + result.SampleRate = quality.SampleRate + if quality.SampleRate > 0 && quality.TotalSamples > 0 { + result.Duration = int(quality.TotalSamples / int64(quality.SampleRate)) + } + } + if coverCacheDir != "" { + cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) + if existing := existingLibraryCoverCachePath(coverCacheDir, cacheKey); existing != "" { + result.CoverPath = existing + } else if coverData, coverErr := coverArtFromParsedFlac(f); coverErr == nil { + result.CoverPath = cacheScannedCover(filePath, coverCacheDir, cacheKey, coverData) + } + } + + applyDefaultLibraryMetadata(filePath, displayNameHint, result) + + return result, nil +} + +func scanM4AFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { + return scanM4AFileWithCoverCache(filePath, result, displayNameHint, "", "") +} + +func scanM4AFileWithCoverCache(filePath string, result *LibraryScanResult, displayNameHint, coverCacheDir, coverCacheKey string) (*LibraryScanResult, error) { + f, err := os.Open(filePath) + if err != nil { + return scanFromFilename(filePath, displayNameHint, result) + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return scanFromFilename(filePath, displayNameHint, result) + } + fileSize := info.Size() + + var metadata *AudioMetadata + ilst, ilstErr := findM4AIlstAtom(f, fileSize) + if ilstErr == nil { + metadata, err = readM4ATagsFromIlst(f, fileSize, ilst) + } else { + err = ilstErr + } + if err != nil { + GoLog("[LibraryScan] M4A read error for %s: %v\n", filePath, err) + } + + if metadata != nil { + applyAudioMetadataToScan(metadata, result) + } + + quality, err := m4aQualityFromFile(f, fileSize) + if err == nil { + result.BitDepth = quality.BitDepth + result.SampleRate = quality.SampleRate + result.Duration = quality.Duration + if quality.Bitrate > 0 { + result.Bitrate = quality.Bitrate + } + if format := libraryFormatForM4ACodec(quality.Codec); format != "" { + result.Format = format + if isLosslessLibraryFormat(format) { + result.Bitrate = 0 + } + } + } + if coverCacheDir != "" { + cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) + if existing := existingLibraryCoverCachePath(coverCacheDir, cacheKey); existing != "" { + result.CoverPath = existing + } else if ilstErr == nil { + if coverData, coverErr := extractCoverFromM4AIlst(f, fileSize, ilst); coverErr == nil { + result.CoverPath = cacheScannedCover(filePath, coverCacheDir, cacheKey, coverData) + } + } + } + + if metadata == nil { + return scanFromFilename(filePath, displayNameHint, result) + } + + applyDefaultLibraryMetadata(filePath, displayNameHint, result) + return result, nil +} + +func libraryFormatForM4ACodec(codec string) string { + switch strings.ToLower(strings.TrimSpace(codec)) { + case "flac": + return "flac" + case "alac": + return "alac" + case "eac3", "ec-3": + return "eac3" + case "ac3", "ac-3": + return "ac3" + case "ac4", "ac-4": + return "ac4" + case "aac", "mp4a": + return "m4a" + default: + return "" + } +} + +func isLosslessLibraryFormat(format string) bool { + switch strings.ToLower(strings.TrimSpace(format)) { + case "flac", "alac", "wav", "aiff", "aif", "aifc": + return true + default: + return false + } +} + +func scanMP3File(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { + metadata, err := ReadID3Tags(filePath) + if err != nil { + GoLog("[LibraryScan] ID3 read error for %s: %v\n", filePath, err) + return scanFromFilename(filePath, displayNameHint, result) + } + + applyAudioMetadataToScan(metadata, result) + + quality, err := GetMP3Quality(filePath) + if err == nil { + result.SampleRate = quality.SampleRate + result.BitDepth = quality.BitDepth // 0 for lossy + result.Duration = quality.Duration + if quality.Bitrate > 0 { + result.Bitrate = quality.Bitrate / 1000 // convert bps to kbps + } + } + + applyDefaultLibraryMetadata(filePath, displayNameHint, result) + + return result, nil +} + +func scanOggFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { + metadata, err := ReadOggVorbisComments(filePath) + if err != nil { + GoLog("[LibraryScan] Ogg/Opus read error for %s: %v\n", filePath, err) + return scanFromFilename(filePath, displayNameHint, result) + } + + result.TrackName = metadata.Title + result.ArtistName = metadata.Artist + result.AlbumName = metadata.Album + result.AlbumArtist = metadata.AlbumArtist + result.ISRC = metadata.ISRC + result.TrackNumber = metadata.TrackNumber + result.TotalTracks = metadata.TotalTracks + result.DiscNumber = metadata.DiscNumber + result.TotalDiscs = metadata.TotalDiscs + result.Genre = metadata.Genre + result.ReleaseDate = metadata.Date + result.Composer = metadata.Composer + result.Label = metadata.Label + result.Copyright = metadata.Copyright + + quality, err := GetOggQuality(filePath) + if err == nil { + result.SampleRate = quality.SampleRate + result.BitDepth = quality.BitDepth // 0 for lossy + result.Duration = quality.Duration + if quality.Bitrate > 0 { + result.Bitrate = quality.Bitrate / 1000 // convert bps to kbps + } + } + + applyDefaultLibraryMetadata(filePath, displayNameHint, result) + + return result, nil +} + +func scanAPEFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { + tag, err := ReadAPETags(filePath) + if err != nil { + GoLog("[LibraryScan] APE tag read error for %s: %v\n", filePath, err) + return scanFromFilename(filePath, displayNameHint, result) + } + + metadata := APETagToAudioMetadata(tag) + if metadata == nil { + return scanFromFilename(filePath, displayNameHint, result) + } + + applyAudioMetadataToScan(metadata, result) + + applyDefaultLibraryMetadata(filePath, displayNameHint, result) + + return result, nil +} + +func scanFromFilename(filePath, displayNameHint string, result *LibraryScanResult) (*LibraryScanResult, error) { + result.MetadataFromFilename = true + nameSource := libraryDisplayNameOrPath(filePath, displayNameHint) + filename := strings.TrimSuffix(filepath.Base(nameSource), filepath.Ext(nameSource)) + + parts := strings.SplitN(filename, " - ", 2) + if len(parts) == 2 { + if len(parts[0]) <= 3 && isNumeric(parts[0]) { + result.TrackName = parts[1] + result.ArtistName = "Unknown Artist" + } else { + result.ArtistName = parts[0] + result.TrackName = parts[1] + } + } else { + if len(filename) > 3 && isNumeric(filename[:2]) { + title := strings.TrimLeft(filename[2:], " .-") + result.TrackName = title + } else { + result.TrackName = filename + } + result.ArtistName = "Unknown Artist" + } + + dir := filepath.Dir(filePath) + result.AlbumName = filepath.Base(dir) + if result.AlbumName == "." || result.AlbumName == "" || result.AlbumName == "fd" || result.AlbumName == "self" { + result.AlbumName = "Unknown Album" + } + + return result, nil +} + +func isNumeric(s string) bool { + for _, c := range s { + if c < '0' || c > '9' { + return false + } + } + return len(s) > 0 +} + +func generateLibraryID(filePath string) string { + return fmt.Sprintf("lib_%x", hashString(filePath)) +} + +func hashString(s string) uint32 { + var hash uint32 = 5381 + for _, c := range s { + hash = ((hash << 5) + hash) + uint32(c) + } + return hash +} diff --git a/go_backend/library_scan_incremental.go b/go_backend/library_scan_incremental.go new file mode 100644 index 00000000..af4d4fb1 --- /dev/null +++ b/go_backend/library_scan_incremental.go @@ -0,0 +1,291 @@ +package gobackend + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +func loadExistingFilesSnapshot(snapshotPath string) (map[string]int64, error) { + existingFiles := make(map[string]int64) + if snapshotPath == "" { + return existingFiles, nil + } + + file, err := os.Open(snapshotPath) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + parts := strings.SplitN(line, "\t", 2) + if len(parts) != 2 { + continue + } + modTime, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + continue + } + existingFiles[parts[1]] = modTime + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return existingFiles, nil +} + +func scanLibraryFolderIncrementalWithExistingFiles(folderPath string, existingFiles map[string]int64) (string, error) { + if folderPath == "" { + return "{}", fmt.Errorf("folder path is empty") + } + + info, err := os.Stat(folderPath) + if err != nil { + return "{}", fmt.Errorf("folder not found: %w", err) + } + if !info.IsDir() { + return "{}", fmt.Errorf("path is not a folder: %s", folderPath) + } + + GoLog("[LibraryScan] Incremental scan starting, %d existing files in database\n", len(existingFiles)) + + libraryScanProgressMu.Lock() + libraryScanProgress = LibraryScanProgress{} + libraryScanProgressMu.Unlock() + + libraryScanCancelMu.Lock() + if libraryScanCancel != nil { + close(libraryScanCancel) + } + libraryScanCancel = make(chan struct{}) + cancelCh := libraryScanCancel + libraryScanCancelMu.Unlock() + + currentFiles, err := collectLibraryAudioFiles(folderPath, cancelCh) + if err != nil { + return "{}", err + } + currentPathSet := make(map[string]bool, len(currentFiles)) + for _, fileInfo := range currentFiles { + currentPathSet[fileInfo.path] = true + } + + totalFiles := len(currentFiles) + libraryScanProgressMu.Lock() + libraryScanProgress.TotalFiles = totalFiles + libraryScanProgressMu.Unlock() + + var filesToScan []libraryAudioFileInfo + skippedCount := 0 + existingCueTrackModTimes := make(map[string]int64) + for existingPath, modTime := range existingFiles { + if idx := strings.LastIndex(existingPath, "#track"); idx > 0 { + baseCuePath := existingPath[:idx] + if _, exists := existingCueTrackModTimes[baseCuePath]; !exists { + existingCueTrackModTimes[baseCuePath] = modTime + } + } + } + + for _, f := range currentFiles { + existingModTime, exists := existingFiles[f.path] + if !exists { + if strings.ToLower(filepath.Ext(f.path)) == ".cue" { + if cueTrackModTime, hasCueTracks := existingCueTrackModTimes[f.path]; hasCueTracks { + if f.modTime == cueTrackModTime { + skippedCount++ + } else { + filesToScan = append(filesToScan, f) + } + continue + } + } + filesToScan = append(filesToScan, f) + } else if f.modTime != existingModTime { + filesToScan = append(filesToScan, f) + } else { + skippedCount++ + } + } + + var deletedPaths []string + for existingPath := range existingFiles { + if idx := strings.LastIndex(existingPath, "#track"); idx > 0 { + baseCuePath := existingPath[:idx] + if currentPathSet[baseCuePath] { + continue + } + deletedPaths = append(deletedPaths, existingPath) + } else if !currentPathSet[existingPath] { + deletedPaths = append(deletedPaths, existingPath) + } + } + + GoLog("[LibraryScan] Incremental: %d to scan, %d skipped, %d deleted\n", + len(filesToScan), skippedCount, len(deletedPaths)) + + if len(filesToScan) == 0 { + libraryScanProgressMu.Lock() + libraryScanProgress.ScannedFiles = totalFiles + libraryScanProgress.IsComplete = true + libraryScanProgress.ProgressPct = 100 + libraryScanProgressMu.Unlock() + + result := IncrementalScanResult{ + Scanned: []LibraryScanResult{}, + DeletedPaths: deletedPaths, + SkippedCount: skippedCount, + TotalFiles: totalFiles, + } + jsonBytes, _ := json.Marshal(result) + return string(jsonBytes), nil + } + + results := make([]LibraryScanResult, 0, len(filesToScan)) + scanTime := time.Now().UTC().Format(time.RFC3339) + errorCount := 0 + + cueReferencedAudioFilesInc := make(map[string]bool) + parsedCueFiles := make(map[string]scannedCueFileInfo) + for _, f := range filesToScan { + ext := strings.ToLower(filepath.Ext(f.path)) + if ext == ".cue" { + sheet, err := ParseCueFile(f.path) + if err == nil && sheet.FileName != "" { + audioPath := ResolveCueAudioPath(f.path, sheet.FileName) + if audioPath != "" { + parsedCueFiles[f.path] = scannedCueFileInfo{ + sheet: sheet, + audioPath: audioPath, + } + cueReferencedAudioFilesInc[audioPath] = true + } + } + } + } + + resultsByIndex := make(map[int][]LibraryScanResult, len(filesToScan)) + audioTasks := make([]libraryScanTask, 0, len(filesToScan)) + completedFiles := skippedCount + + for i, f := range filesToScan { + select { + case <-cancelCh: + return "{}", fmt.Errorf("scan cancelled") + default: + } + + ext := strings.ToLower(filepath.Ext(f.path)) + + if ext == ".cue" { + var cueResults []LibraryScanResult + cueInfo, ok := parsedCueFiles[f.path] + if ok { + cueResults, err = scanCueSheetForLibrary( + f.path, + cueInfo.sheet, + cueInfo.audioPath, + "", + f.modTime, + "", + scanTime, + ) + } else { + cueResults, err = ScanCueFileForLibrary(f.path, scanTime) + } + if err != nil { + errorCount++ + GoLog("[LibraryScan] Error scanning cue %s: %v\n", f.path, err) + completedFiles++ + updateLibraryScanProgress(completedFiles, totalFiles, f.path) + continue + } + resultsByIndex[i] = cueResults + completedFiles++ + updateLibraryScanProgress(completedFiles, totalFiles, f.path) + continue + } + + if cueReferencedAudioFilesInc[f.path] { + completedFiles++ + updateLibraryScanProgress(completedFiles, totalFiles, f.path) + continue + } + + audioTasks = append(audioTasks, libraryScanTask{index: i, info: f}) + } + + audioResults, audioErrors, err := scanLibraryAudioTasksParallel( + audioTasks, + scanTime, + cancelCh, + totalFiles, + &completedFiles, + ) + if err != nil { + return "{}", err + } + errorCount += audioErrors + for index, scanResults := range audioResults { + resultsByIndex[index] = scanResults + } + + for i := range filesToScan { + results = append(results, resultsByIndex[i]...) + } + + libraryScanProgressMu.Lock() + libraryScanProgress.ErrorCount = errorCount + libraryScanProgress.IsComplete = true + libraryScanProgress.ScannedFiles = totalFiles + libraryScanProgress.ProgressPct = 100 + libraryScanProgressMu.Unlock() + + GoLog("[LibraryScan] Incremental scan complete: %d scanned, %d skipped, %d deleted, %d errors\n", + len(results), skippedCount, len(deletedPaths), errorCount) + + scanResult := IncrementalScanResult{ + Scanned: results, + DeletedPaths: deletedPaths, + SkippedCount: skippedCount, + TotalFiles: totalFiles, + } + + jsonBytes, err := json.Marshal(scanResult) + if err != nil { + return "{}", fmt.Errorf("failed to marshal results: %w", err) + } + + return string(jsonBytes), nil +} + +func ScanLibraryFolderIncremental(folderPath, existingFilesJSON string) (string, error) { + existingFiles := make(map[string]int64) + if existingFilesJSON != "" && existingFilesJSON != "{}" { + if err := json.Unmarshal([]byte(existingFilesJSON), &existingFiles); err != nil { + GoLog("[LibraryScan] Warning: failed to parse existing files JSON: %v\n", err) + } + } + return scanLibraryFolderIncrementalWithExistingFiles(folderPath, existingFiles) +} + +func ScanLibraryFolderIncrementalFromSnapshot(folderPath, snapshotPath string) (string, error) { + existingFiles, err := loadExistingFilesSnapshot(snapshotPath) + if err != nil { + return "{}", fmt.Errorf("failed to load incremental snapshot: %w", err) + } + return scanLibraryFolderIncrementalWithExistingFiles(folderPath, existingFiles) +}