From c13fdcc6e1106ddba111b26351d4727275401162 Mon Sep 17 00:00:00 2001 From: zarzet Date: Thu, 16 Jul 2026 10:47:29 +0700 Subject: [PATCH] perf(library): scan flac and m4a in one pass --- go_backend/audio_metadata.go | 61 +++++++---- go_backend/library_scan.go | 108 +++++++++++++++++-- go_backend/library_scan_single_pass_test.go | 110 ++++++++++++++++++++ go_backend/metadata.go | 73 ++++++++++--- 4 files changed, 305 insertions(+), 47 deletions(-) create mode 100644 go_backend/library_scan_single_pass_test.go diff --git a/go_backend/audio_metadata.go b/go_backend/audio_metadata.go index 0c8aba26..6e80411f 100644 --- a/go_backend/audio_metadata.go +++ b/go_backend/audio_metadata.go @@ -1575,39 +1575,56 @@ func resolveLibraryCoverCacheKey(filePath, explicitKey string) string { return cacheKey } -func SaveCoverToCacheWithHintAndKey(filePath, displayNameHint, cacheDir, coverCacheKey string) (string, error) { - cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) +func libraryCoverCachePaths(cacheDir, cacheKey string) (string, string) { hash := hashString(cacheKey) - jpgPath := filepath.Join(cacheDir, fmt.Sprintf("cover_%x.jpg", hash)) pngPath := filepath.Join(cacheDir, fmt.Sprintf("cover_%x.png", hash)) + return jpgPath, pngPath +} + +func existingLibraryCoverCachePath(cacheDir, cacheKey string) string { + jpgPath, pngPath := libraryCoverCachePaths(cacheDir, cacheKey) if _, err := os.Stat(jpgPath); err == nil { - return jpgPath, nil + return jpgPath } if _, err := os.Stat(pngPath); err == nil { - return pngPath, nil + return pngPath + } + return "" +} + +func saveLibraryCoverDataToCache(cacheDir, cacheKey string, imageData []byte, mimeType string) (string, error) { + if existing := existingLibraryCoverCachePath(cacheDir, cacheKey); existing != "" { + return existing, nil + } + if len(imageData) == 0 { + return "", fmt.Errorf("cover data is empty") + } + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return "", fmt.Errorf("failed to create cache dir: %w", err) + } + + jpgPath, pngPath := libraryCoverCachePaths(cacheDir, cacheKey) + cachePath := jpgPath + if strings.Contains(mimeType, "png") { + cachePath = pngPath + } + if err := os.WriteFile(cachePath, imageData, 0644); err != nil { + return "", fmt.Errorf("failed to write cover: %w", err) + } + return cachePath, nil +} + +func SaveCoverToCacheWithHintAndKey(filePath, displayNameHint, cacheDir, coverCacheKey string) (string, error) { + cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) + if existing := existingLibraryCoverCachePath(cacheDir, cacheKey); existing != "" { + return existing, nil } imageData, mimeType, err := extractAnyCoverArtWithHint(filePath, displayNameHint) if err != nil { return "", err } - - if err := os.MkdirAll(cacheDir, 0755); err != nil { - return "", fmt.Errorf("failed to create cache dir: %w", err) - } - - var cachePath string - if strings.Contains(mimeType, "png") { - cachePath = pngPath - } else { - cachePath = jpgPath - } - - if err := os.WriteFile(cachePath, imageData, 0644); err != nil { - return "", fmt.Errorf("failed to write cover: %w", err) - } - - return cachePath, nil + return saveLibraryCoverDataToCache(cacheDir, cacheKey, imageData, mimeType) } diff --git a/go_backend/library_scan.go b/go_backend/library_scan.go index eca7e3d3..757b13ae 100644 --- a/go_backend/library_scan.go +++ b/go_backend/library_scan.go @@ -86,6 +86,7 @@ var supportedAudioFormats = map[string]bool{ type libraryAudioFileInfo struct { path string modTime int64 + size int64 } type scannedCueFileInfo struct { @@ -152,6 +153,7 @@ func collectLibraryAudioFiles(folderPath string, cancelCh <-chan struct{}) ([]li files = append(files, libraryAudioFileInfo{ path: path, modTime: info.ModTime().UnixMilli(), + size: info.Size(), }) return nil }) @@ -163,6 +165,10 @@ func collectLibraryAudioFiles(folderPath string, cancelCh <-chan struct{}) ([]li return files, nil } +func libraryAudioCoverCacheKey(info libraryAudioFileInfo) string { + return fmt.Sprintf("%s|%d|%d", info.path, info.size, info.modTime) +} + func libraryScanWorkerCount(taskCount int) int { if taskCount < 16 { return 1 @@ -205,7 +211,13 @@ func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, can return resultsByIndex, errorCount, fmt.Errorf("scan cancelled") default: } - result, err := scanAudioFileWithKnownModTime(task.info.path, scanTime, task.info.modTime) + result, err := scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey( + task.info.path, + "", + libraryAudioCoverCacheKey(task.info), + scanTime, + task.info.modTime, + ) *completed++ updateLibraryScanProgress(*completed, totalFiles, task.info.path) if err != nil { @@ -232,7 +244,13 @@ func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, can return default: } - result, err := scanAudioFileWithKnownModTime(task.info.path, scanTime, task.info.modTime) + result, err := scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey( + task.info.path, + "", + libraryAudioCoverCacheKey(task.info), + scanTime, + task.info.modTime, + ) taskResult := libraryScanTaskResult{ index: task.index, path: task.info.path, @@ -472,6 +490,12 @@ func scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(filePath, displ libraryCoverCacheMu.RLock() coverCacheDir := libraryCoverCacheDir libraryCoverCacheMu.RUnlock() + if ext == ".flac" { + return scanFLACFileWithCoverCache(filePath, result, displayNameHint, coverCacheDir, coverCacheKey) + } + if ext == ".m4a" || ext == ".mp4" || ext == ".aac" { + return scanM4AFileWithCoverCache(filePath, result, displayNameHint, coverCacheDir, coverCacheKey) + } if coverCacheDir != "" { coverPath, err := SaveCoverToCacheWithHintAndKey( filePath, @@ -485,10 +509,6 @@ func scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(filePath, displ } switch ext { - case ".flac": - return scanFLACFile(filePath, result, displayNameHint) - case ".m4a", ".mp4", ".aac": - return scanM4AFile(filePath, result, displayNameHint) case ".mp3": return scanMP3File(filePath, result, displayNameHint) case ".opus", ".ogg": @@ -504,6 +524,29 @@ func scanAudioFileWithKnownModTimeAndDisplayNameAndCoverCacheKey(filePath, displ } } +func embeddedCoverMIME(data []byte) string { + if len(data) >= 8 && + data[0] == 0x89 && + data[1] == 0x50 && + data[2] == 0x4e && + data[3] == 0x47 { + return "image/png" + } + return "image/jpeg" +} + +func cacheScannedCover(filePath, cacheDir, coverCacheKey string, coverData []byte) string { + if cacheDir == "" || len(coverData) == 0 { + return "" + } + cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) + path, err := saveLibraryCoverDataToCache(cacheDir, cacheKey, coverData, embeddedCoverMIME(coverData)) + if err != nil { + return "" + } + return path +} + func resolveLibraryAudioExt(filePath, displayNameHint string) string { ext := strings.ToLower(filepath.Ext(filePath)) if ext != "" { @@ -533,10 +576,16 @@ func applyDefaultLibraryMetadata(filePath, displayNameHint string, result *Libra } func scanFLACFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - metadata, err := ReadMetadata(filePath) + return scanFLACFileWithCoverCache(filePath, result, displayNameHint, "", "") +} + +func scanFLACFileWithCoverCache(filePath string, result *LibraryScanResult, displayNameHint, coverCacheDir, coverCacheKey string) (*LibraryScanResult, error) { + f, err := parseFlacFile(filePath) if err != nil { return scanFromFilename(filePath, displayNameHint, result) } + defer f.Close() + metadata := metadataFromParsedFlac(f) result.TrackName = metadata.Title result.ArtistName = metadata.Artist @@ -553,7 +602,7 @@ func scanFLACFile(filePath string, result *LibraryScanResult, displayNameHint st result.Label = metadata.Label result.Copyright = metadata.Copyright - quality, err := GetAudioQuality(filePath) + quality, err := audioQualityFromParsedFlac(f) if err == nil { result.BitDepth = quality.BitDepth result.SampleRate = quality.SampleRate @@ -561,6 +610,14 @@ func scanFLACFile(filePath string, result *LibraryScanResult, displayNameHint st result.Duration = int(quality.TotalSamples / int64(quality.SampleRate)) } } + if coverCacheDir != "" { + cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) + if existing := existingLibraryCoverCachePath(coverCacheDir, cacheKey); existing != "" { + result.CoverPath = existing + } else if coverData, coverErr := coverArtFromParsedFlac(f); coverErr == nil { + result.CoverPath = cacheScannedCover(filePath, coverCacheDir, cacheKey, coverData) + } + } applyDefaultLibraryMetadata(filePath, displayNameHint, result) @@ -568,7 +625,28 @@ func scanFLACFile(filePath string, result *LibraryScanResult, displayNameHint st } func scanM4AFile(filePath string, result *LibraryScanResult, displayNameHint string) (*LibraryScanResult, error) { - metadata, err := ReadM4ATags(filePath) + return scanM4AFileWithCoverCache(filePath, result, displayNameHint, "", "") +} + +func scanM4AFileWithCoverCache(filePath string, result *LibraryScanResult, displayNameHint, coverCacheDir, coverCacheKey string) (*LibraryScanResult, error) { + f, err := os.Open(filePath) + if err != nil { + return scanFromFilename(filePath, displayNameHint, result) + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return scanFromFilename(filePath, displayNameHint, result) + } + fileSize := info.Size() + + var metadata *AudioMetadata + ilst, ilstErr := findM4AIlstAtom(f, fileSize) + if ilstErr == nil { + metadata, err = readM4ATagsFromIlst(f, fileSize, ilst) + } else { + err = ilstErr + } if err != nil { GoLog("[LibraryScan] M4A read error for %s: %v\n", filePath, err) } @@ -577,7 +655,7 @@ func scanM4AFile(filePath string, result *LibraryScanResult, displayNameHint str applyAudioMetadataToScan(metadata, result) } - quality, err := GetM4AQuality(filePath) + quality, err := m4aQualityFromFile(f, fileSize) if err == nil { result.BitDepth = quality.BitDepth result.SampleRate = quality.SampleRate @@ -592,6 +670,16 @@ func scanM4AFile(filePath string, result *LibraryScanResult, displayNameHint str } } } + if coverCacheDir != "" { + cacheKey := resolveLibraryCoverCacheKey(filePath, coverCacheKey) + if existing := existingLibraryCoverCachePath(coverCacheDir, cacheKey); existing != "" { + result.CoverPath = existing + } else if ilstErr == nil { + if coverData, coverErr := extractCoverFromM4AIlst(f, fileSize, ilst); coverErr == nil { + result.CoverPath = cacheScannedCover(filePath, coverCacheDir, cacheKey, coverData) + } + } + } if metadata == nil { return scanFromFilename(filePath, displayNameHint, result) diff --git a/go_backend/library_scan_single_pass_test.go b/go_backend/library_scan_single_pass_test.go new file mode 100644 index 00000000..364e5856 --- /dev/null +++ b/go_backend/library_scan_single_pass_test.go @@ -0,0 +1,110 @@ +package gobackend + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/go-flac/flacpicture/v2" + "github.com/go-flac/flacvorbis/v2" + flac "github.com/go-flac/go-flac/v2" +) + +func writeSinglePassTestFlac(t *testing.T, path string, cover []byte) { + t.Helper() + streamInfo := make([]byte, 34) + const sampleRate = 44100 + const bitDepth = 16 + const totalSamples = int64(441000) + streamInfo[10] = byte((sampleRate >> 12) & 0xff) + streamInfo[11] = byte((sampleRate >> 4) & 0xff) + streamInfo[12] = byte((sampleRate&0x0f)<<4 | ((bitDepth - 1) >> 4)) + streamInfo[13] = byte(((bitDepth-1)&0x0f)<<4) | byte(totalSamples>>32) + streamInfo[14] = byte((totalSamples >> 24) & 0xff) + streamInfo[15] = byte((totalSamples >> 16) & 0xff) + streamInfo[16] = byte((totalSamples >> 8) & 0xff) + streamInfo[17] = byte(totalSamples & 0xff) + streamBlock := flac.MetaDataBlock{Type: flac.StreamInfo, Data: streamInfo} + + comments := flacvorbis.New() + setComment(comments, "TITLE", "Single Pass") + setComment(comments, "ARTIST", "Artist") + commentBlock := comments.Marshal() + pictureBlock := (&flacpicture.MetadataBlockPicture{ + PictureType: flacpicture.PictureTypeFrontCover, + MIME: "image/png", + Width: 1, + Height: 1, + ColorDepth: 24, + ImageData: cover, + }).Marshal() + + data := append([]byte("fLaC"), streamBlock.Marshal(false)...) + data = append(data, commentBlock.Marshal(false)...) + data = append(data, pictureBlock.Marshal(true)...) + data = append(data, 0xff, 0xf8) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} + +func TestScanFLACSinglePassReadsMetadataQualityAndCover(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "track.flac") + cover := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3} + writeSinglePassTestFlac(t, path, cover) + + result := &LibraryScanResult{FilePath: path, Format: "flac"} + result, err := scanFLACFileWithCoverCache(path, result, "", filepath.Join(dir, "covers"), "stable-key") + if err != nil { + t.Fatal(err) + } + if result.TrackName != "Single Pass" || result.ArtistName != "Artist" || result.SampleRate != 44100 || result.BitDepth != 16 || result.Duration != 10 { + t.Fatalf("scan result = %#v", result) + } + if result.CoverPath == "" { + t.Fatal("cover was not cached") + } + gotCover, err := os.ReadFile(result.CoverPath) + if err != nil || !bytes.Equal(gotCover, cover) { + t.Fatalf("cached cover = %x, %v", gotCover, err) + } +} + +func TestScanM4ASingleOpenReadsMetadataAndCover(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "track.m4a") + cover := []byte{0xff, 0xd8, 0xff, 1, 2, 3} + ilst := append(buildM4ATextAtom("\xa9nam", "Single Open"), buildM4ACoverAtom(cover)...) + data, _ := buildTestM4A(t, ilst, []byte("audio")) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + + result := &LibraryScanResult{FilePath: path, Format: "m4a"} + result, err := scanM4AFileWithCoverCache(path, result, "", filepath.Join(dir, "covers"), "stable-key") + if err != nil { + t.Fatal(err) + } + if result.TrackName != "Single Open" || result.CoverPath == "" { + t.Fatalf("scan result = %#v", result) + } + gotCover, err := os.ReadFile(result.CoverPath) + if err != nil || !bytes.Equal(gotCover, cover) { + t.Fatalf("cached cover = %x, %v", gotCover, err) + } +} + +func TestAudioQualityFromParsedFlacRejectsMissingStreamInfo(t *testing.T) { + if _, err := audioQualityFromParsedFlac(&flac.File{}); err == nil { + t.Fatal("missing STREAMINFO was accepted") + } +} + +func TestLibraryAudioCoverCacheKeyIncludesFileIdentity(t *testing.T) { + info := libraryAudioFileInfo{path: "track.flac", size: 123, modTime: 456} + if got := libraryAudioCoverCacheKey(info); got != "track.flac|123|456" { + t.Fatalf("cache key = %q", got) + } +} diff --git a/go_backend/metadata.go b/go_backend/metadata.go index 69a4355f..95859cee 100644 --- a/go_backend/metadata.go +++ b/go_backend/metadata.go @@ -323,7 +323,10 @@ func ReadMetadata(filePath string) (*Metadata, error) { return nil, fmt.Errorf("failed to parse FLAC file: %w", err) } defer f.Close() + return metadataFromParsedFlac(f), nil +} +func metadataFromParsedFlac(f *flac.File) *Metadata { metadata := &Metadata{} for _, meta := range f.Meta { @@ -399,7 +402,7 @@ func ReadMetadata(filePath string) (*Metadata, error) { } } - return metadata, nil + return metadata } // EditFlacFields opens a FLAC file and updates only the Vorbis Comment keys @@ -767,7 +770,10 @@ func ExtractCoverArt(filePath string) ([]byte, error) { return nil, fmt.Errorf("failed to parse FLAC file: %w", err) } defer f.Close() + return coverArtFromParsedFlac(f) +} +func coverArtFromParsedFlac(f *flac.File) ([]byte, error) { for _, meta := range f.Meta { if meta.Type == flac.Picture { pic, err := flacpicture.ParseFromMetaDataBlock(*meta) @@ -893,12 +899,15 @@ func ReadM4ATags(filePath string) (*AudioMetadata, error) { if err != nil { return nil, err } + return readM4ATagsFromIlst(f, fi.Size(), ilst) +} +func readM4ATagsFromIlst(f *os.File, fileSize int64, ilst atomHeader) (*AudioMetadata, error) { metadata := &AudioMetadata{} start := ilst.offset + ilst.headerSize end := ilst.offset + ilst.size for pos := start; pos+8 <= end; { - header, err := readAtomHeaderAt(f, pos, fi.Size()) + header, err := readAtomHeaderAt(f, pos, fileSize) if err != nil { return nil, err } @@ -911,32 +920,32 @@ func ReadM4ATags(filePath string) (*AudioMetadata, error) { switch header.typ { case "\xa9nam": - metadata.Title, _ = readM4ATextValue(f, header, fi.Size()) + metadata.Title, _ = readM4ATextValue(f, header, fileSize) case "\xa9ART": - metadata.Artist, _ = readM4ATextValue(f, header, fi.Size()) + metadata.Artist, _ = readM4ATextValue(f, header, fileSize) case "\xa9alb": - metadata.Album, _ = readM4ATextValue(f, header, fi.Size()) + metadata.Album, _ = readM4ATextValue(f, header, fileSize) case "aART": - metadata.AlbumArtist, _ = readM4ATextValue(f, header, fi.Size()) + metadata.AlbumArtist, _ = readM4ATextValue(f, header, fileSize) case "\xa9day": - metadata.Date, _ = readM4ATextValue(f, header, fi.Size()) + metadata.Date, _ = readM4ATextValue(f, header, fileSize) metadata.Year = metadata.Date case "\xa9gen": - metadata.Genre, _ = readM4ATextValue(f, header, fi.Size()) + metadata.Genre, _ = readM4ATextValue(f, header, fileSize) case "\xa9wrt": - metadata.Composer, _ = readM4ATextValue(f, header, fi.Size()) + metadata.Composer, _ = readM4ATextValue(f, header, fileSize) case "\xa9cmt": - metadata.Comment, _ = readM4ATextValue(f, header, fi.Size()) + metadata.Comment, _ = readM4ATextValue(f, header, fileSize) case "cprt": - metadata.Copyright, _ = readM4ATextValue(f, header, fi.Size()) + metadata.Copyright, _ = readM4ATextValue(f, header, fileSize) case "\xa9lyr": - metadata.Lyrics, _ = readM4ATextValue(f, header, fi.Size()) + metadata.Lyrics, _ = readM4ATextValue(f, header, fileSize) case "trkn": - metadata.TrackNumber, metadata.TotalTracks, _ = readM4AIndexPair(f, header, fi.Size()) + metadata.TrackNumber, metadata.TotalTracks, _ = readM4AIndexPair(f, header, fileSize) case "disk": - metadata.DiscNumber, metadata.TotalDiscs, _ = readM4AIndexPair(f, header, fi.Size()) + metadata.DiscNumber, metadata.TotalDiscs, _ = readM4AIndexPair(f, header, fileSize) case "----": - name, value, freeformErr := readM4AFreeformValue(f, header, fi.Size()) + name, value, freeformErr := readM4AFreeformValue(f, header, fileSize) if freeformErr == nil { switch strings.ToUpper(strings.TrimSpace(name)) { case "ISRC": @@ -1015,7 +1024,10 @@ func extractCoverFromM4A(filePath string) ([]byte, error) { if err != nil { return nil, err } + return extractCoverFromM4AIlst(f, fileSize, ilst) +} +func extractCoverFromM4AIlst(f *os.File, fileSize int64, ilst atomHeader) ([]byte, error) { bodyStart := ilst.offset + ilst.headerSize bodySize := ilst.size - ilst.headerSize @@ -1649,6 +1661,34 @@ type AudioQuality struct { Codec string `json:"codec,omitempty"` } +func audioQualityFromParsedFlac(f *flac.File) (AudioQuality, error) { + for _, meta := range f.Meta { + if meta.Type != flac.StreamInfo || len(meta.Data) < 18 { + continue + } + streamInfo := meta.Data + sampleRate := (int(streamInfo[10]) << 12) | (int(streamInfo[11]) << 4) | (int(streamInfo[12]) >> 4) + bitsPerSample := ((int(streamInfo[12]) & 0x01) << 4) | (int(streamInfo[13]) >> 4) + 1 + totalSamples := int64(streamInfo[13]&0x0F)<<32 | + int64(streamInfo[14])<<24 | + int64(streamInfo[15])<<16 | + int64(streamInfo[16])<<8 | + int64(streamInfo[17]) + duration := 0 + if sampleRate > 0 && totalSamples > 0 { + duration = int(totalSamples / int64(sampleRate)) + } + return AudioQuality{ + BitDepth: bitsPerSample, + SampleRate: sampleRate, + TotalSamples: totalSamples, + Duration: duration, + Codec: "flac", + }, nil + } + return AudioQuality{}, fmt.Errorf("FLAC STREAMINFO block not found") +} + func GetAudioQuality(filePath string) (AudioQuality, error) { file, err := os.Open(filePath) if err != nil { @@ -1727,7 +1767,10 @@ func GetM4AQuality(filePath string) (AudioQuality, error) { return AudioQuality{}, fmt.Errorf("failed to stat M4A file: %w", err) } fileSize := info.Size() + return m4aQualityFromFile(f, fileSize) +} +func m4aQualityFromFile(f *os.File, fileSize int64) (AudioQuality, error) { moovHeader, moovFound, err := findAtomInRange(f, 0, fileSize, "moov", fileSize) if err != nil { return AudioQuality{}, fmt.Errorf("failed to find moov atom: %w", err)