From 4d0f1120d2efc0215d06dc0e9fa2d13dd93d73d5 Mon Sep 17 00:00:00 2001 From: zarzet Date: Mon, 13 Jul 2026 19:39:23 +0700 Subject: [PATCH] perf(m4a): stream tag rewrites instead of buffering the whole file --- go_backend/ac4_config.go | 119 ++++++++++++++++++++++++++++-------- go_backend/ac4_metadata.go | 35 ++++++++--- go_backend/flac_save.go | 31 ---------- go_backend/m4a_edit.go | 35 ++++++++--- go_backend/m4a_stream.go | 120 +++++++++++++++++++++++++++++++++++++ go_backend/metadata.go | 38 ++++++++---- 6 files changed, 293 insertions(+), 85 deletions(-) create mode 100644 go_backend/m4a_stream.go diff --git a/go_backend/ac4_config.go b/go_backend/ac4_config.go index d05c9f95..cd52f8a3 100644 --- a/go_backend/ac4_config.go +++ b/go_backend/ac4_config.go @@ -207,23 +207,33 @@ func shiftChunkOffsets(data []byte, moov mp4Box, insertPos, delta int64) { }) } -// normalizeQuickTimeAudioToMP4 rewrites a QuickTime-flavored file (FFmpeg mov -// muxer output: ftyp brand "qt " and a version-1 sound sample entry) into a -// standard ISO MP4: an isom/mp42 brand and a plain version-0 AudioSampleEntry. -// Windows Media Foundation (and other strict parsers) reject the QuickTime -// flavor for AC-4 even when dac4 is present. -func normalizeQuickTimeAudioToMP4(data []byte) []byte { - if ftyp, ok := findChildMP4(data, 0, int64(len(data)), "ftyp"); ok { - if ftyp.body()+4 <= int64(len(data)) { - copy(data[ftyp.body():ftyp.body()+4], []byte("mp42")) - } - for p := ftyp.body() + 8; p+4 <= ftyp.end(); p += 4 { - if string(data[p:p+4]) == "qt " { - copy(data[p:p+4], []byte("isom")) - } +// normalizeQuickTimeBrandsInBuf rewrites a "qt " ftyp brand to isom/mp42 in +// place. Works on any buffer whose top level contains the ftyp box (whole file +// or the ftyp box alone). Returns whether anything changed. +func normalizeQuickTimeBrandsInBuf(data []byte) bool { + ftyp, ok := findChildMP4(data, 0, int64(len(data)), "ftyp") + if !ok { + return false + } + changed := false + if ftyp.body()+4 <= int64(len(data)) && string(data[ftyp.body():ftyp.body()+4]) != "mp42" { + copy(data[ftyp.body():ftyp.body()+4], []byte("mp42")) + changed = true + } + for p := ftyp.body() + 8; p+4 <= ftyp.end(); p += 4 { + if string(data[p:p+4]) == "qt " { + copy(data[p:p+4], []byte("isom")) + changed = true } } + return changed +} +// normalizeQuickTimeAudioEntry rewrites a version-1 QuickTime sound sample +// entry into a plain version-0 AudioSampleEntry, dropping the 16-byte v1 +// extension. base is the buffer's absolute file offset (0 for a whole-file +// buffer, the moov offset for a moov-only buffer). +func normalizeQuickTimeAudioEntry(data []byte, base int64) []byte { loc, ok := locateAC4Entry(data) if !ok { return data @@ -247,7 +257,7 @@ func normalizeQuickTimeAudioToMP4(data []byte) []byte { delta := int64(-16) binary.BigEndian.PutUint16(data[verPos:verPos+2], 0) - shiftChunkOffsets(data, loc.chain[0], extStart, delta) + shiftChunkOffsets(data, loc.chain[0], base+extStart, delta) for _, b := range loc.chain { growBoxSize(data, b, delta) } @@ -259,22 +269,50 @@ func normalizeQuickTimeAudioToMP4(data []byte) []byte { return out } +// normalizeQuickTimeAudioToMP4 rewrites a QuickTime-flavored file (FFmpeg mov +// muxer output: ftyp brand "qt " and a version-1 sound sample entry) into a +// standard ISO MP4: an isom/mp42 brand and a plain version-0 AudioSampleEntry. +// Windows Media Foundation (and other strict parsers) reject the QuickTime +// flavor for AC-4 even when dac4 is present. +func normalizeQuickTimeAudioToMP4(data []byte) []byte { + normalizeQuickTimeBrandsInBuf(data) + return normalizeQuickTimeAudioEntry(data, 0) +} + // EnsureAC4ConfigBox makes a decrypted AC-4 MP4 standards-compliant and // playable: it normalizes FFmpeg's QuickTime-flavored mov output to an ISO MP4 // and injects the AC-4 configuration box (dac4) into the ac-4 sample entry. The // dac4 box is copied verbatim from sourcePath (the original MP4, whose plaintext -// moov still carries it). No-op when the file has no AC-4 track. +// moov still carries it). No-op when the file has no AC-4 track. Only the ftyp +// and moov boxes are held in memory; the audio bulk is streamed. func EnsureAC4ConfigBox(decryptedPath, sourcePath string) error { - dst, err := os.ReadFile(decryptedPath) + f, err := os.Open(decryptedPath) if err != nil { return err } + info, err := f.Stat() + if err != nil { + f.Close() + return err + } + moovBuf, moovOffset, moovFound, err := loadTopLevelMP4Box(f, info.Size(), "moov") + if err != nil || !moovFound { + f.Close() + return err // parse/read failure, or no moov: nothing to do + } + ftypBuf, ftypOffset, ftypFound, err := loadTopLevelMP4Box(f, info.Size(), "ftyp") + f.Close() + if err != nil { + return err + } + moovLen := int64(len(moovBuf)) - if _, ok := locateAC4Entry(dst); !ok { + if _, ok := locateAC4Entry(moovBuf); !ok { return nil // not an AC-4 file; nothing to do } - dst = normalizeQuickTimeAudioToMP4(dst) + ftypChanged := ftypFound && normalizeQuickTimeBrandsInBuf(ftypBuf) + dst := normalizeQuickTimeAudioEntry(moovBuf, moovOffset) loc, ok := locateAC4Entry(dst) if !ok { @@ -288,27 +326,40 @@ func EnsureAC4ConfigBox(decryptedPath, sourcePath string) error { childStart := loc.entry.body() + hdrLen if _, has := findChildMP4(dst, childStart, loc.entry.end(), "dac4"); has { // Already has dac4; still persist any normalization changes. - return writeFileAtomic(decryptedPath, dst, 0o644) + return writeAC4Sections(decryptedPath, ftypChanged, ftypBuf, ftypOffset, dst, moovOffset, moovLen) } - src, err := os.ReadFile(sourcePath) + srcF, err := os.Open(sourcePath) if err != nil { return err } - srcMoov, ok := findChildMP4(src, 0, int64(len(src)), "moov") + srcInfo, err := srcF.Stat() + if err != nil { + srcF.Close() + return err + } + srcMoovBuf, _, srcFound, err := loadTopLevelMP4Box(srcF, srcInfo.Size(), "moov") + srcF.Close() + if err != nil { + return err + } + if !srcFound { + return fmt.Errorf("source has no moov") + } + srcMoov, ok := readMP4Box(srcMoovBuf, 0) if !ok { return fmt.Errorf("source has no moov") } - dac4Box, ok := findBoxBySignature(src, srcMoov.body(), srcMoov.end(), "dac4") + dac4Box, ok := findBoxBySignature(srcMoovBuf, srcMoov.body(), srcMoov.end(), "dac4") if !ok { return fmt.Errorf("dac4 not found in source") } - dac4 := append([]byte{}, src[dac4Box.offset:dac4Box.end()]...) + dac4 := append([]byte{}, srcMoovBuf[dac4Box.offset:dac4Box.end()]...) insertPos := childStart delta := int64(len(dac4)) - shiftChunkOffsets(dst, loc.chain[0], insertPos, delta) + shiftChunkOffsets(dst, loc.chain[0], moovOffset+insertPos, delta) for _, b := range loc.chain { growBoxSize(dst, b, delta) } @@ -319,5 +370,21 @@ func EnsureAC4ConfigBox(decryptedPath, sourcePath string) error { out = append(out, dac4...) out = append(out, dst[insertPos:]...) - return writeFileAtomic(decryptedPath, out, 0o644) + return writeAC4Sections(decryptedPath, ftypChanged, ftypBuf, ftypOffset, out, moovOffset, moovLen) +} + +// writeAC4Sections streams the edited moov (and, when changed, ftyp) back into +// the file. +func writeAC4Sections(path string, ftypChanged bool, ftypBuf []byte, ftypOffset int64, moovBuf []byte, moovOffset, origMoovLen int64) error { + sections := []fileSection{ + {start: moovOffset, end: moovOffset + origMoovLen, data: moovBuf}, + } + if ftypChanged { + sections = append(sections, fileSection{ + start: ftypOffset, + end: ftypOffset + int64(len(ftypBuf)), + data: ftypBuf, + }) + } + return replaceFileSectionsStreaming(path, sections) } diff --git a/go_backend/ac4_metadata.go b/go_backend/ac4_metadata.go index cd8543a7..7c8093ad 100644 --- a/go_backend/ac4_metadata.go +++ b/go_backend/ac4_metadata.go @@ -121,8 +121,10 @@ func buildITunesUdta(md ac4Metadata, cover []byte) []byte { } // writeMP4iTunesMetadata replaces (or inserts) a udta>meta>ilst metadata box in -// the moov of an MP4 buffer and returns the rewritten bytes. -func writeMP4iTunesMetadata(data []byte, md ac4Metadata, cover []byte) []byte { +// the moov of an MP4 buffer and returns the rewritten bytes. base is the +// buffer's absolute file offset (0 for a whole-file buffer) so stco/co64 +// shifts compare against the absolute positions the entries hold. +func writeMP4iTunesMetadata(data []byte, base int64, md ac4Metadata, cover []byte) []byte { moov, ok := findChildMP4(data, 0, int64(len(data)), "moov") if !ok { return data @@ -131,7 +133,7 @@ func writeMP4iTunesMetadata(data []byte, md ac4Metadata, cover []byte) []byte { if udta, ok := findChildMP4(data, moov.body(), moov.end(), "udta"); ok { delta := int64(len(newUdta)) - udta.size - shiftChunkOffsets(data, moov, udta.offset, delta) + shiftChunkOffsets(data, moov, base+udta.offset, delta) growBoxSize(data, moov, delta) out := make([]byte, 0, len(data)+len(newUdta)) out = append(out, data[:udta.offset]...) @@ -142,7 +144,7 @@ func writeMP4iTunesMetadata(data []byte, md ac4Metadata, cover []byte) []byte { delta := int64(len(newUdta)) insertPos := moov.end() - shiftChunkOffsets(data, moov, insertPos, delta) + shiftChunkOffsets(data, moov, base+insertPos, delta) growBoxSize(data, moov, delta) out := make([]byte, 0, len(data)+len(newUdta)) out = append(out, data[:insertPos]...) @@ -154,12 +156,27 @@ func writeMP4iTunesMetadata(data []byte, md ac4Metadata, cover []byte) []byte { // WriteAC4MetadataIfApplicable writes iTunes metadata into an AC-4 MP4. Returns // true when the file was an AC-4 track and metadata was written; false when the // file is not AC-4 (the caller should fall back to its normal metadata path). +// Only the moov box is held in memory; the audio bulk is streamed. func WriteAC4MetadataIfApplicable(decryptedPath, metadataJSON, coverPath string) (bool, error) { - data, err := os.ReadFile(decryptedPath) + f, err := os.Open(decryptedPath) if err != nil { return false, err } - if _, ok := locateAC4Entry(data); !ok { + info, err := f.Stat() + if err != nil { + f.Close() + return false, err + } + moovBuf, moovOffset, found, err := loadTopLevelMP4Box(f, info.Size(), "moov") + f.Close() + if err != nil { + return false, err + } + if !found { + return false, nil + } + moovLen := int64(len(moovBuf)) + if _, ok := locateAC4Entry(moovBuf); !ok { return false, nil } @@ -174,8 +191,10 @@ func WriteAC4MetadataIfApplicable(decryptedPath, metadataJSON, coverPath string) } } - out := writeMP4iTunesMetadata(data, md, cover) - if err := writeFileAtomic(decryptedPath, out, 0o644); err != nil { + out := writeMP4iTunesMetadata(moovBuf, moovOffset, md, cover) + if err := replaceFileSectionsStreaming(decryptedPath, []fileSection{ + {start: moovOffset, end: moovOffset + moovLen, data: out}, + }); err != nil { return false, err } return true, nil diff --git a/go_backend/flac_save.go b/go_backend/flac_save.go index fbc779d3..73e11f79 100644 --- a/go_backend/flac_save.go +++ b/go_backend/flac_save.go @@ -57,37 +57,6 @@ func saveFlacFile(f *flac.File, filePath string) error { return nil } -// writeFileAtomic replaces filePath's contents via temp+fsync+rename so an -// interrupted write never leaves a truncated file under the final name. -func writeFileAtomic(filePath string, data []byte, perm os.FileMode) error { - tmpPath := filePath + ".tag.partial" - os.Remove(tmpPath) - tmp, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) - if err != nil { - return err - } - if _, err := tmp.Write(data); err != nil { - tmp.Close() - os.Remove(tmpPath) - return err - } - if err := tmp.Sync(); err != nil { - tmp.Close() - os.Remove(tmpPath) - return err - } - if err := tmp.Close(); err != nil { - os.Remove(tmpPath) - return err - } - if err := os.Rename(tmpPath, filePath); err != nil { - os.Remove(tmpPath) - return err - } - syncDir(filepath.Dir(filePath)) - return nil -} - // syncDir best-effort fsyncs a directory so a just-renamed entry survives // power loss. Unsupported on some platforms/filesystems; errors are ignored. func syncDir(dir string) { diff --git a/go_backend/m4a_edit.go b/go_backend/m4a_edit.go index 5e393133..b8cc734a 100644 --- a/go_backend/m4a_edit.go +++ b/go_backend/m4a_edit.go @@ -101,8 +101,10 @@ func buildM4AMetaBox() []byte { // ensureM4AIlstInBuf returns a buffer guaranteed to contain the // moov>[udta>]meta>ilst chain, creating the missing tail at the end of the -// deepest existing ancestor. Chunk offsets are shifted for the inserted bytes. -func ensureM4AIlstInBuf(data []byte) ([]byte, m4aIlstLocation, error) { +// deepest existing ancestor. Chunk offsets are shifted for the inserted bytes; +// base is the buffer's absolute file offset (0 for a whole-file buffer) so the +// shift compares against the absolute positions stco/co64 entries hold. +func ensureM4AIlstInBuf(data []byte, base int64) ([]byte, m4aIlstLocation, error) { if loc, ok := locateM4AIlstInBuf(data); ok { return data, loc, nil } @@ -143,7 +145,7 @@ func ensureM4AIlstInBuf(data []byte) ([]byte, m4aIlstLocation, error) { growBoxSize(updated, b, delta) } if newMoov, ok := findChildMP4(updated, 0, int64(len(updated)), "moov"); ok { - shiftChunkOffsets(updated, newMoov, insertPos, delta) + shiftChunkOffsets(updated, newMoov, base+insertPos, delta) } loc, ok := locateM4AIlstInBuf(updated) @@ -191,14 +193,29 @@ func m4aIndexPairInBuf(data []byte, box mp4Box) (int, int) { // EditM4AFields updates only the ilst entries whose keys are explicitly // present in the fields map (set-or-clear semantics, mirroring EditFlacFields) // while preserving every other atom. Standard atoms, freeform ISRC/LABEL, and -// ReplayGain freeform tags are all written in a single file rewrite. +// ReplayGain freeform tags are all written in a single file rewrite. Only the +// moov box is held in memory; the audio bulk is streamed. func EditM4AFields(filePath string, fields map[string]string) error { - data, err := os.ReadFile(filePath) + f, err := os.Open(filePath) if err != nil { return err } + info, err := f.Stat() + if err != nil { + f.Close() + return err + } + moovBuf, moovOffset, found, err := loadTopLevelMP4Box(f, info.Size(), "moov") + f.Close() + if err != nil { + return err + } + if !found { + return fmt.Errorf("moov not found") + } + moovLen := int64(len(moovBuf)) - data, loc, err := ensureM4AIlstInBuf(data) + data, loc, err := ensureM4AIlstInBuf(moovBuf, moovOffset) if err != nil { return err } @@ -333,8 +350,10 @@ func EditM4AFields(filePath string, fields map[string]string) error { growBoxSize(updated, b, delta) } if moov, ok := findChildMP4(updated, 0, int64(len(updated)), "moov"); ok { - shiftChunkOffsets(updated, moov, loc.ilst.offset, delta) + shiftChunkOffsets(updated, moov, moovOffset+loc.ilst.offset, delta) } - return writeFileAtomic(filePath, updated, 0o644) + return replaceFileSectionsStreaming(filePath, []fileSection{ + {start: moovOffset, end: moovOffset + moovLen, data: updated}, + }) } diff --git a/go_backend/m4a_stream.go b/go_backend/m4a_stream.go new file mode 100644 index 00000000..4c4928fe --- /dev/null +++ b/go_backend/m4a_stream.go @@ -0,0 +1,120 @@ +package gobackend + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" +) + +// Streaming support for MP4 tag rewrites: only the boxes being edited (moov, +// ftyp) are held in memory while the mdat bulk is streamed between file +// handles, so peak memory tracks the moov size instead of the file size. + +// loadTopLevelMP4Box scans f's top-level boxes and loads only the first box of +// the given type into memory, returning its bytes and absolute file offset. +// ok is false when the box does not exist; err reports read/parse failures. +func loadTopLevelMP4Box(f *os.File, fileSize int64, typ string) (buf []byte, offset int64, ok bool, err error) { + for pos := int64(0); pos+8 <= fileSize; { + header, err := readAtomHeaderAt(f, pos, fileSize) + if err != nil { + return nil, 0, false, err + } + size := header.size + if size == 0 { + size = fileSize - pos + } + if size < header.headerSize || pos+size > fileSize { + return nil, 0, false, fmt.Errorf("invalid atom size for %s", header.typ) + } + if header.typ == typ { + buf := make([]byte, size) + if _, err := f.ReadAt(buf, pos); err != nil { + return nil, 0, false, err + } + return buf, pos, true, nil + } + pos += size + } + return nil, 0, false, nil +} + +// fileSection is one byte range [start,end) to substitute during a streaming +// rewrite. +type fileSection struct { + start, end int64 + data []byte +} + +// replaceFileSectionsStreaming rewrites filePath with each section replaced by +// its data, streaming every byte outside the sections, and publishes via +// temp+fsync+rename so an interruption never leaves a truncated file under the +// final name. Sections must not overlap; they are sorted internally. +func replaceFileSectionsStreaming(filePath string, sections []fileSection) error { + sorted := append([]fileSection{}, sections...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].start < sorted[j].start }) + for i := 1; i < len(sorted); i++ { + if sorted[i].start < sorted[i-1].end { + return fmt.Errorf("overlapping file sections") + } + } + + src, err := os.Open(filePath) + if err != nil { + return err + } + + tmpPath := filePath + ".tag.partial" + os.Remove(tmpPath) + tmp, err := os.Create(tmpPath) + if err != nil { + src.Close() + return err + } + fail := func(err error) error { + tmp.Close() + src.Close() + os.Remove(tmpPath) + return err + } + + pos := int64(0) + for _, sec := range sorted { + if sec.start < pos { + return fail(fmt.Errorf("file section out of order")) + } + if _, err := io.CopyN(tmp, src, sec.start-pos); err != nil { + return fail(err) + } + if _, err := tmp.Write(sec.data); err != nil { + return fail(err) + } + if _, err := src.Seek(sec.end, io.SeekStart); err != nil { + return fail(err) + } + pos = sec.end + } + if _, err := io.Copy(tmp, src); err != nil { + return fail(err) + } + if err := tmp.Sync(); err != nil { + return fail(err) + } + if err := tmp.Close(); err != nil { + src.Close() + os.Remove(tmpPath) + return err + } + // Release the read handle before the rename (required on Windows). + if err := src.Close(); err != nil { + os.Remove(tmpPath) + return err + } + if err := os.Rename(tmpPath, filePath); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("failed to publish tagged file: %w", err) + } + syncDir(filepath.Dir(filePath)) + return nil +} diff --git a/go_backend/metadata.go b/go_backend/metadata.go index 4cb675db..0d463c19 100644 --- a/go_backend/metadata.go +++ b/go_backend/metadata.go @@ -1480,8 +1480,10 @@ func buildITunNORMTag(trackGain, trackPeak string) string { return strings.Join(parts, " ") } +var replayGainNumberPattern = regexp.MustCompile(`([+-]?\d+(?:\.\d+)?)`) + func parseReplayGainDb(value string) (float64, bool) { - match := regexp.MustCompile(`([+-]?\d+(?:\.\d+)?)`).FindStringSubmatch(strings.TrimSpace(value)) + match := replayGainNumberPattern.FindStringSubmatch(strings.TrimSpace(value)) if len(match) < 2 { return 0, false } @@ -1618,8 +1620,10 @@ func writeM4AFreeformTags(filePath string, remove map[string]struct{}, tags []m4 return err } - data, err := os.ReadFile(filePath) - if err != nil { + // Only the moov box is buffered; the audio bulk is streamed on write. + base := path.moov.offset + moovBuf := make([]byte, path.moov.size) + if _, err := f.ReadAt(moovBuf, base); err != nil { return err } @@ -1649,7 +1653,7 @@ func writeM4AFreeformTags(filePath string, remove map[string]struct{}, tags []m4 } } if keep { - newBody = append(newBody, data[pos:pos+header.size]...) + newBody = append(newBody, moovBuf[pos-base:pos-base+header.size]...) } pos += header.size @@ -1663,27 +1667,35 @@ func writeM4AFreeformTags(filePath string, remove map[string]struct{}, tags []m4 } newIlst := buildM4AAtom("ilst", newBody) - updated := append([]byte{}, data[:path.ilst.offset]...) + ilstRel := path.ilst.offset - base + updated := append([]byte{}, moovBuf[:ilstRel]...) updated = append(updated, newIlst...) - updated = append(updated, data[path.ilst.offset+path.ilst.size:]...) + updated = append(updated, moovBuf[ilstRel+path.ilst.size:]...) + // The path headers carry absolute file offsets; rebase them onto the + // moov-rooted buffer before patching sizes. + rel := func(h atomHeader) atomHeader { + h.offset -= base + return h + } delta := int64(len(newIlst)) - path.ilst.size - if err := writeAtomSize(updated, path.ilst, path.ilst.size+delta); err != nil { + if err := writeAtomSize(updated, rel(path.ilst), path.ilst.size+delta); err != nil { return err } - if err := writeAtomSize(updated, path.meta, path.meta.size+delta); err != nil { + if err := writeAtomSize(updated, rel(path.meta), path.meta.size+delta); err != nil { return err } if path.udta != nil { - if err := writeAtomSize(updated, *path.udta, path.udta.size+delta); err != nil { + if err := writeAtomSize(updated, rel(*path.udta), path.udta.size+delta); err != nil { return err } } - if err := writeAtomSize(updated, path.moov, path.moov.size+delta); err != nil { + if err := writeAtomSize(updated, rel(path.moov), path.moov.size+delta); err != nil { return err } // Keep sample pointers valid when moov precedes mdat: every stco/co64 - // entry at or beyond the resized ilst must shift with it. + // entry at or beyond the resized ilst must shift with it. Entries hold + // absolute file offsets, so compare against the absolute ilst position. if delta != 0 { if moov, ok := findChildMP4(updated, 0, int64(len(updated)), "moov"); ok { shiftChunkOffsets(updated, moov, path.ilst.offset, delta) @@ -1692,7 +1704,9 @@ func writeM4AFreeformTags(filePath string, remove map[string]struct{}, tags []m4 // Release the read handle before replacing the file (required on Windows). f.Close() - return writeFileAtomic(filePath, updated, 0o644) + return replaceFileSectionsStreaming(filePath, []fileSection{ + {start: base, end: base + path.moov.size, data: updated}, + }) } // EditM4AFreeformText writes ISRC and label tags into an M4A/MP4 file as iTunes