diff --git a/lib/services/audio_analysis_ffmpeg.dart b/lib/services/audio_analysis_ffmpeg.dart index 2949e6e5c..988b4f95b 100644 --- a/lib/services/audio_analysis_ffmpeg.dart +++ b/lib/services/audio_analysis_ffmpeg.dart @@ -59,12 +59,15 @@ String _buildShowspectrumOptions({required int width, required String color}) { 'limit=0'; } +/// [inputLabel] names the audio to draw; another graph that ends in that +/// label (such as a null test residual) can precede this filter. String buildAudioSpectrogramFilter({ int channel = -1, bool includeCutoffPlane = false, double? durationSeconds, int? sampleRate, int? channels, + String inputLabel = '[0:a:0]', }) { final channelFilter = channel >= 0 ? 'pan=mono|c0=c$channel,' : ''; final samplingFilter = _buildAudioSpectrogramSamplingFilter( @@ -74,7 +77,7 @@ String buildAudioSpectrogramFilter({ channels: channels, ); final input = - '[0:a:0]$channelFilter${samplingFilter}aformat=sample_fmts=fltp'; + '$inputLabel$channelFilter${samplingFilter}aformat=sample_fmts=fltp'; final display = _buildShowspectrumOptions( width: audioSpectrogramWidth, color: 'intensity', @@ -261,15 +264,17 @@ int? _parseLastAstatsInt(String text, String label) { return value; } +/// Escapes a file path for a quoted filter option such as +/// `ametadata=print:file='...'`. +String escapeFFmpegFilterPath(String path) => + path.replaceAll(r'\', r'\\').replaceAll(':', r'\:').replaceAll("'", r"\'"); + String buildAudioMetricsFilter({ required double durationSeconds, required String metadataPath, }) { final metadataStart = math.max(0.0, durationSeconds - 2.0); - final escapedPath = metadataPath - .replaceAll(r'\', r'\\') - .replaceAll(':', r'\:') - .replaceAll("'", r"\'"); + final escapedPath = escapeFFmpegFilterPath(metadataPath); return 'astats=metadata=1:reset=0:' 'measure_perchannel=Peak_level+RMS_level+Peak_count:' 'measure_overall=Peak_level+RMS_level,' diff --git a/lib/services/audio_analysis_runner.dart b/lib/services/audio_analysis_runner.dart index de2c64fd4..da5232d56 100644 --- a/lib/services/audio_analysis_runner.dart +++ b/lib/services/audio_analysis_runner.dart @@ -3,6 +3,7 @@ import 'dart:io'; import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; +import 'package:ffmpeg_kit_flutter_new_audio/media_information.dart'; import 'package:flutter/services.dart'; import 'package:path_provider/path_provider.dart'; import 'package:mornye/services/audio_analysis_ffmpeg.dart'; @@ -60,7 +61,8 @@ double? _estimateEffectiveSpectralCutoffFileInIsolate( ); } -class _MediaInfo { +/// What the analysers read from an ffprobe result. +class AudioAnalysisMediaInfo { final int fileSize; final String codecName; final String codec; @@ -74,7 +76,7 @@ class _MediaInfo { final AudioAnalysisBitrate bitrate; final int totalSamples; - const _MediaInfo({ + const AudioAnalysisMediaInfo({ required this.fileSize, required this.codecName, required this.codec, @@ -412,7 +414,7 @@ class AudioAnalysisRunner { } } - Future<_MediaInfo> _getMediaInfo(String filePath) async { + Future _getMediaInfo(String filePath) async { final info = await FFmpegExecution.mediaInformation(filePath); if (info == null) { @@ -424,6 +426,15 @@ class AudioAnalysisRunner { fileSize = await File(filePath).length(); } catch (_) {} + return parseMediaInfo(info, fileSize: fileSize); + } + + /// Reads the first audio stream of an ffprobe result. Throws when the + /// file has no audio stream. + static AudioAnalysisMediaInfo parseMediaInfo( + MediaInformation info, { + int fileSize = 0, + }) { final streams = info.getStreams(); final audioStream = streams.firstWhere( (s) => s.getAllProperties()?['codec_type'] == 'audio', @@ -487,7 +498,7 @@ class AudioAnalysisRunner { } } - return _MediaInfo( + return AudioAnalysisMediaInfo( fileSize: fileSize, codecName: codecName, codec: _formatCodecLabel(codecName, codecLongName), @@ -508,7 +519,7 @@ class AudioAnalysisRunner { ); } - String _formatCodecLabel(String codecName, String codecLongName) { + static String _formatCodecLabel(String codecName, String codecLongName) { final name = codecName.trim(); final longName = _normalizeAnalysisLabel(codecLongName); if (name.isEmpty) return longName; @@ -518,21 +529,24 @@ class AudioAnalysisRunner { return '${name.toUpperCase()} ($longName)'; } - String _formatContainerLabel(String formatName, String formatLongName) { + static String _formatContainerLabel( + String formatName, + String formatLongName, + ) { final longName = _normalizeAnalysisLabel(formatLongName); if (longName.isNotEmpty) return longName; final name = formatName.trim(); return name.isEmpty ? '' : name.toUpperCase(); } - String _normalizeAnalysisLabel(String value) { + static String _normalizeAnalysisLabel(String value) { final trimmed = value.trim(); final lower = trimmed.toLowerCase(); if (lower.isEmpty || lower == 'unknown' || lower == 'n/a') return ''; return trimmed; } - int _estimateTotalSamples({ + static int _estimateTotalSamples({ required Map props, required double duration, required int sampleRate, @@ -565,7 +579,7 @@ class AudioAnalysisRunner { return 0; } - bool _codecHasStoredBitDepth(String codecName) { + static bool _codecHasStoredBitDepth(String codecName) { if (codecName.isEmpty) return false; return codecName == 'flac' || codecName == 'alac' || diff --git a/lib/services/audio_compare_dsp.dart b/lib/services/audio_compare_dsp.dart new file mode 100644 index 000000000..6576c631c --- /dev/null +++ b/lib/services/audio_compare_dsp.dart @@ -0,0 +1,965 @@ +// Pure signal math for the "Compare with another file" null test. FFmpeg +// decodes and subtracts whole songs; this file aligns short excerpts, picks +// the gain, and turns per-window levels into the summary the sheet shows. +import 'dart:math' as math; +import 'dart:typed_data'; + +/// Sample rate of the decimated excerpts used by the wide alignment search. +/// The band below 2 kHz survives every lossy encoder, so it lines up a lossy +/// copy as reliably as the full band while keeping the search cheap. +const int audioCompareCoarseRate = 4000; + +/// How far the other file may start earlier or later than the reference. +const double audioCompareSearchSeconds = 3.0; + +/// Length of the reference stretch decoded for the wide search; the loudest +/// [audioCompareMatchSeconds] of it are correlated. +const double audioCompareScanSeconds = 30.0; +const double audioCompareMatchSeconds = 10.0; + +/// Full-rate window used to refine the offset and to estimate the gain. +const double audioCompareFineSeconds = 3.0; + +/// The comparison runs at the reference's sample rate up to this rate. +const int audioCompareMaxSampleRate = 96000; + +/// About this many points describe the whole song on the timeline. +const int audioCompareTargetPoints = 1000; + +/// Shortest window for very short files, in seconds. +const double audioCompareMinWindowSeconds = 0.005; + +/// Lowest value on the timeline: the files cancel completely. +const double audioCompareFloorDb = -120; + +/// Highest value kept for a window (the residual can exceed the reference +/// when the files are unrelated). +const double audioCompareCeilingDb = 20; + +/// A reference window quieter than this is treated as silence: its relative +/// level would only magnify noise, so it is measured against this level and +/// left out of the median and the peak. +const double audioCompareSilenceDb = -80; + +// Thresholds below were chosen from synthetic experiments: a 40 s tone and +// pink-noise mix encoded with FFmpeg 9 (LAME MP3 at 32/64/128 kbps, the +// native AAC encoder at 128 kbps, an 8 kHz low-pass, a treble cut, a bass +// boost plus compression as a "remaster", a 0.01% speed change as drift, and +// unrelated white noise). Measured on 5 s excerpts: +// +// case correlation residual +// AAC 128 kbps 0.9999 -37 dB +// MP3 128 kbps 0.9996 -31 dB +// MP3 32 kbps 0.996 -21 dB +// 8 kHz low-pass 0.998 -24 dB +// remaster (EQ + comp.) 0.991 -17 dB +// 0.01% drift 0.879 -6 dB +// unrelated noise 0.006 0 dB +// +// The gaps between same-source copies and the rest are wide, so the +// thresholds sit in the middle of them rather than at the edges. Whole-song +// runs of the engine on a 90 s mix (test/audio_compare_engine_test.dart) +// agree: lossy copies -32 to -38 dB overall at correlation 0.999 or more, a +// 48 kHz resample -53 dB, a 3 dB quieter 24-bit copy -99 dB, the drifting +// copy +2.5 dB at 0.92 with 16 ms of drift, unrelated noise 0 dB at 0.01. + +/// Below this normalized correlation peak the files did not line up clearly. +const double audioCompareMinCorrelation = 0.5; + +/// A whole-song or median residual above this level means the files are not +/// the same source (or are misaligned), so the null test is not meaningful. +const double audioCompareNotSameSourceDb = -10; + +/// Lengths that differ by more than this after the offset suggest an edit. +const double audioCompareDurationToleranceSeconds = 3; + +/// Offsets measured near the start and near the end of the song that differ +/// by more than this many coarse samples (0.5 ms) mean the timing drifts. +const int audioCompareDriftToleranceCoarse = 2; + +/// Boundary between the low and high bands of the plain-language reading. +const double audioCompareBandSplitHz = 8000; + +/// One band's relative residual this much above the other's makes the +/// differences "concentrated" there; within [audioCompareBandEvenDb] they +/// cover the whole spectrum. +const double audioCompareBandConcentrationDb = 10; +const double audioCompareBandEvenDb = 3; + +/// Window length, in frames, that gives about [audioCompareTargetPoints] +/// points for [totalFrames]. +int audioCompareWindowFrames(int totalFrames, int sampleRate) { + final minimum = math.max( + 1, + (sampleRate * audioCompareMinWindowSeconds).round(), + ); + if (totalFrames <= 0) return minimum; + return math.max(minimum, (totalFrames / audioCompareTargetPoints).ceil()); +} + +/// Power (mean square) to dB, with [audioCompareFloorDb] for silence. +double audioComparePowerDb(double power) { + if (!power.isFinite || power <= 0) return audioCompareFloorDb; + return math.max(audioCompareFloorDb, 10 * math.log(power) / math.ln10); +} + +/// Mean square from an FFmpeg `RMS_level` value in dBFS. +double audioCompareDbToPower(double db) { + if (db.isNaN || db == double.negativeInfinity) return 0; + if (db == double.infinity) return double.maxFinite; + return math.pow(10, db / 10).toDouble(); +} + +final double _silencePower = audioCompareDbToPower(audioCompareSilenceDb); + +/// The residual of one window relative to the reference, in dB. +double audioCompareRelativeDb(double residual, double reference) { + if (!residual.isFinite || residual <= 0) return audioCompareFloorDb; + if (reference < _silencePower && residual < _silencePower) { + return audioCompareFloorDb; + } + final db = + 10 * math.log(residual / math.max(reference, _silencePower)) / math.ln10; + return db.clamp(audioCompareFloorDb, audioCompareCeilingDb).toDouble(); +} + +/// Which channel the timeline shows. +enum AudioCompareChannel { combined, left, right } + +/// Levels of one timeline window, as mean squares per channel. +class AudioCompareWindow { + const AudioCompareWindow({ + required this.startSeconds, + required this.frames, + required this.residual, + required this.reference, + required this.residualHigh, + required this.referenceHigh, + }); + + final double startSeconds; + final int frames; + final Float64List residual; + final Float64List reference; + + /// The same levels above [audioCompareBandSplitHz]. + final Float64List residualHigh; + final Float64List referenceHigh; + + static double _pick(Float64List values, AudioCompareChannel channel) { + if (values.isEmpty) return 0; + switch (channel) { + case AudioCompareChannel.left: + return values[0]; + case AudioCompareChannel.right: + return values.length > 1 ? values[1] : values[0]; + case AudioCompareChannel.combined: + var sum = 0.0; + for (final value in values) { + sum += value; + } + return sum / values.length; + } + } + + double residualPower(AudioCompareChannel channel) => _pick(residual, channel); + double referencePower(AudioCompareChannel channel) => + _pick(reference, channel); + + double relativeDb(AudioCompareChannel channel) => + audioCompareRelativeDb(residualPower(channel), referencePower(channel)); + + bool isSilent(AudioCompareChannel channel) => + referencePower(channel) < _silencePower; +} + +/// The timeline values of [windows] for [channel], in dB. +List audioCompareSeries( + List windows, + AudioCompareChannel channel, +) => [for (final window in windows) window.relativeDb(channel)]; + +/// The whole-song residual relative to the reference, in dB. +double audioCompareOverallDb( + List windows, [ + AudioCompareChannel channel = AudioCompareChannel.combined, +]) { + var residual = 0.0; + var reference = 0.0; + for (final window in windows) { + residual += window.residualPower(channel) * window.frames; + reference += window.referencePower(channel) * window.frames; + } + if (residual <= 0) return audioCompareFloorDb; + if (reference <= 0) return audioCompareCeilingDb; + return (10 * math.log(residual / reference) / math.ln10) + .clamp(audioCompareFloorDb, audioCompareCeilingDb) + .toDouble(); +} + +// --------------------------------------------------------------------------- +// Alignment + +/// The best lag of one correlation search. +class AudioCompareLagMatch { + const AudioCompareLagMatch(this.lag, this.correlation); + + /// Index in the searched excerpt where the reference excerpt starts. + final int lag; + + /// Normalized correlation at [lag], -1 to 1; negative for inverted + /// polarity. + final double correlation; +} + +/// In-place radix-2 FFT of `re + i*im`; the length must be a power of two. +void audioCompareFft(Float64List re, Float64List im, {bool inverse = false}) { + final n = re.length; + if (n <= 1) return; + for (var i = 1, j = 0; i < n; i++) { + var bit = n >> 1; + for (; j & bit != 0; bit >>= 1) { + j ^= bit; + } + j ^= bit; + if (i < j) { + final tr = re[i]; + re[i] = re[j]; + re[j] = tr; + final ti = im[i]; + im[i] = im[j]; + im[j] = ti; + } + } + // One twiddle table for every stage keeps rounding error flat. + final half = n >> 1; + final cosTable = Float64List(half); + final sinTable = Float64List(half); + final sign = inverse ? 1.0 : -1.0; + for (var k = 0; k < half; k++) { + final angle = 2 * math.pi * k / n; + cosTable[k] = math.cos(angle); + sinTable[k] = sign * math.sin(angle); + } + for (var size = 2; size <= n; size <<= 1) { + final halfSize = size >> 1; + final step = n ~/ size; + for (var start = 0; start < n; start += size) { + for (var k = 0; k < halfSize; k++) { + final wr = cosTable[k * step]; + final wi = sinTable[k * step]; + final a = start + k; + final b = a + halfSize; + final xr = re[b] * wr - im[b] * wi; + final xi = re[b] * wi + im[b] * wr; + re[b] = re[a] - xr; + im[b] = im[a] - xi; + re[a] += xr; + im[a] += xi; + } + } + } + if (inverse) { + final scale = 1 / n; + for (var i = 0; i < n; i++) { + re[i] *= scale; + im[i] *= scale; + } + } +} + +/// `c[k] = sum(a[n] * b[n + k])` for every k with `a` inside `b`. +Float64List audioCompareCrossCorrelation( + Float32List a, + Float32List b, { + bool? useFft, +}) { + final lags = b.length - a.length + 1; + if (lags <= 0 || a.isEmpty) return Float64List(0); + final fft = useFft ?? a.length.toDouble() * lags > 2e7; + final result = Float64List(lags); + if (!fft) { + for (var k = 0; k < lags; k++) { + var sum = 0.0; + for (var n = 0; n < a.length; n++) { + sum += a[n] * b[n + k]; + } + result[k] = sum; + } + return result; + } + // Circular correlation is exact for these lags once the size covers b. + var size = 1; + while (size < b.length) { + size <<= 1; + } + final aRe = Float64List(size)..setRange(0, a.length, a); + final aIm = Float64List(size); + final bRe = Float64List(size)..setRange(0, b.length, b); + final bIm = Float64List(size); + audioCompareFft(aRe, aIm); + audioCompareFft(bRe, bIm); + for (var i = 0; i < size; i++) { + // conj(A) * B + final re = aRe[i] * bRe[i] + aIm[i] * bIm[i]; + final im = aRe[i] * bIm[i] - aIm[i] * bRe[i]; + bRe[i] = re; + bIm[i] = im; + } + audioCompareFft(bRe, bIm, inverse: true); + result.setRange(0, lags, bRe); + return result; +} + +/// Finds where [reference] best matches inside [searched], by normalized +/// cross-correlation. Polarity is ignored for the search and reported in +/// the sign of the correlation. +AudioCompareLagMatch audioCompareFindLag( + Float32List reference, + Float32List searched, { + bool? useFft, +}) { + final candidates = audioCompareLagCandidates( + reference, + searched, + count: 1, + useFft: useFft, + ); + return candidates.isEmpty + ? const AudioCompareLagMatch(0, 0) + : candidates.first; +} + +/// The [count] strongest separate correlation peaks, strongest first, at +/// least [separation] lags apart. Music that nearly repeats (a held chord, +/// a loop) has several peaks of almost the same height; the full-rate step +/// decides between them. +List audioCompareLagCandidates( + Float32List reference, + Float32List searched, { + int count = 5, + int separation = 8, + bool? useFft, +}) { + final lags = searched.length - reference.length + 1; + if (reference.isEmpty || lags <= 0) return const []; + var referenceEnergy = 0.0; + for (final value in reference) { + referenceEnergy += value * value; + } + if (referenceEnergy <= 0) return const []; + final correlation = audioCompareCrossCorrelation( + reference, + searched, + useFft: useFft, + ); + final energy = Float64List(searched.length + 1); + for (var i = 0; i < searched.length; i++) { + energy[i + 1] = energy[i] + searched[i] * searched[i]; + } + final normalized = Float64List(lags); + for (var k = 0; k < lags; k++) { + final searchedEnergy = energy[k + reference.length] - energy[k]; + // Running sums can leave a tiny residue over digital silence. + if (searchedEnergy <= referenceEnergy * 1e-12) continue; + normalized[k] = + (correlation[k] / math.sqrt(referenceEnergy * searchedEnergy)) + .clamp(-1.0, 1.0) + .toDouble(); + } + final peaks = []; + for (var k = 0; k < lags; k++) { + final value = normalized[k].abs(); + if (value == 0) continue; + if (k > 0 && normalized[k - 1].abs() > value) continue; + if (k < lags - 1 && normalized[k + 1].abs() > value) continue; + peaks.add(k); + } + peaks.sort((a, b) => normalized[b].abs().compareTo(normalized[a].abs())); + // Rank the leading peaks by their height between samples: at a low rate + // the true lag usually falls between two samples, and a near-repeat that + // happens to land on a sample would otherwise outrank it. + final ranked = peaks.take(math.max(count * 8, 32)).toList(); + final heights = { + for (final k in ranked) k: _interpolatedPeakHeight(normalized, k), + }; + ranked.sort((a, b) => heights[b]!.compareTo(heights[a]!)); + peaks + ..removeRange(0, ranked.length) + ..insertAll(0, ranked); + final chosen = []; + for (final k in peaks) { + if (chosen.length >= count) break; + if (chosen.any((match) => (match.lag - k).abs() < separation)) continue; + chosen.add(AudioCompareLagMatch(k, normalized[k])); + } + return chosen; +} + +/// The largest absolute value of a band-limited [values] near index [k], +/// found by windowed-sinc interpolation at tenth-sample steps. +double _interpolatedPeakHeight(Float64List values, int k) { + const taps = 16; + var best = values[k].abs(); + for (var step = -5; step <= 5; step++) { + if (step == 0) continue; + final position = step / 10; + var sum = 0.0; + for (var j = -taps; j <= taps; j++) { + final index = k + j; + if (index < 0 || index >= values.length) continue; + final x = position - j; + final window = 0.5 * (1 + math.cos(math.pi * x / (taps + 1))); + final sinc = math.sin(math.pi * x) / (math.pi * x); + sum += values[index] * sinc * window; + } + best = math.max(best, sum.abs()); + } + return math.min(best, 1.0); +} + +/// Start of the loudest [length]-sample stretch of [samples], tried every +/// [step] samples. +int audioCompareLoudestStart(Float32List samples, int length, int step) { + if (samples.length <= length) return 0; + final energy = Float64List(samples.length + 1); + for (var i = 0; i < samples.length; i++) { + energy[i + 1] = energy[i] + samples[i] * samples[i]; + } + var best = 0; + var bestEnergy = -1.0; + for (var start = 0; start + length <= samples.length; start += step) { + final value = energy[start + length] - energy[start]; + if (value > bestEnergy) { + bestEnergy = value; + best = start; + } + } + return best; +} + +/// The average of [channels] interleaved channels. +Float32List audioCompareMixToMono(Float32List interleaved, int channels) { + if (channels <= 1) return interleaved; + final frames = interleaved.length ~/ channels; + final mono = Float32List(frames); + for (var frame = 0; frame < frames; frame++) { + var sum = 0.0; + for (var channel = 0; channel < channels; channel++) { + sum += interleaved[frame * channels + channel]; + } + mono[frame] = sum / channels; + } + return mono; +} + +/// The single gain `g` that minimizes `sum((reference - g * other)^2)`. +double audioCompareLeastSquaresGain(Float32List reference, Float32List other) { + final length = math.min(reference.length, other.length); + var cross = 0.0; + var otherEnergy = 0.0; + for (var i = 0; i < length; i++) { + cross += reference[i] * other[i]; + otherEnergy += other[i] * other[i]; + } + if (otherEnergy <= 0) return 1; + return cross / otherEnergy; +} + +/// Inputs of the wide search, all at [audioCompareCoarseRate] in mono. +class AudioCompareCoarseInput { + const AudioCompareCoarseInput({ + required this.reference, + required this.referenceStart, + required this.other, + required this.otherStart, + }); + + /// Reference scan excerpt and its first sample's index in the song. + final Float32List reference; + final int referenceStart; + + /// Other file's excerpt, covering the scan plus the search margin. + final Float32List other; + final int otherStart; +} + +/// Result of the wide search. +class AudioCompareCoarseMatch { + const AudioCompareCoarseMatch({ + required this.candidates, + required this.fineCenter, + }); + + /// Lags (other-file index minus reference index of the same sound, in + /// coarse samples) of the strongest correlation peaks, strongest first. + final List candidates; + + int get lag => candidates.isEmpty ? 0 : candidates.first.lag; + double get correlation => + candidates.isEmpty ? 0 : candidates.first.correlation; + + /// The candidate lag nearest to [lag] among the peaks almost as strong as + /// the best one, so a near-repeat does not pass for timing drift. + int lagNearest(int lag) { + final strongest = correlation.abs(); + var best = this.lag; + for (final candidate in candidates) { + if (candidate.correlation.abs() < strongest - 0.02) continue; + if ((candidate.lag - lag).abs() < (best - lag).abs()) { + best = candidate.lag; + } + } + return best; + } + + /// Reference index (coarse) at the middle of the loudest second of the + /// matched stretch, where the full-rate refinement looks. + final int fineCenter; +} + +/// Correlates the loudest [audioCompareMatchSeconds] of the reference scan +/// against the other excerpt, within [audioCompareSearchSeconds] each way. +AudioCompareCoarseMatch audioCompareCoarseAlign(AudioCompareCoarseInput input) { + const rate = audioCompareCoarseRate; + final radius = (audioCompareSearchSeconds * rate).round(); + // Keep the matched stretch where the other excerpt covers the whole search + // range around it; a file that ends one sample early must not hide the + // true lag. Very short excerpts fall back to the whole scan. + var regionStart = 0; + var regionEnd = input.reference.length; + final coveredStart = input.otherStart + radius - input.referenceStart; + final coveredEnd = + input.otherStart + input.other.length - radius - input.referenceStart; + if (coveredEnd - coveredStart >= 2 * rate) { + regionStart = coveredStart.clamp(0, regionEnd); + regionEnd = coveredEnd.clamp(regionStart, regionEnd); + } + final region = Float32List.sublistView( + input.reference, + regionStart, + regionEnd, + ); + final matchLength = math.min( + region.length, + (audioCompareMatchSeconds * rate).round(), + ); + final start = + regionStart + audioCompareLoudestStart(region, matchLength, rate ~/ 4); + final matched = Float32List.sublistView( + input.reference, + start, + start + matchLength, + ); + final referenceIndex = input.referenceStart + start; + final secondLength = math.min(rate, matchLength); + final loudSecond = audioCompareLoudestStart( + matched, + secondLength, + rate ~/ 10, + ); + final fineCenter = referenceIndex + loudSecond + secondLength ~/ 2; + final from = (referenceIndex - radius - input.otherStart).clamp( + 0, + input.other.length, + ); + final to = (referenceIndex + matchLength + radius - input.otherStart).clamp( + from, + input.other.length, + ); + if (matchLength == 0 || to - from < matchLength) { + return AudioCompareCoarseMatch( + candidates: const [], + fineCenter: fineCenter, + ); + } + final matches = audioCompareLagCandidates( + matched, + Float32List.sublistView(input.other, from, to), + count: 8, + ); + final strongest = matches.fold( + 0.0, + (best, match) => math.max(best, match.correlation.abs()), + ); + return AudioCompareCoarseMatch( + candidates: [ + for (final match in matches) + // Peaks far below the best one are not worth a full-rate look. + if (match.correlation.abs() >= strongest * 0.9) + AudioCompareLagMatch( + input.otherStart + from + match.lag - referenceIndex, + match.correlation, + ), + ], + fineCenter: fineCenter, + ); +} + +/// One place in the other file where the full-rate step looks. +class AudioCompareFineCandidate { + const AudioCompareFineCandidate({ + required this.other, + required this.otherStart, + required this.predictedLag, + }); + + /// Interleaved other-file excerpt and its first frame's index. + final Float32List other; + final int otherStart; + + /// A coarse lag scaled to the comparison rate. + final int predictedLag; +} + +/// Inputs of the full-rate refinement: interleaved excerpts at the +/// comparison rate. +class AudioCompareFineInput { + const AudioCompareFineInput({ + required this.reference, + required this.referenceStart, + required this.candidates, + required this.channels, + required this.sampleRate, + required this.radius, + }); + + final Float32List reference; + final int referenceStart; + final List candidates; + final int channels; + final int sampleRate; + + /// How far the refinement may move from each predicted lag. + final int radius; +} + +/// Offset, gain and excerpt residual from the full-rate refinement. +class AudioCompareFineMatch { + const AudioCompareFineMatch({ + required this.lag, + required this.correlation, + required this.gain, + required this.excerptResidualDb, + }); + + /// Other-file frame minus reference frame of the same sound. + final int lag; + final double correlation; + final double gain; + + /// Residual of the aligned, gain-matched excerpt relative to the + /// reference. + final double excerptResidualDb; +} + +/// Refines the offset on the loudest second of the reference window, then +/// fits the gain on the whole window. +AudioCompareFineMatch audioCompareFineAlign(AudioCompareFineInput input) { + final channels = math.max(1, input.channels); + final referenceMono = audioCompareMixToMono(input.reference, channels); + final frames = referenceMono.length; + final length = math.min(frames, input.sampleRate); + final start = audioCompareLoudestStart( + referenceMono, + length, + math.max(1, length ~/ 10), + ); + final loudest = Float32List.sublistView(referenceMono, start, start + length); + var lag = input.candidates.isEmpty ? 0 : input.candidates.first.predictedLag; + var correlation = 0.0; + AudioCompareFineCandidate? chosen = input.candidates.isEmpty + ? null + : input.candidates.first; + for (final candidate in input.candidates) { + final otherMono = audioCompareMixToMono(candidate.other, channels); + // Other-file index of reference index `start` at the predicted lag. + final expected = + input.referenceStart + + start + + candidate.predictedLag - + candidate.otherStart; + final from = (expected - input.radius).clamp(0, otherMono.length); + final to = (expected + input.radius + length).clamp(from, otherMono.length); + if (length == 0 || to - from < length) continue; + final match = audioCompareFindLag( + loudest, + Float32List.sublistView(otherMono, from, to), + ); + if (match.correlation.abs() > correlation.abs()) { + lag = + candidate.otherStart + + from + + match.lag - + (input.referenceStart + start); + correlation = match.correlation; + chosen = candidate; + } + } + final reference = input.reference; + final aligned = Float32List(reference.length); + if (chosen != null) { + final other = chosen.other; + final otherFrames = other.length ~/ channels; + final otherFrame = lag + input.referenceStart - chosen.otherStart; + for (var frame = 0; frame < frames; frame++) { + final source = otherFrame + frame; + if (source < 0 || source >= otherFrames) continue; + for (var channel = 0; channel < channels; channel++) { + aligned[frame * channels + channel] = + other[source * channels + channel]; + } + } + } + final gain = audioCompareLeastSquaresGain(reference, aligned); + var residual = 0.0; + var energy = 0.0; + for (var i = 0; i < reference.length; i++) { + final difference = reference[i] - gain * aligned[i]; + residual += difference * difference; + energy += reference[i] * reference[i]; + } + return AudioCompareFineMatch( + lag: lag, + correlation: correlation, + gain: gain, + excerptResidualDb: energy <= 0 + ? audioCompareFloorDb + : audioComparePowerDb(residual / energy), + ); +} + +// --------------------------------------------------------------------------- +// Reference residual measurement + +/// A second-order high-pass (RBJ cookbook, Q = 1/sqrt(2)), the filter FFmpeg +/// runs for `highpass=poles=2`. +class _HighPass { + _HighPass(double cutoff, int sampleRate) { + final w0 = 2 * math.pi * math.min(cutoff, sampleRate * 0.49) / sampleRate; + final alpha = math.sin(w0) / (2 * math.sqrt1_2); + final cosW0 = math.cos(w0); + final a0 = 1 + alpha; + _b0 = (1 + cosW0) / 2 / a0; + _b1 = -(1 + cosW0) / a0; + _b2 = _b0; + _a1 = -2 * cosW0 / a0; + _a2 = (1 - alpha) / a0; + } + + late final double _b0, _b1, _b2, _a1, _a2; + double _x1 = 0, _x2 = 0, _y1 = 0, _y2 = 0; + + double process(double x) { + final y = _b0 * x + _b1 * _x1 + _b2 * _x2 - _a1 * _y1 - _a2 * _y2; + _x2 = _x1; + _x1 = x; + _y2 = _y1; + _y1 = y; + return y; + } +} + +/// Measures the residual windows directly from decoded samples, as the +/// FFmpeg pass does for whole songs: `reference - gain * other`, with the +/// other file shifted by [offsetFrames] and silent outside its length. +/// Used on excerpts and as the reference implementation in tests. +List audioCompareMeasureWindows({ + required Float32List reference, + required Float32List other, + required int channels, + required int sampleRate, + required int offsetFrames, + required double gain, + required int windowFrames, + double bandSplitHz = audioCompareBandSplitHz, +}) { + final frames = reference.length ~/ channels; + final otherFrames = other.length ~/ channels; + final residualFilters = [ + for (var c = 0; c < channels; c++) + [_HighPass(bandSplitHz, sampleRate), _HighPass(bandSplitHz, sampleRate)], + ]; + final referenceFilters = [ + for (var c = 0; c < channels; c++) + [_HighPass(bandSplitHz, sampleRate), _HighPass(bandSplitHz, sampleRate)], + ]; + final windows = []; + for (var start = 0; start < frames; start += windowFrames) { + final end = math.min(frames, start + windowFrames); + final residual = Float64List(channels); + final referencePower = Float64List(channels); + final residualHigh = Float64List(channels); + final referenceHigh = Float64List(channels); + for (var frame = start; frame < end; frame++) { + final source = frame + offsetFrames; + for (var c = 0; c < channels; c++) { + final r = reference[frame * channels + c]; + final o = source >= 0 && source < otherFrames + ? other[source * channels + c] + : 0.0; + final d = r - gain * o; + residual[c] += d * d; + referencePower[c] += r * r; + final dh = residualFilters[c][1].process( + residualFilters[c][0].process(d), + ); + final rh = referenceFilters[c][1].process( + referenceFilters[c][0].process(r), + ); + residualHigh[c] += dh * dh; + referenceHigh[c] += rh * rh; + } + } + final count = end - start; + for (var c = 0; c < channels; c++) { + residual[c] /= count; + referencePower[c] /= count; + residualHigh[c] /= count; + referenceHigh[c] /= count; + } + windows.add( + AudioCompareWindow( + startSeconds: start / sampleRate, + frames: count, + residual: residual, + reference: referencePower, + residualHigh: residualHigh, + referenceHigh: referenceHigh, + ), + ); + } + return windows; +} + +// --------------------------------------------------------------------------- +// Summary + +/// How large the whole-song difference is. +enum AudioCompareLevel { + identical, + nearlyIdentical, + mostlyIdentical, + moderate, + large, +} + +/// Where in the spectrum the differences sit. +enum AudioCompareBand { none, high, low, broadband } + +/// Why the null test is probably not meaningful. +enum AudioCompareNotice { weakAlignment, residualHigh, drift, durationMismatch } + +AudioCompareLevel audioCompareLevelFor(double overallDb) { + if (overallDb <= -110) return AudioCompareLevel.identical; + if (overallDb <= -70) return AudioCompareLevel.nearlyIdentical; + if (overallDb <= -40) return AudioCompareLevel.mostlyIdentical; + if (overallDb <= -20) return AudioCompareLevel.moderate; + return AudioCompareLevel.large; +} + +/// Compares the residual relative to the reference above and below +/// [audioCompareBandSplitHz] over the whole song. +AudioCompareBand audioCompareBandFor(List windows) { + var residual = 0.0; + var reference = 0.0; + var residualHigh = 0.0; + var referenceHigh = 0.0; + for (final window in windows) { + final n = window.frames.toDouble(); + for (var c = 0; c < window.residual.length; c++) { + residual += window.residual[c] * n; + reference += window.reference[c] * n; + residualHigh += window.residualHigh[c] * n; + referenceHigh += window.referenceHigh[c] * n; + } + } + if (reference <= 0 || residual <= 0) return AudioCompareBand.none; + if (audioComparePowerDb(residual / reference) <= -90) { + return AudioCompareBand.none; + } + // Without real high-frequency content there is nothing to compare. + if (referenceHigh / reference < 1e-4) return AudioCompareBand.none; + final residualLow = math.max(0.0, residual - residualHigh); + final referenceLow = math.max(reference * 1e-9, reference - referenceHigh); + final highRelative = residualHigh / referenceHigh; + final lowRelative = residualLow / referenceLow; + if (highRelative <= 0 && lowRelative <= 0) return AudioCompareBand.none; + if (lowRelative <= 0) return AudioCompareBand.high; + if (highRelative <= 0) return AudioCompareBand.low; + final difference = 10 * math.log(highRelative / lowRelative) / math.ln10; + if (difference >= audioCompareBandConcentrationDb) { + return AudioCompareBand.high; + } + if (difference <= -audioCompareBandConcentrationDb) { + return AudioCompareBand.low; + } + if (difference.abs() <= audioCompareBandEvenDb) { + return AudioCompareBand.broadband; + } + return AudioCompareBand.none; +} + +/// The loudest window of a series, ignoring silent reference windows. +class AudioComparePeak { + const AudioComparePeak(this.seconds, this.db); + + final double seconds; + final double db; +} + +AudioComparePeak? audioComparePeakFor( + List windows, [ + AudioCompareChannel channel = AudioCompareChannel.combined, +]) { + AudioComparePeak? peak; + for (final window in windows) { + if (window.isSilent(channel)) continue; + final db = window.relativeDb(channel); + if (db <= audioCompareFloorDb) continue; + if (peak == null || db > peak.db) { + peak = AudioComparePeak(window.startSeconds, db); + } + } + return peak; +} + +/// Median relative level of the non-silent windows. +double audioCompareMedianDb( + List windows, [ + AudioCompareChannel channel = AudioCompareChannel.combined, +]) { + final values = [ + for (final window in windows) + if (!window.isSilent(channel)) window.relativeDb(channel), + ]..sort(); + if (values.isEmpty) return audioCompareFloorDb; + return values[values.length ~/ 2]; +} + +/// The notices that make a null test not meaningful. +Set audioCompareNoticesFor({ + required double correlation, + required double overallDb, + required double medianDb, + int? driftCoarse, + required double referenceSeconds, + required double otherSeconds, + required double offsetSeconds, +}) { + return { + if (correlation.abs() < audioCompareMinCorrelation) + AudioCompareNotice.weakAlignment, + if (overallDb > audioCompareNotSameSourceDb || + medianDb > audioCompareNotSameSourceDb) + AudioCompareNotice.residualHigh, + if (driftCoarse != null && + driftCoarse.abs() > audioCompareDriftToleranceCoarse) + AudioCompareNotice.drift, + if (referenceSeconds > 0 && + otherSeconds > 0 && + ((otherSeconds - offsetSeconds) - referenceSeconds).abs() > + audioCompareDurationToleranceSeconds) + AudioCompareNotice.durationMismatch, + }; +} diff --git a/lib/services/audio_compare_engine.dart b/lib/services/audio_compare_engine.dart new file mode 100644 index 000000000..e7c8a9f90 --- /dev/null +++ b/lib/services/audio_compare_engine.dart @@ -0,0 +1,701 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:isolate'; +import 'dart:math' as math; +import 'dart:typed_data'; +import 'dart:ui' as ui; + +import 'package:ffmpeg_kit_flutter_new_audio/media_information.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:mornye/services/audio_analysis_ffmpeg.dart'; +import 'package:mornye/services/audio_analysis_jobs.dart'; +import 'package:mornye/services/audio_analysis_runner.dart'; +import 'package:mornye/services/audio_compare_dsp.dart'; +import 'package:mornye/services/audio_compare_ffmpeg.dart'; +import 'package:mornye/services/ffmpeg_models.dart'; +import 'package:mornye/services/ffmpeg_support.dart'; +import 'package:mornye/services/network_audio_analysis_input.dart'; +import 'package:mornye/services/platform_bridge.dart'; +import 'package:mornye/utils/logger.dart'; + +final _log = AppLogger('AudioCompare'); + +/// Shorter files cannot be lined up reliably. +const double audioCompareMinSeconds = 2; + +/// The step a running comparison is on, in order. +enum AudioCompareStage { preparing, aligning, measuring, spectrogram } + +enum AudioCompareFailure { unreadable, tooShort, decodeFailed } + +class AudioCompareException implements Exception { + const AudioCompareException(this.failure, {this.fileName, this.detail}); + + final AudioCompareFailure failure; + + /// The file that could not be read, for [AudioCompareFailure.unreadable]. + final String? fileName; + + /// FFmpeg's log or another technical reason, for the app log. + final String? detail; + + @override + String toString() => + 'AudioCompareException($failure${fileName == null ? '' : ', $fileName'})'; +} + +/// What the summary shows about one file. +class AudioCompareFileInfo { + const AudioCompareFileInfo({ + required this.name, + required this.codec, + required this.sampleRate, + required this.bitsPerSample, + required this.channels, + required this.durationSeconds, + }); + + factory AudioCompareFileInfo.fromMedia( + String name, + AudioAnalysisMediaInfo media, + ) => AudioCompareFileInfo( + name: name, + codec: media.codecName.toUpperCase(), + sampleRate: media.sampleRate, + bitsPerSample: media.bitsPerSample, + channels: media.channels, + durationSeconds: media.totalSamples > 0 && media.sampleRate > 0 + ? media.totalSamples / media.sampleRate + : media.duration, + ); + + final String name; + + /// Short codec name, such as FLAC or MP3. + final String codec; + final int sampleRate; + + /// Stored bit depth of lossless files; 0 for lossy codecs. + final int bitsPerSample; + final int channels; + final double durationSeconds; +} + +/// Everything the result view shows. Owns [spectrogram]. +class AudioCompareResult { + AudioCompareResult({ + required this.reference, + required this.other, + required this.sampleRate, + required this.channels, + required this.offsetFrames, + required this.gain, + required this.correlation, + required this.excerptResidualDb, + required this.windows, + this.driftMs, + this.spectrogram, + }) : overallDb = audioCompareOverallDb(windows), + medianDb = audioCompareMedianDb(windows), + peak = audioComparePeakFor(windows), + band = audioCompareBandFor(windows) { + level = audioCompareLevelFor(overallDb); + notices = audioCompareNoticesFor( + correlation: correlation, + overallDb: overallDb, + medianDb: medianDb, + driftCoarse: driftMs == null + ? null + : (driftMs! * audioCompareCoarseRate / 1000).round(), + referenceSeconds: reference.durationSeconds, + otherSeconds: other.durationSeconds, + offsetSeconds: offsetFrames / sampleRate, + ); + } + + final AudioCompareFileInfo reference; + final AudioCompareFileInfo other; + + /// The rate and channel count both files were compared at. + final int sampleRate; + final int channels; + + /// Other-file frame minus reference frame of the same sound: positive + /// when the other file starts later. + final int offsetFrames; + + /// Applied to the other file before subtracting. + final double gain; + + /// Normalized correlation peak of the full-rate alignment, 0 to 1. + final double correlation; + final double excerptResidualDb; + + /// Change of the offset between the start and the end of the song. + final double? driftMs; + final List windows; + final double overallDb; + final double medianDb; + final AudioComparePeak? peak; + final AudioCompareBand band; + late final AudioCompareLevel level; + late final Set notices; + ui.Image? spectrogram; + + double get offsetMs => offsetFrames * 1000 / sampleRate; + + /// The gain in dB; positive when the other file was quieter. + double get gainDb => + gain == 0 ? audioCompareFloorDb : 20 * math.log(gain.abs()) / math.ln10; + bool get polarityInverted => gain < 0; + + bool get meaningful => notices.isEmpty; + + /// One of the files was converted to [sampleRate]. + bool get resampled => + reference.sampleRate != sampleRate || other.sampleRate != sampleRate; + + /// The channel layouts differ, or a surround file was folded to stereo. + bool get channelsConverted => + reference.channels != other.channels || reference.channels > 2; + + double get durationSeconds => reference.durationSeconds; + + void dispose() { + spectrogram?.dispose(); + spectrogram = null; + } +} + +/// The file name shown for [path], also for content and network URIs. +String audioCompareDisplayName(String path) { + var name = path; + final uri = Uri.tryParse(path); + // One-letter schemes are Windows drive letters, not URIs. + if (uri != null && uri.scheme.length > 1) { + name = uri.pathSegments.isEmpty ? path : uri.pathSegments.last; + } + final last = name.split(RegExp(r'[/\\:]')).last; + return last.isEmpty ? name : last; +} + +typedef AudioCompareStart = + Future<({int id, Future completed})> Function( + List arguments, + ); + +typedef AudioCompareImageDecoder = + Future Function(Uint8List rgba, int width, int height); + +Future _decodeRgba(Uint8List rgba, int width, int height) { + final completer = Completer(); + ui.decodeImageFromPixels( + rgba, + width, + height, + ui.PixelFormat.rgba8888, + completer.complete, + ); + return completer.future; +} + +Future> _coarseAlignInIsolate( + List inputs, +) => Isolate.run( + () => [for (final input in inputs) audioCompareCoarseAlign(input)], +); + +Future _fineAlignInIsolate( + AudioCompareFineInput input, +) => Isolate.run(() => audioCompareFineAlign(input)); + +/// Runs the null test of one file against a reference. Each FFmpeg session +/// is owned and cancellable; every temporary file lives in one directory +/// that is removed when [compare] finishes, fails or is cancelled. +class AudioCompareEngine { + AudioCompareEngine({ + AudioCompareStart? start, + Future Function(int id)? cancel, + Future Function(String path)? probe, + Future Function()? temporaryDirectory, + Future Function(String uri)? copyContentUri, + NetworkAudioAnalysisInput? networkInput, + AudioCompareImageDecoder? decodeImage, + }) : _jobs = AudioAnalysisJobs( + start: start ?? FFmpegExecution.start, + cancel: cancel ?? FFmpegExecution.cancel, + ), + _probe = probe ?? FFmpegExecution.mediaInformation, + _temporaryDirectory = temporaryDirectory ?? getTemporaryDirectory, + _copyContentUri = copyContentUri ?? PlatformBridge.copyContentUriToTemp, + networkInput = networkInput ?? NetworkAudioAnalysisInput(), + _decodeImage = decodeImage ?? _decodeRgba; + + final AudioAnalysisJobs _jobs; + final Future Function(String path) _probe; + final Future Function() _temporaryDirectory; + final Future Function(String uri) _copyContentUri; + final NetworkAudioAnalysisInput networkInput; + final AudioCompareImageDecoder _decodeImage; + int _generation = 0; + bool _disposed = false; + + /// Stops the running comparison; it ends with [AudioAnalysisCancelled]. + void cancel() { + _generation++; + _jobs.invalidate(); + networkInput.cancel(); + } + + void dispose() { + _disposed = true; + _generation++; + _jobs.dispose(); + networkInput.cancel(); + } + + /// Compares [otherPath] (a local file) with [referencePath], which may + /// also be an Android content URI or a network source. [onStage] reports + /// each step. Throws [AudioCompareException], + /// [UnsupportedAudioAnalysisCodecException] or [AudioAnalysisCancelled]. + Future compare({ + required String referencePath, + required String otherPath, + void Function(AudioCompareStage stage)? onStage, + bool includeSpectrogram = true, + }) async { + final generation = _generation; + bool isCancelled() => _disposed || generation != _generation; + void check() { + if (isCancelled()) throw const AudioAnalysisCancelled(); + } + + check(); + final referenceName = audioCompareDisplayName(referencePath); + final otherName = audioCompareDisplayName(otherPath); + Future run(String path) => _compareFiles( + referencePath: path, + otherPath: otherPath, + referenceName: referenceName, + otherName: otherName, + check: check, + onStage: onStage, + includeSpectrogram: includeSpectrogram, + ); + + if (NetworkAudioAnalysisInput.isNetwork(referencePath)) { + onStage?.call(AudioCompareStage.preparing); + AudioCompareResult? produced; + try { + return await networkInput.withFile( + referencePath, + (path) async => produced = await run(path), + ); + } catch (_) { + produced?.dispose(); + rethrow; + } + } + if (referencePath.startsWith('content://')) { + onStage?.call(AudioCompareStage.preparing); + final copy = await _copyContentUri(referencePath); + if (copy == null) { + throw AudioCompareException( + AudioCompareFailure.unreadable, + fileName: referenceName, + ); + } + try { + check(); + return await run(copy); + } finally { + try { + await File(copy).delete(); + } catch (_) {} + } + } + return run(referencePath); + } + + Future _probeFile(String path, String name) async { + MediaInformation? info; + try { + info = await _probe(path); + } on FFmpegUnavailableException { + rethrow; + } catch (e) { + _log.w('Probe failed for $name: $e'); + } + if (info == null) { + throw AudioCompareException( + AudioCompareFailure.unreadable, + fileName: name, + ); + } + final AudioAnalysisMediaInfo media; + try { + media = AudioAnalysisRunner.parseMediaInfo(info); + } catch (e) { + throw AudioCompareException( + AudioCompareFailure.unreadable, + fileName: name, + detail: '$e', + ); + } + final unsupported = unsupportedAudioAnalysisCodecLabel(media.codecName); + if (unsupported != null) { + throw UnsupportedAudioAnalysisCodecException(unsupported); + } + return AudioCompareFileInfo.fromMedia(name, media); + } + + Future _runFFmpeg(List arguments) async { + final session = await _jobs.run(arguments); + if (!session.success) { + final logs = await session.logs(); + final tail = logs.length > 2000 + ? logs.substring(logs.length - 2000) + : logs; + _log.w('FFmpeg failed (${session.returnCode}): $tail'); + throw AudioCompareException( + AudioCompareFailure.decodeFailed, + detail: tail, + ); + } + } + + static Future _readFloats(String path) async { + final file = File(path); + if (!await file.exists()) return Float32List(0); + final bytes = await file.readAsBytes(); + final count = bytes.length ~/ 4; + if (Endian.host == Endian.little && bytes.offsetInBytes % 4 == 0) { + return bytes.buffer.asFloat32List(bytes.offsetInBytes, count); + } + final data = ByteData.sublistView(bytes); + return Float32List.fromList([ + for (var i = 0; i < count; i++) data.getFloat32(i * 4, Endian.little), + ]); + } + + Future _compareFiles({ + required String referencePath, + required String otherPath, + required String referenceName, + required String otherName, + required void Function() check, + required void Function(AudioCompareStage stage)? onStage, + required bool includeSpectrogram, + }) async { + onStage?.call(AudioCompareStage.preparing); + final infos = await Future.wait([ + _probeFile(referencePath, referenceName), + _probeFile(otherPath, otherName), + ]); + check(); + final reference = infos[0]; + final other = infos[1]; + if (reference.durationSeconds < audioCompareMinSeconds || + other.durationSeconds < audioCompareMinSeconds) { + throw const AudioCompareException(AudioCompareFailure.tooShort); + } + final rate = reference.sampleRate > 0 + ? math.min(reference.sampleRate, audioCompareMaxSampleRate) + : 48000; + final channels = reference.channels == 1 ? 1 : 2; + + final root = await _temporaryDirectory(); + final directory = await root.createTemp('audio-compare-'); + try { + check(); + onStage?.call(AudioCompareStage.aligning); + final alignment = await _align( + directory: directory, + referencePath: referencePath, + otherPath: otherPath, + durationSeconds: reference.durationSeconds, + rate: rate, + channels: channels, + check: check, + ); + + onStage?.call(AudioCompareStage.measuring); + final totalFrames = (reference.durationSeconds * rate).round(); + final windowFrames = audioCompareWindowFrames(totalFrames, rate); + final metadataPath = + '${directory.path}${Platform.pathSeparator}levels.txt'; + await _runFFmpeg( + buildAudioCompareStatsArguments( + referencePath: referencePath, + otherPath: otherPath, + sampleRate: rate, + channels: channels, + offsetFrames: alignment.fine.lag, + gain: alignment.fine.gain, + windowFrames: windowFrames, + metadataPath: metadataPath, + ), + ); + check(); + final metadataFile = File(metadataPath); + final metadata = await metadataFile.exists() + ? await metadataFile.readAsString() + : ''; + final windows = audioCompareWindowsFromMetadata( + parseAudioCompareMetadata(metadata, channels: channels * 4), + channels: channels, + sampleRate: rate, + windowFrames: windowFrames, + totalFrames: totalFrames, + ); + if (windows.isEmpty) { + throw const AudioCompareException( + AudioCompareFailure.decodeFailed, + detail: 'The level pass wrote no windows', + ); + } + + final result = AudioCompareResult( + reference: reference, + other: other, + sampleRate: rate, + channels: channels, + offsetFrames: alignment.fine.lag, + gain: alignment.fine.gain, + correlation: alignment.fine.correlation.abs(), + excerptResidualDb: alignment.fine.excerptResidualDb, + windows: windows, + driftMs: alignment.driftMs, + ); + if (!includeSpectrogram) return result; + + onStage?.call(AudioCompareStage.spectrogram); + try { + result.spectrogram = await _residualSpectrogram( + directory: directory, + referencePath: referencePath, + otherPath: otherPath, + rate: rate, + channels: channels, + offsetFrames: alignment.fine.lag, + gain: alignment.fine.gain, + durationSeconds: reference.durationSeconds, + ); + check(); + } on AudioAnalysisCancelled { + result.dispose(); + rethrow; + } catch (e) { + // The timeline and summary stand on their own without the picture. + _log.w('Residual spectrogram failed: $e'); + } + return result; + } finally { + try { + await directory.delete(recursive: true); + } catch (_) {} + } + } + + Future< + ({ + AudioCompareCoarseMatch coarse, + AudioCompareFineMatch fine, + double? driftMs, + }) + > + _align({ + required Directory directory, + required String referencePath, + required String otherPath, + required double durationSeconds, + required int rate, + required int channels, + required void Function() check, + }) async { + const coarseRate = audioCompareCoarseRate; + final separator = Platform.pathSeparator; + final scan = math.min(audioCompareScanSeconds, durationSeconds); + double scanStart(double fraction) => math.max( + math.min(audioCompareSearchSeconds, durationSeconds - scan), + math.min(durationSeconds * fraction, durationSeconds - scan), + ); + final starts = [scanStart(0.25)]; + // A second excerpt near the end shows whether the offset drifts. + final lateStart = scanStart(0.7); + if (lateStart >= starts.first + scan) starts.add(lateStart); + + final excerpts = []; + final inputs = <({int referenceStart, int otherStart})>[]; + for (final (index, start) in starts.indexed) { + final referenceStart = (start * coarseRate).round(); + final referenceEnd = ((start + scan) * coarseRate).round(); + final otherStart = math.max( + 0, + ((start - audioCompareSearchSeconds) * coarseRate).round(), + ); + final otherEnd = ((start + scan + audioCompareSearchSeconds) * coarseRate) + .round(); + excerpts + ..add( + AudioCompareExcerpt( + input: 0, + startSample: referenceStart, + endSample: referenceEnd, + outputPath: '${directory.path}${separator}coarse_ref_$index.f32', + ), + ) + ..add( + AudioCompareExcerpt( + input: 1, + startSample: otherStart, + endSample: otherEnd, + outputPath: '${directory.path}${separator}coarse_other_$index.f32', + ), + ); + inputs.add((referenceStart: referenceStart, otherStart: otherStart)); + } + await _runFFmpeg( + buildAudioCompareExcerptArguments( + referencePath: referencePath, + otherPath: otherPath, + sampleRate: rate, + channels: channels, + decimatedRate: coarseRate, + excerpts: excerpts, + ), + ); + check(); + final coarseInputs = []; + for (final (index, input) in inputs.indexed) { + coarseInputs.add( + AudioCompareCoarseInput( + reference: await _readFloats(excerpts[index * 2].outputPath), + referenceStart: input.referenceStart, + other: await _readFloats(excerpts[index * 2 + 1].outputPath), + otherStart: input.otherStart, + ), + ); + } + check(); + final coarse = await _coarseAlignInIsolate(coarseInputs); + check(); + final first = coarse.first; + + // Each strong coarse peak gets a short full-rate excerpt of the other + // file around it; the best full-rate match wins. + final ratio = rate / coarseRate; + final radius = 4 * ratio.ceil() + 64; + final fineFrames = (audioCompareFineSeconds * rate).round(); + final totalFrames = (durationSeconds * rate).round(); + final referenceStart = + ((first.fineCenter * ratio).round() - fineFrames ~/ 2) + .clamp(0, math.max(0, totalFrames - fineFrames)) + .toInt(); + final predictions = [ + for (final candidate in first.candidates) (candidate.lag * ratio).round(), + ]; + if (predictions.isEmpty) predictions.add(0); + final fineExcerpts = [ + AudioCompareExcerpt( + input: 0, + startSample: referenceStart, + endSample: referenceStart + fineFrames, + outputPath: '${directory.path}${separator}fine_ref.f32', + ), + for (final (index, predicted) in predictions.indexed) + AudioCompareExcerpt( + input: 1, + startSample: math.max(0, referenceStart + predicted - radius), + endSample: math.max( + 1, + referenceStart + fineFrames + predicted + radius, + ), + outputPath: '${directory.path}${separator}fine_other_$index.f32', + ), + ]; + await _runFFmpeg( + buildAudioCompareExcerptArguments( + referencePath: referencePath, + otherPath: otherPath, + sampleRate: rate, + channels: channels, + excerpts: fineExcerpts, + ), + ); + check(); + final candidates = []; + for (final (index, predicted) in predictions.indexed) { + final excerpt = fineExcerpts[index + 1]; + candidates.add( + AudioCompareFineCandidate( + other: await _readFloats(excerpt.outputPath), + otherStart: excerpt.startSample, + predictedLag: predicted, + ), + ); + } + final fine = await _fineAlignInIsolate( + AudioCompareFineInput( + reference: await _readFloats(fineExcerpts.first.outputPath), + referenceStart: referenceStart, + candidates: candidates, + channels: channels, + sampleRate: rate, + radius: radius, + ), + ); + check(); + + double? driftMs; + if (coarse.length > 1 && + fine.correlation.abs() >= audioCompareMinCorrelation && + coarse[1].correlation.abs() >= audioCompareMinCorrelation) { + final startLag = fine.lag / ratio; + final endLag = coarse[1].lagNearest(startLag.round()); + driftMs = (endLag - startLag) * 1000 / coarseRate; + } + return (coarse: first, fine: fine, driftMs: driftMs); + } + + Future _residualSpectrogram({ + required Directory directory, + required String referencePath, + required String otherPath, + required int rate, + required int channels, + required int offsetFrames, + required double gain, + required double durationSeconds, + }) async { + final outputPath = + '${directory.path}${Platform.pathSeparator}residual_spectrum.rgba'; + await _runFFmpeg( + buildAudioCompareSpectrogramArguments( + referencePath: referencePath, + otherPath: otherPath, + sampleRate: rate, + channels: channels, + offsetFrames: offsetFrames, + gain: gain, + durationSeconds: durationSeconds, + outputPath: outputPath, + ), + ); + const expected = audioSpectrogramWidth * audioSpectrogramHeight * 4; + final bytes = await File(outputPath).readAsBytes(); + if (bytes.length < expected) { + throw AudioCompareException( + AudioCompareFailure.decodeFailed, + detail: 'Incomplete residual spectrogram (${bytes.length}/$expected)', + ); + } + return _decodeImage( + bytes.length == expected + ? bytes + : Uint8List.sublistView(bytes, 0, expected), + audioSpectrogramWidth, + audioSpectrogramHeight, + ); + } +} diff --git a/lib/services/audio_compare_ffmpeg.dart b/lib/services/audio_compare_ffmpeg.dart new file mode 100644 index 000000000..1437a46a2 --- /dev/null +++ b/lib/services/audio_compare_ffmpeg.dart @@ -0,0 +1,293 @@ +// FFmpeg commands of the null test. Input 0 is the reference, input 1 the +// other file. Every pass decodes both from their first sample through the +// same chain, so a sample index means the same thing in each pass. +import 'dart:typed_data'; + +import 'package:mornye/services/audio_analysis_ffmpeg.dart'; +import 'package:mornye/services/audio_compare_dsp.dart'; + +/// Decodes one input to float PCM at [sampleRate] with [channels] (1 or 2). +String audioCompareDecodeChain({ + required int sampleRate, + required int channels, +}) => + 'aresample=$sampleRate,' + 'aformat=sample_fmts=flt:channel_layouts=${channels == 1 ? 'mono' : 'stereo'}'; + +/// One excerpt to write as raw little-endian float PCM. +class AudioCompareExcerpt { + const AudioCompareExcerpt({ + required this.input, + required this.startSample, + required this.endSample, + required this.outputPath, + }); + + /// 0 for the reference, 1 for the other file. + final int input; + + /// First sample kept and first sample dropped, at the excerpt's rate. + final int startSample; + final int endSample; + final String outputPath; +} + +/// Writes [excerpts] in one decode of both files. With [decimatedRate] the +/// excerpts are mono at that rate (the wide search); otherwise they keep +/// [channels] at [sampleRate]. +List buildAudioCompareExcerptArguments({ + required String referencePath, + required String otherPath, + required int sampleRate, + required int channels, + int? decimatedRate, + required List excerpts, +}) { + final chain = decimatedRate == null + ? audioCompareDecodeChain(sampleRate: sampleRate, channels: channels) + : '${audioCompareDecodeChain(sampleRate: sampleRate, channels: 1)},' + 'aresample=$decimatedRate'; + final parts = []; + for (final input in const [0, 1]) { + final indexes = [ + for (var i = 0; i < excerpts.length; i++) + if (excerpts[i].input == input) i, + ]; + if (indexes.isEmpty) continue; + String trim(int index) { + final excerpt = excerpts[index]; + return 'atrim=start_sample=${excerpt.startSample}:' + 'end_sample=${excerpt.endSample}[cmp_x$index]'; + } + + if (indexes.length == 1) { + parts.add('[$input:a:0]$chain,${trim(indexes.single)}'); + continue; + } + final splits = [for (final index in indexes) '[cmp_s$index]'].join(); + parts.add('[$input:a:0]$chain,asplit=${indexes.length}$splits'); + for (final index in indexes) { + parts.add('[cmp_s$index]${trim(index)}'); + } + } + return [ + '-hide_banner', + '-nostats', + '-y', + '-i', + referencePath, + '-i', + otherPath, + '-filter_complex', + parts.join(';'), + for (var i = 0; i < excerpts.length; i++) ...[ + '-map', + '[cmp_x$i]', + '-f', + 'f32le', + '-c:a', + 'pcm_f32le', + excerpts[i].outputPath, + ], + ]; +} + +String _formatGain(double gain) => gain.toStringAsFixed(9); + +/// `c=c-gain*c`, written so a negative gain stays parseable. +String _subtract(int out, int a, int b, double gain) => gain >= 0 + ? 'c$out=c$a-${_formatGain(gain)}*c$b' + : 'c$out=c$a+${_formatGain(-gain)}*c$b'; + +/// Lines both decoded files up and merges them into one stream whose first +/// [channels] channels are the reference and the next [channels] the other +/// file, shifted by [offsetFrames] (other-file frame minus reference frame) +/// and padded with silence after its end. The graph ends unlabeled, so the +/// caller appends filters to it. +String buildAudioCompareMergeGraph({ + required int sampleRate, + required int channels, + required int offsetFrames, +}) { + final chain = audioCompareDecodeChain( + sampleRate: sampleRate, + channels: channels, + ); + final offset = offsetFrames > 0 + ? 'atrim=start_sample=$offsetFrames,' + : offsetFrames < 0 + ? 'adelay=delays=${-offsetFrames}S:all=1,' + : ''; + return '[0:a:0]$chain,asetpts=N/SR/TB[cmp_ref];' + '[1:a:0]$chain,${offset}asetpts=N/SR/TB,apad[cmp_oth];' + '[cmp_ref][cmp_oth]amerge=inputs=2'; +} + +/// The residual `reference - gain * other`, appended to the merge graph. +String _residualPan(int channels, double gain) => channels == 1 + ? 'pan=mono|${_subtract(0, 0, 1, gain)}' + : 'pan=stereo|${_subtract(0, 0, 2, gain)}|${_subtract(1, 1, 3, gain)}'; + +/// Per-window levels of the whole song. The metadata file lists, for each +/// window of [windowFrames] frames, the RMS level of the residual channels, +/// then the reference channels, then both again above [bandSplitHz]. +List buildAudioCompareStatsArguments({ + required String referencePath, + required String otherPath, + required int sampleRate, + required int channels, + required int offsetFrames, + required double gain, + required int windowFrames, + required String metadataPath, + double bandSplitHz = audioCompareBandSplitHz, +}) { + final merge = buildAudioCompareMergeGraph( + sampleRate: sampleRate, + channels: channels, + offsetFrames: offsetFrames, + ); + final split = bandSplitHz.clamp(1.0, sampleRate * 0.45).round(); + final pan = channels == 1 + ? 'pan=2c|${_subtract(0, 0, 1, gain)}|c1=c0' + : 'pan=4c|${_subtract(0, 0, 2, gain)}|${_subtract(1, 1, 3, gain)}|' + 'c2=c0|c3=c1'; + final graph = + '$merge,$pan,asplit=2[cmp_full][cmp_hp];' + '[cmp_hp]highpass=f=$split:poles=2,highpass=f=$split:poles=2[cmp_high];' + '[cmp_full][cmp_high]amerge=inputs=2,' + 'asetnsamples=n=$windowFrames:p=0,' + 'astats=metadata=1:reset=1:measure_perchannel=RMS_level:' + 'measure_overall=none,' + "ametadata=print:file='${escapeFFmpegFilterPath(metadataPath)}'" + '[cmp_stats]'; + return [ + '-hide_banner', + '-nostats', + '-i', + referencePath, + '-i', + otherPath, + '-filter_complex', + graph, + '-map', + '[cmp_stats]', + '-f', + 'null', + '-', + ]; +} + +/// The existing spectrogram picture, drawn from the residual instead of a +/// file. +List buildAudioCompareSpectrogramArguments({ + required String referencePath, + required String otherPath, + required int sampleRate, + required int channels, + required int offsetFrames, + required double gain, + required double durationSeconds, + required String outputPath, +}) { + final merge = buildAudioCompareMergeGraph( + sampleRate: sampleRate, + channels: channels, + offsetFrames: offsetFrames, + ); + final spectrogram = buildAudioSpectrogramFilter( + durationSeconds: durationSeconds, + sampleRate: sampleRate, + channels: channels, + inputLabel: '[cmp_residual]', + ); + return [ + '-hide_banner', + '-nostats', + '-y', + '-i', + referencePath, + '-i', + otherPath, + '-filter_complex', + '$merge,${_residualPan(channels, gain)}[cmp_residual];$spectrogram', + '-map', + '[spectrum]', + '-frames:v', + '1', + '-f', + 'rawvideo', + '-pix_fmt', + 'rgba', + outputPath, + ]; +} + +final RegExp _frameLine = RegExp(r'^frame:\s*(\d+)', multiLine: false); +final RegExp _levelLine = RegExp( + r'^lavfi\.astats\.(\d+)\.RMS_level=(.+)$', + multiLine: false, +); + +double _parseLevel(String value) { + final text = value.trim().toLowerCase(); + if (text == '-inf') return double.negativeInfinity; + if (text == 'inf' || text == '+inf') return double.infinity; + return double.tryParse(text) ?? double.nan; +} + +/// RMS levels in dBFS per window from the stats pass's metadata file, each +/// list holding [channels] values in channel order. +List parseAudioCompareMetadata( + String metadata, { + required int channels, +}) { + final windows = []; + Float64List? current; + for (final line in metadata.split('\n')) { + final trimmed = line.trim(); + if (_frameLine.hasMatch(trimmed)) { + current = Float64List(channels)..fillRange(0, channels, double.nan); + windows.add(current); + continue; + } + final match = _levelLine.firstMatch(trimmed); + if (match == null || current == null) continue; + final channel = int.parse(match.group(1)!) - 1; + if (channel < 0 || channel >= channels) continue; + current[channel] = _parseLevel(match.group(2)!); + } + return windows; +} + +/// Timeline windows from the stats pass, for [channels] audio channels (the +/// metadata holds four times as many). [totalFrames], when known, sizes the +/// final partial window. +List audioCompareWindowsFromMetadata( + List levels, { + required int channels, + required int sampleRate, + required int windowFrames, + int? totalFrames, +}) { + Float64List powers(Float64List values, int from) => Float64List.fromList([ + for (var c = 0; c < channels; c++) + audioCompareDbToPower( + from + c < values.length ? values[from + c] : double.nan, + ), + ]); + + return [ + for (var i = 0; i < levels.length; i++) + AudioCompareWindow( + startSeconds: i * windowFrames / sampleRate, + frames: i == levels.length - 1 && totalFrames != null + ? (totalFrames - i * windowFrames).clamp(1, windowFrames).toInt() + : windowFrames, + residual: powers(levels[i], 0), + reference: powers(levels[i], channels), + residualHigh: powers(levels[i], channels * 2), + referenceHigh: powers(levels[i], channels * 3), + ), + ]; +} diff --git a/test/audio_compare_dsp_test.dart b/test/audio_compare_dsp_test.dart new file mode 100644 index 000000000..b86fd050a --- /dev/null +++ b/test/audio_compare_dsp_test.dart @@ -0,0 +1,438 @@ +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:mornye/services/audio_compare_dsp.dart'; + +/// A deterministic music-like signal: drifting tones, an envelope and a +/// little seeded noise, so no lag repeats exactly. +Float32List _music(int frames, int rate, {int channels = 1, int seed = 7}) { + final random = math.Random(seed); + final out = Float32List(frames * channels); + for (var n = 0; n < frames; n++) { + final t = n / rate; + final envelope = 0.6 + 0.4 * math.sin(2 * math.pi * 0.7 * t); + for (var c = 0; c < channels; c++) { + final tone = + 0.3 * math.sin(2 * math.pi * (220 + 30 * c) * t + math.sin(t * 3)) + + 0.15 * math.sin(2 * math.pi * 659.3 * t) + + 0.08 * math.sin(2 * math.pi * 1530 * t * (1 + 0.01 * t)); + out[n * channels + c] = + envelope * tone + 0.05 * (random.nextDouble() * 2 - 1); + } + } + return out; +} + +/// The other file of a test: [source] starting [lag] frames later (silence +/// before it) or, for a negative lag, with its first frames cut. +Float32List _shift(Float32List source, int lag, {int channels = 1}) { + final frames = source.length ~/ channels; + final out = Float32List((frames + lag) * channels); + for (var n = 0; n < frames; n++) { + final target = n + lag; + if (target < 0 || target >= frames + lag) continue; + for (var c = 0; c < channels; c++) { + out[target * channels + c] = source[n * channels + c]; + } + } + return out; +} + +/// Crude decimation by an integer factor, standing in for FFmpeg's +/// resampler in the wide search. +Float32List _decimate(Float32List mono, int factor) { + final out = Float32List(mono.length ~/ factor); + for (var i = 0; i < out.length; i++) { + var sum = 0.0; + for (var k = 0; k < factor; k++) { + sum += mono[i * factor + k]; + } + out[i] = sum / factor; + } + return out; +} + +/// Zero-phase brick-wall low-pass through the FFT; the length must be a +/// power of two. +Float32List _lowPass(Float32List mono, int rate, double cutoffHz) { + final re = Float64List.fromList(mono); + final im = Float64List(mono.length); + audioCompareFft(re, im); + final n = mono.length; + for (var k = 0; k < n; k++) { + final bin = k <= n ~/ 2 ? k : n - k; + if (bin * rate / n > cutoffHz) { + re[k] = 0; + im[k] = 0; + } + } + audioCompareFft(re, im, inverse: true); + return Float32List.fromList(re); +} + +/// Runs the same two-step alignment as the engine on in-memory signals at +/// 48 kHz (the coarse step decimates by 12 to 4 kHz). +({int lag, double correlation, double gain}) _align( + Float32List reference, + Float32List other, { + int channels = 1, +}) { + const rate = 48000; + const factor = rate ~/ audioCompareCoarseRate; + final referenceMono = audioCompareMixToMono(reference, channels); + final otherMono = audioCompareMixToMono(other, channels); + final coarseReference = _decimate(referenceMono, factor); + final coarseOther = _decimate(otherMono, factor); + // Like the engine, scan the reference after the search margin so an other + // file that starts earlier still holds the matched stretch. + final scanStart = (audioCompareSearchSeconds * audioCompareCoarseRate) + .round(); + final coarse = audioCompareCoarseAlign( + AudioCompareCoarseInput( + reference: Float32List.sublistView(coarseReference, scanStart), + referenceStart: scanStart, + other: coarseOther, + otherStart: 0, + ), + ); + const radius = 4 * factor + 64; + const fineFrames = 3 * rate; + final referenceStart = (coarse.fineCenter * factor - fineFrames ~/ 2) + .clamp(0, referenceMono.length - fineFrames) + .toInt(); + final fine = audioCompareFineAlign( + AudioCompareFineInput( + reference: Float32List.sublistView( + reference, + referenceStart * channels, + (referenceStart + fineFrames) * channels, + ), + referenceStart: referenceStart, + candidates: [ + for (final candidate in coarse.candidates) + () { + final predicted = candidate.lag * factor; + final otherStart = math.max(0, referenceStart + predicted - radius); + final otherEnd = math.min( + otherMono.length, + referenceStart + fineFrames + predicted + radius, + ); + return AudioCompareFineCandidate( + other: Float32List.sublistView( + other, + otherStart * channels, + math.max(otherStart, otherEnd) * channels, + ), + otherStart: otherStart, + predictedLag: predicted, + ); + }(), + ], + channels: channels, + sampleRate: rate, + radius: radius, + ), + ); + return (lag: fine.lag, correlation: fine.correlation, gain: fine.gain); +} + +void main() { + group('alignment', () { + final reference = _music(48000 * 12, 48000); + + // MP3 decoders add about 576 to 1105 samples, AAC priming is 2112; the + // larger values cover files that start up to 2.5 s apart. + for (final lag in [ + 0, + 1, + 576, + 1105, + 2112, + 12345, + 120000, + -1, + -576, + -1105, + -2112, + -72000, + ]) { + test('recovers an offset of $lag samples exactly', () { + final result = _align(reference, _shift(reference, lag)); + expect(result.lag, lag); + expect(result.correlation, greaterThan(0.999)); + expect(result.gain, closeTo(1, 1e-6)); + }); + } + + test('recovers the offset and gain of a stereo copy', () { + final stereo = _music(48000 * 10, 48000, channels: 2); + final quieter = _shift(stereo, 1105, channels: 2); + for (var i = 0; i < quieter.length; i++) { + quieter[i] *= 0.7; + } + final result = _align(stereo, quieter, channels: 2); + expect(result.lag, 1105); + // reference = gain * other, so the gain undoes the 0.7. + expect(result.gain, closeTo(1 / 0.7, 1e-4)); + expect(20 * math.log(result.gain) / math.ln10, closeTo(3.098, 0.001)); + }); + + test('reports inverted polarity through the gain sign', () { + final inverted = Float32List.fromList([for (final v in reference) -v]); + final result = _align(reference, inverted); + expect(result.lag, 0); + expect(result.correlation, lessThan(-0.999)); + expect(result.gain, closeTo(-1, 1e-6)); + }); + + test('the FFT and direct correlations agree', () { + final a = Float32List.sublistView(reference, 1000, 1400); + final b = Float32List.sublistView(reference, 900, 2100); + final direct = audioCompareCrossCorrelation(a, b, useFft: false); + final fft = audioCompareCrossCorrelation(a, b, useFft: true); + expect(fft.length, direct.length); + for (var k = 0; k < direct.length; k++) { + expect(fft[k], closeTo(direct[k], 1e-6)); + } + expect(audioCompareFindLag(a, b, useFft: true).lag, 100); + expect(audioCompareFindLag(a, b, useFft: false).lag, 100); + }); + + test('unrelated signals give a weak correlation peak', () { + final unrelated = _music(48000 * 12, 48000, seed: 99); + final noise = Float32List.fromList([ + for (final (i, v) in unrelated.indexed) + v * math.sin(i * 0.37) + math.sin(i * 0.0021), + ]); + final result = _align(reference, noise); + expect(result.correlation.abs(), lessThan(audioCompareMinCorrelation)); + }); + }); + + group('residual', () { + const rate = 48000; + + test('identical files cancel to the floor', () { + final stereo = _music(rate * 4, rate, channels: 2); + final windows = audioCompareMeasureWindows( + reference: stereo, + other: Float32List.fromList(stereo), + channels: 2, + sampleRate: rate, + offsetFrames: 0, + gain: 1, + windowFrames: audioCompareWindowFrames(rate * 4, rate), + ); + expect( + audioCompareSeries(windows, AudioCompareChannel.combined), + everyElement(audioCompareFloorDb), + ); + expect(audioCompareOverallDb(windows), audioCompareFloorDb); + expect( + audioCompareLevelFor(audioCompareOverallDb(windows)), + AudioCompareLevel.identical, + ); + expect(audioCompareBandFor(windows), AudioCompareBand.none); + expect(audioComparePeakFor(windows), isNull); + }); + + test('a shifted copy cancels once the offset is applied', () { + final mono = _music(rate * 4, rate); + final windows = audioCompareMeasureWindows( + reference: mono, + other: _shift(mono, 2112), + channels: 1, + sampleRate: rate, + offsetFrames: 2112, + gain: 1, + windowFrames: 4800, + ); + expect(audioCompareOverallDb(windows), audioCompareFloorDb); + }); + + test('a low-passed copy differs only where there are highs', () { + const frames = 1 << 18; // about 5.5 s + final random = math.Random(3); + final mono = Float32List(frames); + for (var n = 0; n < frames; n++) { + final t = n / rate; + var value = + 0.3 * math.sin(2 * math.pi * 220 * t) + + 0.2 * math.sin(2 * math.pi * 523.25 * t) + + 0.1 * math.sin(2 * math.pi * 1760 * t); + // Highs only in the middle third, like a dense chorus. + if (n >= frames ~/ 3 && n < frames * 2 ~/ 3) { + value += + 0.1 * math.sin(2 * math.pi * 12000 * t) + + 0.05 * (random.nextDouble() * 2 - 1); + } + mono[n] = value; + } + final lowPassed = _lowPass(mono, rate, 8000); + final gain = audioCompareLeastSquaresGain(mono, lowPassed); + final windows = audioCompareMeasureWindows( + reference: mono, + other: lowPassed, + channels: 1, + sampleRate: rate, + offsetFrames: 0, + gain: gain, + windowFrames: 4096, + ); + final series = audioCompareSeries(windows, AudioCompareChannel.combined); + double median(Iterable values) { + final sorted = values.toList()..sort(); + return sorted[sorted.length ~/ 2]; + } + + final third = series.length ~/ 3; + final middle = median(series.sublist(third + 2, 2 * third - 2)); + final outside = median([ + ...series.sublist(2, third - 2), + ...series.sublist(2 * third + 2, series.length - 2), + ]); + expect(middle, greaterThan(-25)); + expect(middle - outside, greaterThan(40)); + expect(audioCompareBandFor(windows), AudioCompareBand.high); + final peak = audioComparePeakFor(windows)!; + expect(peak.seconds, greaterThan(frames / 3 / rate - 0.1)); + expect(peak.seconds, lessThan(frames * 2 / 3 / rate)); + }); + + test('unrelated signals stay near 0 dB and raise the notice', () { + final a = _music(rate * 4, rate); + final b = Float32List.fromList([ + for (final (i, v) in _music(rate * 4, rate, seed: 42).indexed) + v * math.cos(i * 0.011), + ]); + final gain = audioCompareLeastSquaresGain(a, b); + final windows = audioCompareMeasureWindows( + reference: a, + other: b, + channels: 1, + sampleRate: rate, + offsetFrames: 0, + gain: gain, + windowFrames: 4800, + ); + final overall = audioCompareOverallDb(windows); + expect(overall, greaterThan(audioCompareNotSameSourceDb)); + final notices = audioCompareNoticesFor( + correlation: 0.05, + overallDb: overall, + medianDb: audioCompareMedianDb(windows), + referenceSeconds: 4, + otherSeconds: 4, + offsetSeconds: 0, + ); + expect( + notices, + containsAll([ + AudioCompareNotice.weakAlignment, + AudioCompareNotice.residualHigh, + ]), + ); + }); + }); + + group('windows and levels', () { + test('a song becomes a few hundred to 2000 points', () { + for (final seconds in [3, 30, 200, 600, 3600]) { + for (final rate in [44100, 48000, 96000]) { + final total = seconds * rate; + final window = audioCompareWindowFrames(total, rate); + final points = (total / window).ceil(); + expect(points, inInclusiveRange(300, 2000), reason: '$seconds s'); + } + } + expect(audioCompareWindowFrames(0, 48000), 240); + }); + + test('dB math', () { + expect(audioCompareDbToPower(-20), closeTo(0.01, 1e-12)); + expect(audioCompareDbToPower(double.negativeInfinity), 0); + expect(audioCompareDbToPower(double.nan), 0); + expect(audioComparePowerDb(0.01), closeTo(-20, 1e-9)); + expect(audioComparePowerDb(0), audioCompareFloorDb); + expect(audioCompareRelativeDb(0.001, 0.1), closeTo(-20, 1e-9)); + expect(audioCompareRelativeDb(0, 0.1), audioCompareFloorDb); + expect(audioCompareRelativeDb(1e-20, 0.1), audioCompareFloorDb); + expect(audioCompareRelativeDb(1000, 0.1), audioCompareCeilingDb); + // Silence in both stays on the floor instead of magnifying noise. + expect(audioCompareRelativeDb(1e-10, 1e-12), audioCompareFloorDb); + // A residual where the reference is silent is measured against the + // silence level. + expect(audioCompareRelativeDb(1e-6, 0), closeTo(20, 1e-9)); + }); + + AudioCompareWindow window( + double start, + int frames, + List residual, + List reference, + ) => AudioCompareWindow( + startSeconds: start, + frames: frames, + residual: Float64List.fromList(residual), + reference: Float64List.fromList(reference), + residualHigh: Float64List(residual.length), + referenceHigh: Float64List(residual.length), + ); + + test('overall level weights windows by length and channel', () { + final windows = [ + window(0, 300, [0.001, 0.0], [0.1, 0.1]), + window(1, 100, [0.0, 0.0], [0.1, 0.1]), + ]; + // 0.001 * 300 / 2 channels over 0.1 * 400. + expect(audioCompareOverallDb(windows), closeTo(-24.26, 0.01)); + expect( + audioCompareOverallDb(windows, AudioCompareChannel.left), + closeTo(-21.25, 0.01), + ); + expect( + audioCompareOverallDb(windows, AudioCompareChannel.right), + audioCompareFloorDb, + ); + expect( + audioCompareSeries(windows, AudioCompareChannel.left).first, + closeTo(-20, 1e-9), + ); + expect(audioComparePeakFor(windows)!.seconds, 0); + }); + + test('notices for drift and different lengths', () { + Set notices({ + int? drift, + double other = 200, + double offset = 0, + }) => audioCompareNoticesFor( + correlation: 0.99, + overallDb: -30, + medianDb: -32, + driftCoarse: drift, + referenceSeconds: 200, + otherSeconds: other, + offsetSeconds: offset, + ); + expect(notices(), isEmpty); + expect(notices(drift: 2), isEmpty); + expect(notices(drift: -3), {AudioCompareNotice.drift}); + // An MP3 that starts 0.05 s later and is that much longer is fine. + expect(notices(other: 200.05, offset: 0.05), isEmpty); + expect(notices(other: 195), {AudioCompareNotice.durationMismatch}); + expect(notices(other: 204, offset: 0.5), { + AudioCompareNotice.durationMismatch, + }); + }); + + test('level reading boundaries', () { + expect(audioCompareLevelFor(-120), AudioCompareLevel.identical); + expect(audioCompareLevelFor(-90), AudioCompareLevel.nearlyIdentical); + expect(audioCompareLevelFor(-45), AudioCompareLevel.mostlyIdentical); + expect(audioCompareLevelFor(-31), AudioCompareLevel.moderate); + expect(audioCompareLevelFor(-12), AudioCompareLevel.large); + }); + }); +} diff --git a/test/audio_compare_engine_test.dart b/test/audio_compare_engine_test.dart new file mode 100644 index 000000000..b36ae895d --- /dev/null +++ b/test/audio_compare_engine_test.dart @@ -0,0 +1,561 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:ffmpeg_kit_flutter_new_audio/media_information.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:mornye/services/audio_analysis_ffmpeg.dart'; +import 'package:mornye/services/audio_analysis_jobs.dart'; +import 'package:mornye/services/audio_compare_dsp.dart'; +import 'package:mornye/services/audio_compare_engine.dart'; +import 'package:mornye/services/ffmpeg_desktop.dart'; +import 'package:mornye/services/ffmpeg_models.dart'; + +MediaInformation _media({ + String codec = 'flac', + int sampleRate = 44100, + int channels = 2, + double duration = 60, +}) => MediaInformation({ + 'streams': [ + { + 'codec_type': 'audio', + 'codec_name': codec, + 'sample_rate': '$sampleRate', + 'channels': channels, + 'duration': '$duration', + 'bits_per_raw_sample': codec == 'flac' ? '24' : '0', + }, + ], + 'format': {'format_name': codec, 'duration': '$duration'}, +}); + +/// FFmpeg stand-in: writes the files a command names, then waits for the +/// test to finish or cancel it. +class _FakeFFmpeg { + final started = >[]; + final sessions = >{}; + final startedSignal = StreamController.broadcast(); + int _next = 1; + bool autoComplete = false; + int returnCode = 0; + + Future<({int id, Future completed})> start( + List arguments, + ) async { + started.add(arguments); + for (final argument in arguments) { + if (argument.endsWith('.f32') || argument.endsWith('.rgba')) { + await File(argument).writeAsBytes(Uint8List(64)); + } + } + final id = _next++; + final completer = Completer(); + sessions[id] = completer; + if (autoComplete) { + completer.complete( + FFmpegSessionResult( + returnCode: returnCode, + logs: () async => 'fake log', + ), + ); + } + startedSignal.add(id); + return (id: id, completed: completer.future); + } + + Future cancel(int id) async { + final completer = sessions[id]; + if (completer != null && !completer.isCompleted) { + completer.complete( + FFmpegSessionResult(returnCode: 255, logs: () async => 'cancelled'), + ); + } + } +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late Directory temp; + setUp(() async { + temp = await Directory.systemTemp.createTemp('audio-compare-test-'); + }); + tearDown(() async { + if (await temp.exists()) await temp.delete(recursive: true); + }); + + Future> leftovers() async => [ + await for (final entity in temp.list()) + if (entity.path.contains('audio-compare-')) entity.path, + ]; + + AudioCompareEngine engine( + _FakeFFmpeg ffmpeg, { + Map probes = const {}, + }) => AudioCompareEngine( + start: ffmpeg.start, + cancel: ffmpeg.cancel, + probe: (path) async => probes.containsKey(path) ? probes[path] : _media(), + temporaryDirectory: () async => temp, + ); + + test('cancelling mid-run stops FFmpeg and removes temporary files', () async { + final ffmpeg = _FakeFFmpeg(); + final compare = engine(ffmpeg); + final stages = []; + final firstStart = ffmpeg.startedSignal.stream.first; + final result = compare.compare( + referencePath: '/music/a.flac', + otherPath: '/music/a.mp3', + onStage: stages.add, + ); + await firstStart; + // The excerpt pass has written its outputs into the run directory. + expect(await leftovers(), hasLength(1)); + compare.cancel(); + await expectLater(result, throwsA(isA())); + expect(await leftovers(), isEmpty); + expect(stages, [AudioCompareStage.preparing, AudioCompareStage.aligning]); + expect(ffmpeg.started, hasLength(1)); + }); + + test('disposing the engine also cancels and cleans up', () async { + final ffmpeg = _FakeFFmpeg(); + final compare = engine(ffmpeg); + final firstStart = ffmpeg.startedSignal.stream.first; + final result = compare.compare( + referencePath: '/music/a.flac', + otherPath: '/music/a.mp3', + ); + await firstStart; + compare.dispose(); + await expectLater(result, throwsA(isA())); + expect(await leftovers(), isEmpty); + }); + + test('a failed decode is an error and still cleans up', () async { + final ffmpeg = _FakeFFmpeg() + ..autoComplete = true + ..returnCode = 1; + await expectLater( + engine( + ffmpeg, + ).compare(referencePath: '/music/a.flac', otherPath: '/music/b.mp3'), + throwsA( + isA().having( + (e) => e.failure, + 'failure', + AudioCompareFailure.decodeFailed, + ), + ), + ); + expect(await leftovers(), isEmpty); + }); + + test('AC-4 is refused before any decode', () async { + final ffmpeg = _FakeFFmpeg(); + await expectLater( + engine( + ffmpeg, + probes: {'/music/b.m4a': _media(codec: 'ac4')}, + ).compare(referencePath: '/music/a.flac', otherPath: '/music/b.m4a'), + throwsA( + isA().having( + (e) => e.codecLabel, + 'codec', + 'AC-4', + ), + ), + ); + expect(ffmpeg.started, isEmpty); + }); + + test('an unreadable file names the file', () async { + final ffmpeg = _FakeFFmpeg(); + await expectLater( + engine( + ffmpeg, + probes: {'/music/broken.mp3': null}, + ).compare(referencePath: '/music/a.flac', otherPath: '/music/broken.mp3'), + throwsA( + isA() + .having((e) => e.failure, 'failure', AudioCompareFailure.unreadable) + .having((e) => e.fileName, 'file', 'broken.mp3'), + ), + ); + }); + + test('files shorter than two seconds are refused', () async { + final ffmpeg = _FakeFFmpeg(); + await expectLater( + engine( + ffmpeg, + probes: {'/music/short.mp3': _media(duration: 1.5)}, + ).compare(referencePath: '/music/a.flac', otherPath: '/music/short.mp3'), + throwsA( + isA().having( + (e) => e.failure, + 'failure', + AudioCompareFailure.tooShort, + ), + ), + ); + expect(ffmpeg.started, isEmpty); + }); + + test('a content URI reference is copied and the copy deleted', () async { + final ffmpeg = _FakeFFmpeg() + ..autoComplete = true + ..returnCode = 1; + final copy = File('${temp.path}/saf-copy.flac'); + final compare = AudioCompareEngine( + start: ffmpeg.start, + cancel: ffmpeg.cancel, + probe: (path) async => _media(), + temporaryDirectory: () async => temp, + copyContentUri: (uri) async { + await copy.writeAsBytes([1, 2, 3]); + return copy.path; + }, + ); + await expectLater( + compare.compare( + referencePath: 'content://provider/tree/primary%3AMusic%2Fsong.flac', + otherPath: '/music/b.mp3', + ), + throwsA(isA()), + ); + expect(ffmpeg.started.first, contains(copy.path)); + expect(await copy.exists(), isFalse); + }); + + test('display names of paths and URIs', () { + expect(audioCompareDisplayName('/music/Artist/Song.flac'), 'Song.flac'); + expect(audioCompareDisplayName(r'C:\Music\Song.mp3'), 'Song.mp3'); + expect( + audioCompareDisplayName( + 'content://provider/document/primary%3AMusic%2FSong.m4a', + ), + 'Song.m4a', + ); + expect( + audioCompareDisplayName('network://nas/Music/Song.flac'), + 'Song.flac', + ); + }); + + group('end to end with FFmpeg on this machine', () { + late DesktopFFmpegTools tools; + late Directory fixtures; + var available = false; + + setUpAll(() async { + tools = await DesktopFFmpegTools.locate(); + available = tools.ffmpegPath != null && tools.ffprobePath != null; + if (!available) return; + fixtures = await Directory.systemTemp.createTemp('audio-compare-e2e-'); + }); + tearDownAll(() async { + if (available) await fixtures.delete(recursive: true); + }); + + Future ffmpeg(List arguments) async { + final result = await Process.run(tools.ffmpegPath!, [ + '-hide_banner', + '-loglevel', + 'error', + '-y', + ...arguments, + ]); + return result.exitCode == 0; + } + + // A 90 s tone, chord and pink-noise mix with quiet and dense passages, + // long enough for the drift check's second excerpt. + Future reference() async { + final path = '${fixtures.path}/reference.flac'; + if (await File(path).exists()) return path; + final ok = await ffmpeg([ + '-f', + 'lavfi', + '-i', + "aevalsrc='0.3*sin(2*PI*220*t)*(1+0.5*sin(2*PI*0.5*t))" + '+0.15*sin(2*PI*1760*t+sin(2*PI*3*t))|' + '0.3*sin(2*PI*330*t)+0.1*sin(2*PI*5000*t)*gt(mod(t,4),2)' + "':s=44100:d=90", + '-f', + 'lavfi', + '-i', + 'anoisesrc=d=90:c=pink:r=44100:a=0.08:seed=11', + '-filter_complex', + "[1]volume='if(gt(mod(t,8),5),1,0.15)':eval=frame," + 'pan=stereo|c0=c0|c1=c0[n];[0][n]amix=inputs=2:normalize=0[o]', + '-map', + '[o]', + '-c:a', + 'flac', + '-sample_fmt', + 's32', + '-bits_per_raw_sample', + '24', + path, + ]); + expect(ok, isTrue); + return path; + } + + final cases = >{ + 'identical copy': ['-c:a', 'flac', 'identical.flac'], + 'MP3 128 kbps': ['-c:a', 'libmp3lame', '-b:a', '128k', 'mp3.mp3'], + 'MP3 without LAME header': [ + '-c:a', + 'libmp3lame', + '-b:a', + '128k', + '-write_xing', + '0', + 'mp3_noxing.mp3', + ], + 'AAC 128 kbps (M4A)': ['-c:a', 'aac', '-b:a', '128k', 'aac.m4a'], + 'AAC 128 kbps (ADTS)': [ + '-c:a', + 'aac', + '-b:a', + '128k', + '-f', + 'adts', + 'aac_adts.aac', + ], + 'starts 12345 samples later': [ + '-af', + 'adelay=12345S:all=1', + '-c:a', + 'flac', + 'delayed.flac', + ], + 'starts 7000 samples earlier': [ + '-af', + 'atrim=start_sample=7000', + '-c:a', + 'flac', + 'trimmed.flac', + ], + '8 kHz low-pass': [ + '-af', + 'lowpass=f=8000,lowpass=f=8000', + '-c:a', + 'flac', + 'lowpass.flac', + ], + 'resampled to 48 kHz': ['-ar', '48000', '-c:a', 'flac', 'r48.flac'], + '0.01% faster (drift)': [ + '-af', + 'asetrate=44104.41,aresample=44100', + '-c:a', + 'flac', + 'drift.flac', + ], + '3 dB quieter': ['-af', 'volume=-3dB', '-c:a', 'flac', 'quiet.flac'], + }; + + final results = {}; + + Future run(String name) async { + if (!available) { + markTestSkipped('ffmpeg and ffprobe are not on PATH'); + return null; + } + final referencePath = await reference(); + final arguments = cases[name]!; + final output = '${fixtures.path}/${arguments.last}'; + final encoded = await ffmpeg([ + '-i', + referencePath, + ...arguments.sublist(0, arguments.length - 1), + output, + ]); + if (!encoded) { + markTestSkipped('this FFmpeg cannot encode $name'); + return null; + } + final runner = DesktopFFmpegRunner(tools: () async => tools); + final compare = AudioCompareEngine( + start: runner.start, + cancel: runner.cancel, + probe: runner.probe, + temporaryDirectory: () async => temp, + ); + final watch = Stopwatch()..start(); + final result = await compare.compare( + referencePath: referencePath, + otherPath: output, + ); + watch.stop(); + results[name] = result; + // ignore: avoid_print + print( + '$name: offset ${result.offsetFrames} samples ' + '(${result.offsetMs.toStringAsFixed(2)} ms), ' + 'gain ${result.gainDb.toStringAsFixed(3)} dB, ' + 'correlation ${result.correlation.toStringAsFixed(4)}, ' + 'overall ${result.overallDb.toStringAsFixed(1)} dB, ' + 'median ${result.medianDb.toStringAsFixed(1)} dB, ' + 'excerpt ${result.excerptResidualDb.toStringAsFixed(1)} dB, ' + 'peak ${result.peak?.seconds.toStringAsFixed(1)} s ' + '${result.peak?.db.toStringAsFixed(1)} dB, ' + 'band ${result.band.name}, level ${result.level.name}, ' + 'drift ${result.driftMs?.toStringAsFixed(2)} ms, ' + 'notices ${result.notices.map((n) => n.name).join(',')}, ' + 'windows ${result.windows.length}, ' + 'spectrogram ${result.spectrogram != null}, ' + '${watch.elapsedMilliseconds} ms', + ); + expect(await leftovers(), isEmpty); + return result; + } + + tearDownAll(() { + for (final result in results.values) { + result.dispose(); + } + }); + + test('an identical copy cancels completely', () async { + final result = await run('identical copy'); + if (result == null) return; + expect(result.offsetFrames, 0); + expect(result.gain, 1); + expect(result.overallDb, audioCompareFloorDb); + expect(result.level, AudioCompareLevel.identical); + expect(result.notices, isEmpty); + expect(result.spectrogram, isNotNull); + expect(result.windows.length, inInclusiveRange(300, 2000)); + }); + + test('lossy copies line up and show their losses', () async { + for (final name in [ + 'MP3 128 kbps', + 'MP3 without LAME header', + 'AAC 128 kbps (M4A)', + 'AAC 128 kbps (ADTS)', + ]) { + final result = await run(name); + if (result == null) continue; + expect(result.correlation, greaterThan(0.99), reason: name); + expect(result.overallDb, inInclusiveRange(-60, -15), reason: name); + expect(result.notices, isEmpty, reason: name); + expect(result.gainDb.abs(), lessThan(1), reason: name); + } + final noXing = results['MP3 without LAME header']; + final withXing = results['MP3 128 kbps']; + if (noXing != null && withXing != null) { + // Without the header FFmpeg cannot remove the encoder delay. + expect(noXing.offsetFrames - withXing.offsetFrames, greaterThan(500)); + } + }); + + test('a shifted copy is found exactly in both directions', () async { + final later = await run('starts 12345 samples later'); + if (later == null) return; + expect(later.offsetFrames, 12345); + expect(later.overallDb, audioCompareFloorDb); + final earlier = (await run('starts 7000 samples earlier'))!; + expect(earlier.offsetFrames, -7000); + expect(earlier.notices, isEmpty); + // Only the windows covering the 7000 samples the trimmed file lacks + // differ. + final series = audioCompareSeries( + earlier.windows, + AudioCompareChannel.combined, + ); + final missing = (7000 / earlier.windows.first.frames).ceil(); + expect(series.take(missing), everyElement(greaterThan(-10))); + expect(series.skip(missing), everyElement(audioCompareFloorDb)); + }); + + test('a low-passed copy differs in the highs', () async { + final result = await run('8 kHz low-pass'); + if (result == null) return; + expect(result.band, AudioCompareBand.high); + expect(result.notices, isEmpty); + }); + + test('a resampled copy is compared at the reference rate', () async { + final result = await run('resampled to 48 kHz'); + if (result == null) return; + expect(result.sampleRate, 44100); + expect(result.resampled, isTrue); + expect(result.offsetFrames.abs(), lessThanOrEqualTo(1)); + expect(result.overallDb, lessThan(-40)); + }); + + test('drifting timing raises the notice', () async { + final result = await run('0.01% faster (drift)'); + if (result == null) return; + expect(result.meaningful, isFalse); + expect( + result.notices.intersection({ + AudioCompareNotice.drift, + AudioCompareNotice.residualHigh, + }), + isNotEmpty, + ); + }); + + test('a quieter copy is level matched', () async { + final result = await run('3 dB quieter'); + if (result == null) return; + expect(result.gainDb, closeTo(3, 0.01)); + expect(result.overallDb, lessThan(-90)); + }); + + test('unrelated audio is not the same source', () async { + if (!available) { + markTestSkipped('ffmpeg and ffprobe are not on PATH'); + return; + } + final referencePath = await reference(); + final noise = '${fixtures.path}/noise.flac'; + expect( + await ffmpeg([ + '-f', + 'lavfi', + '-i', + 'anoisesrc=d=90:c=white:r=44100:a=0.2:seed=5', + '-ac', + '2', + '-c:a', + 'flac', + noise, + ]), + isTrue, + ); + final runner = DesktopFFmpegRunner(tools: () async => tools); + final result = + await AudioCompareEngine( + start: runner.start, + cancel: runner.cancel, + probe: runner.probe, + temporaryDirectory: () async => temp, + ).compare( + referencePath: referencePath, + otherPath: noise, + includeSpectrogram: false, + ); + // ignore: avoid_print + print( + 'unrelated: correlation ${result.correlation.toStringAsFixed(4)}, ' + 'overall ${result.overallDb.toStringAsFixed(1)} dB, ' + 'notices ${result.notices.map((n) => n.name).join(',')}', + ); + expect( + result.notices, + containsAll([ + AudioCompareNotice.weakAlignment, + AudioCompareNotice.residualHigh, + ]), + ); + }); + }); +} diff --git a/test/audio_compare_ffmpeg_test.dart b/test/audio_compare_ffmpeg_test.dart new file mode 100644 index 000000000..98bc2c8bb --- /dev/null +++ b/test/audio_compare_ffmpeg_test.dart @@ -0,0 +1,313 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:mornye/services/audio_compare_dsp.dart'; +import 'package:mornye/services/audio_compare_ffmpeg.dart'; + +String _graph(List arguments) => + arguments[arguments.indexOf('-filter_complex') + 1]; + +void main() { + group('excerpt arguments', () { + test('decode both files once and write each excerpt as float PCM', () { + final arguments = buildAudioCompareExcerptArguments( + referencePath: '/music/a.flac', + otherPath: '/music/b.mp3', + sampleRate: 44100, + channels: 2, + excerpts: const [ + AudioCompareExcerpt( + input: 0, + startSample: 100, + endSample: 200, + outputPath: '/tmp/r.f32', + ), + AudioCompareExcerpt( + input: 1, + startSample: 50, + endSample: 260, + outputPath: '/tmp/o.f32', + ), + ], + ); + expect(arguments.sublist(0, 7), [ + '-hide_banner', + '-nostats', + '-y', + '-i', + '/music/a.flac', + '-i', + '/music/b.mp3', + ]); + expect( + _graph(arguments), + '[0:a:0]aresample=44100,' + 'aformat=sample_fmts=flt:channel_layouts=stereo,' + 'atrim=start_sample=100:end_sample=200[cmp_x0];' + '[1:a:0]aresample=44100,' + 'aformat=sample_fmts=flt:channel_layouts=stereo,' + 'atrim=start_sample=50:end_sample=260[cmp_x1]', + ); + expect(arguments.sublist(9), [ + '-map', + '[cmp_x0]', + '-f', + 'f32le', + '-c:a', + 'pcm_f32le', + '/tmp/r.f32', + '-map', + '[cmp_x1]', + '-f', + 'f32le', + '-c:a', + 'pcm_f32le', + '/tmp/o.f32', + ]); + }); + + test('the wide search splits one mono decimated decode per file', () { + final graph = _graph( + buildAudioCompareExcerptArguments( + referencePath: 'a', + otherPath: 'b', + sampleRate: 96000, + channels: 2, + decimatedRate: audioCompareCoarseRate, + excerpts: const [ + AudioCompareExcerpt( + input: 0, + startSample: 0, + endSample: 10, + outputPath: 'r0', + ), + AudioCompareExcerpt( + input: 1, + startSample: 0, + endSample: 20, + outputPath: 'o0', + ), + AudioCompareExcerpt( + input: 0, + startSample: 30, + endSample: 40, + outputPath: 'r1', + ), + AudioCompareExcerpt( + input: 1, + startSample: 25, + endSample: 45, + outputPath: 'o1', + ), + ], + ), + ); + expect( + graph, + '[0:a:0]aresample=96000,' + 'aformat=sample_fmts=flt:channel_layouts=mono,aresample=4000,' + 'asplit=2[cmp_s0][cmp_s2];' + '[cmp_s0]atrim=start_sample=0:end_sample=10[cmp_x0];' + '[cmp_s2]atrim=start_sample=30:end_sample=40[cmp_x2];' + '[1:a:0]aresample=96000,' + 'aformat=sample_fmts=flt:channel_layouts=mono,aresample=4000,' + 'asplit=2[cmp_s1][cmp_s3];' + '[cmp_s1]atrim=start_sample=0:end_sample=20[cmp_x1];' + '[cmp_s3]atrim=start_sample=25:end_sample=45[cmp_x3]', + ); + }); + }); + + group('merge graph', () { + test('a later other file is trimmed, an earlier one delayed', () { + String merge(int offset) => buildAudioCompareMergeGraph( + sampleRate: 48000, + channels: 2, + offsetFrames: offset, + ); + const chain = + 'aresample=48000,aformat=sample_fmts=flt:channel_layouts=stereo'; + expect( + merge(1105), + '[0:a:0]$chain,asetpts=N/SR/TB[cmp_ref];' + '[1:a:0]$chain,atrim=start_sample=1105,asetpts=N/SR/TB,apad[cmp_oth];' + '[cmp_ref][cmp_oth]amerge=inputs=2', + ); + expect( + merge(-2112), + contains('[1:a:0]$chain,adelay=delays=2112S:all=1,asetpts=N/SR/TB'), + ); + expect(merge(0), contains('[1:a:0]$chain,asetpts=N/SR/TB,apad')); + }); + }); + + group('stats arguments', () { + test('stereo residual and reference levels per window, in two bands', () { + final arguments = buildAudioCompareStatsArguments( + referencePath: '/music/a.flac', + otherPath: '/music/b.mp3', + sampleRate: 44100, + channels: 2, + offsetFrames: 0, + gain: 1.0512, + windowFrames: 3969, + metadataPath: r'C:\Temp\levels.txt', + ); + final graph = _graph(arguments); + expect( + graph, + contains( + 'pan=4c|c0=c0-1.051200000*c2|c1=c1-1.051200000*c3|c2=c0|c3=c1,' + 'asplit=2[cmp_full][cmp_hp];' + '[cmp_hp]highpass=f=8000:poles=2,highpass=f=8000:poles=2[cmp_high];' + '[cmp_full][cmp_high]amerge=inputs=2,' + 'asetnsamples=n=3969:p=0,' + 'astats=metadata=1:reset=1:measure_perchannel=RMS_level:' + 'measure_overall=none,', + ), + ); + expect( + graph, + endsWith(r"ametadata=print:file='C\:\\Temp\\levels.txt'[cmp_stats]"), + ); + expect(arguments.sublist(arguments.length - 5), [ + '-map', + '[cmp_stats]', + '-f', + 'null', + '-', + ]); + }); + + test('mono and an inverted gain stay valid pan expressions', () { + final graph = _graph( + buildAudioCompareStatsArguments( + referencePath: 'a', + otherPath: 'b', + sampleRate: 22050, + channels: 1, + offsetFrames: 0, + gain: -0.5, + windowFrames: 100, + metadataPath: '/tmp/m.txt', + ), + ); + expect(graph, contains('pan=2c|c0=c0+0.500000000*c1|c1=c0,')); + // The band split stays below Nyquist at low rates. + expect(graph, contains('highpass=f=8000')); + final low = _graph( + buildAudioCompareStatsArguments( + referencePath: 'a', + otherPath: 'b', + sampleRate: 16000, + channels: 1, + offsetFrames: 0, + gain: 1, + windowFrames: 100, + metadataPath: '/tmp/m.txt', + ), + ); + expect(low, contains('highpass=f=7200')); + }); + }); + + test('the residual spectrogram reuses the spectrogram filter', () { + final arguments = buildAudioCompareSpectrogramArguments( + referencePath: 'a', + otherPath: 'b', + sampleRate: 44100, + channels: 2, + offsetFrames: -10, + gain: 0.9, + durationSeconds: 600, + outputPath: '/tmp/s.rgba', + ); + final graph = _graph(arguments); + expect( + graph, + contains( + 'pan=stereo|c0=c0-0.900000000*c2|c1=c1-0.900000000*c3[cmp_residual];' + '[cmp_residual]aselect=', + ), + ); + expect(graph, contains('showspectrumpic=s=1600x800')); + expect(graph, endsWith('format=rgba[spectrum]')); + expect(arguments.sublist(arguments.length - 9), [ + '-map', + '[spectrum]', + '-frames:v', + '1', + '-f', + 'rawvideo', + '-pix_fmt', + 'rgba', + '/tmp/s.rgba', + ]); + }); + + group('metadata', () { + const metadata = ''' +frame:0 pts:0 pts_time:0 +lavfi.astats.1.RMS_level=-43.777571 +lavfi.astats.2.RMS_level=-inf +lavfi.astats.3.RMS_level=-10.000000 +lavfi.astats.4.RMS_level=-13.467247 +lavfi.astats.5.RMS_level=-64.218687 +lavfi.astats.6.RMS_level=-64.342160 +lavfi.astats.7.RMS_level=-62.779902 +lavfi.astats.8.RMS_level=-63.032422 +frame:1 pts:3969 pts_time:0.09 +lavfi.astats.1.RMS_level=-20.000000 +lavfi.astats.2.RMS_level=-20.000000 +lavfi.astats.3.RMS_level=-10.000000 +lavfi.astats.4.RMS_level=-10.000000 +lavfi.astats.5.RMS_level=-inf +lavfi.astats.6.RMS_level=-inf +lavfi.astats.7.RMS_level=-30.000000 +lavfi.astats.8.RMS_level=-30.000000 +'''; + + test('parses each window in channel order', () { + final levels = parseAudioCompareMetadata(metadata, channels: 8); + expect(levels, hasLength(2)); + expect(levels[0][0], closeTo(-43.777571, 1e-9)); + expect(levels[0][1], double.negativeInfinity); + expect(levels[1][6], -30); + }); + + test('turns levels into timeline windows', () { + final windows = audioCompareWindowsFromMetadata( + parseAudioCompareMetadata(metadata, channels: 8), + channels: 2, + sampleRate: 44100, + windowFrames: 3969, + totalFrames: 3969 + 1000, + ); + expect(windows, hasLength(2)); + expect(windows[1].startSeconds, closeTo(0.09, 1e-9)); + expect(windows[0].frames, 3969); + expect(windows[1].frames, 1000); + expect(windows[0].residual[1], 0); + expect(windows[1].relativeDb(AudioCompareChannel.combined), -10); + expect( + windows[0].relativeDb(AudioCompareChannel.right), + audioCompareFloorDb, + ); + expect(windows[1].referenceHigh[0], closeTo(0.001, 1e-12)); + expect(windows[1].residualHigh[0], 0); + }); + + test('a truncated window keeps missing channels silent', () { + final levels = parseAudioCompareMetadata( + 'frame:0 pts:0 pts_time:0\nlavfi.astats.1.RMS_level=-6\n', + channels: 4, + ); + final windows = audioCompareWindowsFromMetadata( + levels, + channels: 1, + sampleRate: 48000, + windowFrames: 480, + ); + expect(windows.single.reference[0], 0); + expect(windows.single.residual[0], greaterThan(0)); + }); + }); +}