diff --git a/lib/widgets/audio_analysis_models.dart b/lib/widgets/audio_analysis_models.dart index 2b6d0c39..4ff6e3a0 100644 --- a/lib/widgets/audio_analysis_models.dart +++ b/lib/widgets/audio_analysis_models.dart @@ -3,7 +3,7 @@ part of 'audio_analysis_widget.dart'; // Analysis result models and per-run parameter records. class AudioAnalysisData { - static const cacheVersion = 6; + static const cacheVersion = 7; final String filePath; final int fileSize; @@ -145,8 +145,13 @@ class ChannelAnalysisStats { class _GeneratedSpectrogram { final ui.Image image; final Uint8List rgba; + final Uint8List? cutoffIntensity; - const _GeneratedSpectrogram({required this.image, required this.rgba}); + const _GeneratedSpectrogram({ + required this.image, + required this.rgba, + this.cutoffIntensity, + }); } class _AudioAnalysisRunResult { @@ -160,24 +165,24 @@ class _AudioAnalysisRunResult { } class _SpectralCutoffParams { - final Uint8List rgba; + final Uint8List intensity; final int width; final int height; final double maxFrequencyHz; const _SpectralCutoffParams({ - required this.rgba, + required this.intensity, required this.width, required this.height, required this.maxFrequencyHz, }); } -double? _estimateBroadbandSpectralCutoffInIsolate( +double? _estimateEffectiveSpectralCutoffInIsolate( _SpectralCutoffParams params, ) { - return estimateBroadbandSpectralCutoffHz( - rgba: params.rgba, + return estimateEffectiveSpectralCutoffHz( + intensity: params.intensity, width: params.width, height: params.height, maxFrequencyHz: params.maxFrequencyHz, diff --git a/lib/widgets/audio_analysis_widget.dart b/lib/widgets/audio_analysis_widget.dart index 9c7c62f2..14fde45d 100644 --- a/lib/widgets/audio_analysis_widget.dart +++ b/lib/widgets/audio_analysis_widget.dart @@ -21,29 +21,60 @@ part 'audio_analysis_spectrogram.dart'; const int audioSpectrogramWidth = 1600; const int audioSpectrogramHeight = 800; +const int audioSpectralAnalysisWidth = 400; const double audioSpectrogramDynamicRangeDb = 120; -String buildAudioSpectrogramFilter({int channel = -1}) { - final channelFilter = channel >= 0 ? 'pan=mono|c0=c$channel,' : ''; - return '[0:a:0]${channelFilter}aformat=sample_fmts=fltp,' - 'showspectrumpic=' - 's=${audioSpectrogramWidth}x$audioSpectrogramHeight:' - 'legend=0:mode=combined:color=intensity:scale=log:fscale=lin:' +String _buildShowspectrumOptions({required int width, required String color}) { + return 'showspectrumpic=' + 's=${width}x$audioSpectrogramHeight:' + 'legend=0:mode=combined:color=$color:scale=log:fscale=lin:' 'win_func=hann:drange=${audioSpectrogramDynamicRangeDb.toStringAsFixed(0)}:' - 'limit=0,format=rgba[spectrum]'; + 'limit=0'; +} + +String buildAudioSpectrogramFilter({ + int channel = -1, + bool includeCutoffPlane = false, +}) { + final channelFilter = channel >= 0 ? 'pan=mono|c0=c$channel,' : ''; + final input = '[0:a:0]${channelFilter}aformat=sample_fmts=fltp'; + final display = _buildShowspectrumOptions( + width: audioSpectrogramWidth, + color: 'intensity', + ); + if (!includeCutoffPlane) { + return '$input,$display,format=rgba[spectrum]'; + } + + // The display palette encodes quiet bins as saturated blue/purple pixels, + // so RGB brightness is not a monotonic measure of spectral magnitude. Keep + // the colorful UI output, but generate a small fixed-hue plane whose gray + // values can be used safely for effective-bandwidth detection. + final cutoff = _buildShowspectrumOptions( + width: audioSpectralAnalysisWidth, + color: 'green', + ); + return '$input,asplit=2[display_input][cutoff_input];' + '[display_input]$display,format=rgba[spectrum];' + '[cutoff_input]$cutoff,format=gray[cutoff]'; } List buildAudioSpectrogramArguments({ required String inputPath, required String outputPath, + String? cutoffOutputPath, int channel = -1, }) { - return [ + final arguments = [ '-hide_banner', + '-y', '-i', inputPath, '-filter_complex', - buildAudioSpectrogramFilter(channel: channel), + buildAudioSpectrogramFilter( + channel: channel, + includeCutoffPlane: cutoffOutputPath != null, + ), '-map', '[spectrum]', '-frames:v', @@ -52,9 +83,22 @@ List buildAudioSpectrogramArguments({ 'rawvideo', '-pix_fmt', 'rgba', - '-y', outputPath, ]; + if (cutoffOutputPath != null) { + arguments.addAll([ + '-map', + '[cutoff]', + '-frames:v', + '1', + '-f', + 'rawvideo', + '-pix_fmt', + 'gray', + cutoffOutputPath, + ]); + } + return arguments; } class AudioAstatsSummary { @@ -216,8 +260,8 @@ double? _parseLastMetadataValue(String metadata, String key) { return value; } -double? estimateBroadbandSpectralCutoffHz({ - required Uint8List rgba, +double? estimateEffectiveSpectralCutoffHz({ + required Uint8List intensity, required int width, required int height, required double maxFrequencyHz, @@ -225,85 +269,161 @@ double? estimateBroadbandSpectralCutoffHz({ if (width <= 0 || height <= 0 || maxFrequencyHz <= 0 || - rgba.length < width * height * 4) { + intensity.length < width * height) { return null; } - // A high percentile captures musical energy without letting a handful of - // transient pixels dictate the cutoff. Work in an integer histogram so the - // calculation stays deterministic and cheap enough for a background isolate. - final rowStrength = Float64List(height); + // Use a high temporal percentile so sustained musical bandwidth wins over + // silence, while short ultrasonic transients do not define the cutoff. + // These bytes come from FFmpeg's fixed-hue `color=green` output, where gray + // value is monotonic with magnitude; display-palette RGB is deliberately not + // accepted here because its blue noise floor has deceptively large channels. + final profile = Float64List(height); final histogram = Uint32List(256); final percentileTarget = math.max(0, (width * 0.90).floor()); for (var y = 0; y < height; y++) { histogram.fillRange(0, histogram.length, 0); - final rowStart = y * width * 4; + final rowStart = y * width; for (var x = 0; x < width; x++) { - final offset = rowStart + x * 4; - final intensity = math.max( - rgba[offset], - math.max(rgba[offset + 1], rgba[offset + 2]), - ); - histogram[intensity]++; + histogram[intensity[rowStart + x]]++; } var cumulative = 0; for (var value = 0; value < histogram.length; value++) { cumulative += histogram[value]; if (cumulative > percentileTarget) { - rowStrength[y] = value / 255.0; + // showspectrumpic stores Nyquist at the top. Reverse it here so array + // indices increase with frequency, which makes edge detection clearer. + profile[height - y - 1] = value.toDouble(); break; } } } final hzPerRow = maxFrequencyHz / height; - final smoothingRadius = math.max(1, (375 / hzPerRow).ceil()); + final smoothingRadius = math.max(1, (50 / hzPerRow).ceil()); final smoothed = Float64List(height); - var maximum = 0.0; var running = 0.0; var windowStart = 0; var windowEnd = -1; - for (var y = 0; y < height; y++) { - final desiredStart = math.max(0, y - smoothingRadius); - final desiredEnd = math.min(height - 1, y + smoothingRadius); + for (var index = 0; index < height; index++) { + final desiredStart = math.max(0, index - smoothingRadius); + final desiredEnd = math.min(height - 1, index + smoothingRadius); while (windowEnd < desiredEnd) { windowEnd++; - running += rowStrength[windowEnd]; + running += profile[windowEnd]; } while (windowStart < desiredStart) { - running -= rowStrength[windowStart]; + running -= profile[windowStart]; windowStart++; } - final value = running / (windowEnd - windowStart + 1); - smoothed[y] = value; - if (value > maximum) maximum = value; + smoothed[index] = running / (windowEnd - windowStart + 1); } - if (maximum <= 0) return null; - // Absolute floor rejects the deep-blue -100 dBFS noise floor. The relative - // term adapts to quiet masters. A valid edge must span at least 1.5 kHz, - // which deliberately rejects isolated ultrasonic pilot tones. - final activeThreshold = math.max(0.035, maximum * 0.10); - final minimumBandRows = math.max(3, (1500 / hzPerRow).ceil()); - var runStart = -1; - var runLength = 0; - for (var y = 0; y < height; y++) { - if (smoothed[y] >= activeThreshold) { - if (runStart < 0) runStart = y; - runLength++; - if (runLength >= minimumBandRows) { - final frequency = (height - runStart) / height * maxFrequencyHz; - return frequency.clamp(0.0, maxFrequencyHz).toDouble(); + final centralStart = math.max(0, (height * 0.05).floor()); + final centralEnd = math.min(height, (height * 0.95).ceil()); + final lowLevel = _spectralPercentile( + smoothed, + centralStart, + centralEnd, + 0.10, + ); + final highLevel = _spectralPercentile( + smoothed, + centralStart, + centralEnd, + 0.95, + ); + final dynamicSpan = highLevel - lowLevel; + if (highLevel < 24) return null; + if (dynamicSpan < 1) return maxFrequencyHz; + + // Locate a sharp downward edge over roughly 200 Hz, then validate it using + // wider bands on both sides. This detects codec/low-pass bandwidth edges but + // rejects a gradual musical roll-off or a narrow ultrasonic pilot. + final edgeSpanRows = math.max(2, (200 / hzPerRow).ceil()); + final topGuardRows = math.max(edgeSpanRows, (500 / hzPerRow).ceil()); + final minSearchHz = math.max(1000.0, math.min(4000.0, maxFrequencyHz * 0.20)); + final searchStart = math.max(edgeSpanRows, (minSearchHz / hzPerRow).floor()); + final searchEnd = height - edgeSpanRows - topGuardRows; + final candidateStarts = []; + for (var start = searchStart; start < searchEnd; start++) { + final drop = smoothed[start] - smoothed[start + edgeSpanRows]; + if (drop > 0) candidateStarts.add(start); + } + candidateStarts.sort((a, b) { + final aDrop = smoothed[a] - smoothed[a + edgeSpanRows]; + final bDrop = smoothed[b] - smoothed[b + edgeSpanRows]; + return bDrop.compareTo(aDrop); + }); + + final minimumDrop = math.max(12.0, dynamicSpan * 0.18); + for (final bestStart in candidateStarts) { + final bestLocalDrop = + smoothed[bestStart] - smoothed[bestStart + edgeSpanRows]; + if (bestLocalDrop < minimumDrop * 0.60) break; + final edgeIndex = (bestStart + edgeSpanRows / 2).round(); + final gapRows = math.max(1, (100 / hzPerRow).ceil()); + final supportRows = math.max(3, (1200 / hzPerRow).ceil()); + final belowEnd = edgeIndex - gapRows; + final belowStart = math.max(0, belowEnd - supportRows); + final aboveStart = edgeIndex + gapRows; + final aboveEnd = math.min(height, aboveStart + supportRows); + if (belowEnd > belowStart && aboveEnd > aboveStart) { + final belowLevel = _spectralMedian(smoothed, belowStart, belowEnd); + final aboveLevel = _spectralMedian(smoothed, aboveStart, aboveEnd); + final tailLevel = _spectralMedian(smoothed, aboveStart, height); + final baseStart = math.max(0, edgeIndex - (3000 / hzPerRow).ceil()); + final baseLevel = _spectralMedian(smoothed, baseStart, belowEnd); + if (belowLevel - aboveLevel >= minimumDrop && + belowLevel - tailLevel >= minimumDrop && + baseLevel - tailLevel >= minimumDrop) { + final cutoff = (edgeIndex + 0.5) * hzPerRow; + return cutoff.clamp(0.0, maxFrequencyHz).toDouble(); } - } else { - runStart = -1; - runLength = 0; } } + + // A genuinely broadband signal with no internal falling edge reaches the + // analysis ceiling. Report Nyquist only when both its baseband and top band + // are populated; silence or an isolated high-frequency line returns null. + final basebandLevel = _spectralMedian( + smoothed, + (height * 0.05).floor(), + math.max(1, (height * 0.50).floor()), + ); + final topBandLevel = _spectralMedian( + smoothed, + (height * 0.90).floor(), + math.max(1, (height * 0.98).floor()), + ); + if (basebandLevel >= 24 && topBandLevel >= basebandLevel - minimumDrop) { + return maxFrequencyHz; + } return null; } +double _spectralMedian(Float64List values, int start, int end) { + return _spectralPercentile(values, start, end, 0.50); +} + +double _spectralPercentile( + Float64List values, + int start, + int end, + double percentile, +) { + final safeStart = start.clamp(0, values.length).toInt(); + final safeEnd = end.clamp(safeStart, values.length).toInt(); + if (safeEnd <= safeStart) return 0; + final sorted = values.sublist(safeStart, safeEnd)..sort(); + final index = ((sorted.length - 1) * percentile) + .round() + .clamp(0, sorted.length - 1) + .toInt(); + return sorted[index]; +} + class AudioAnalysisCard extends StatefulWidget { final String filePath; @@ -608,12 +728,20 @@ class _AudioAnalysisCardState extends State { final info = await _getMediaInfo(workingPath); _GeneratedSpectrogram? spectrogram; try { - spectrogram = await _generateSpectrogram(workingPath, channel: -1); + spectrogram = await _generateSpectrogram( + workingPath, + channel: -1, + includeCutoffPlane: true, + ); + final cutoffIntensity = spectrogram.cutoffIntensity; + if (cutoffIntensity == null) { + throw Exception('FFmpeg spectral cutoff plane was not generated'); + } final spectralCutoffHz = await compute( - _estimateBroadbandSpectralCutoffInIsolate, + _estimateEffectiveSpectralCutoffInIsolate, _SpectralCutoffParams( - rgba: spectrogram.rgba, - width: audioSpectrogramWidth, + intensity: cutoffIntensity, + width: audioSpectralAnalysisWidth, height: audioSpectrogramHeight, maxFrequencyHz: info.sampleRate / 2, ), @@ -701,17 +829,20 @@ class _AudioAnalysisCardState extends State { Future<_GeneratedSpectrogram> _generateSpectrogram( String inputPath, { required int channel, + bool includeCutoffPlane = false, }) async { final tempDir = await getTemporaryDirectory(); final rawPath = '${tempDir.path}/analysis_spectrum_' '${DateTime.now().microsecondsSinceEpoch}_${channel + 1}.rgba'; + final cutoffPath = includeCutoffPlane ? '$rawPath.cutoff.gray' : null; try { final session = await FFmpegKit.executeWithArguments( buildAudioSpectrogramArguments( inputPath: inputPath, outputPath: rawPath, + cutoffOutputPath: cutoffPath, channel: channel, ), ); @@ -733,6 +864,21 @@ class _AudioAnalysisCardState extends State { final rgba = rawBytes.length == expectedLength ? rawBytes : Uint8List.sublistView(rawBytes, 0, expectedLength); + Uint8List? cutoffIntensity; + if (cutoffPath != null) { + final expectedCutoffLength = + audioSpectralAnalysisWidth * audioSpectrogramHeight; + final cutoffBytes = await File(cutoffPath).readAsBytes(); + if (cutoffBytes.length < expectedCutoffLength) { + throw Exception( + 'Incomplete spectral cutoff output ' + '(${cutoffBytes.length}/$expectedCutoffLength bytes)', + ); + } + cutoffIntensity = cutoffBytes.length == expectedCutoffLength + ? cutoffBytes + : Uint8List.sublistView(cutoffBytes, 0, expectedCutoffLength); + } final completer = Completer(); ui.decodeImageFromPixels( rgba, @@ -741,11 +887,20 @@ class _AudioAnalysisCardState extends State { ui.PixelFormat.rgba8888, completer.complete, ); - return _GeneratedSpectrogram(image: await completer.future, rgba: rgba); + return _GeneratedSpectrogram( + image: await completer.future, + rgba: rgba, + cutoffIntensity: cutoffIntensity, + ); } finally { try { await File(rawPath).delete(); } catch (_) {} + if (cutoffPath != null) { + try { + await File(cutoffPath).delete(); + } catch (_) {} + } } } diff --git a/test/audio_analysis_spectrogram_test.dart b/test/audio_analysis_spectrogram_test.dart index 6d716bd5..2365be8d 100644 --- a/test/audio_analysis_spectrogram_test.dart +++ b/test/audio_analysis_spectrogram_test.dart @@ -117,17 +117,40 @@ lavfi.r128.true_peak=0.907 expect(arguments, isNot(contains('-ac'))); expect(arguments, isNot(contains('-loglevel'))); }); + + test('renders a monotonic cutoff plane beside the display image', () { + final arguments = buildAudioSpectrogramArguments( + inputPath: 'source.flac', + outputPath: 'spectrum.rgba', + cutoffOutputPath: 'cutoff.gray', + ); + final filter = arguments[arguments.indexOf('-filter_complex') + 1]; + + expect(filter, contains('asplit=2')); + expect(filter, contains('color=intensity')); + expect(filter, contains('s=400x800')); + expect(filter, contains('color=green')); + expect(filter, contains('format=gray[cutoff]')); + expect( + arguments, + containsAllInOrder(['-map', '[cutoff]', '-frames:v', '1']), + ); + expect( + arguments, + containsAllInOrder(['-pix_fmt', 'gray', 'cutoff.gray']), + ); + }); }); - group('broadband spectral cutoff', () { + group('effective spectral cutoff', () { const width = 200; const height = 800; const nyquist = 96000.0; test('ignores a narrow ultrasonic pilot above a 22 kHz music band', () { - final rgba = _blankRgba(width, height); + final intensity = _blankIntensity(width, height, value: 12); _paintFrequencyBand( - rgba, + intensity, width: width, height: height, maxFrequencyHz: nyquist, @@ -136,7 +159,7 @@ lavfi.r128.true_peak=0.907 intensity: 220, ); _paintFrequencyBand( - rgba, + intensity, width: width, height: height, maxFrequencyHz: nyquist, @@ -145,8 +168,8 @@ lavfi.r128.true_peak=0.907 intensity: 255, ); - final cutoff = estimateBroadbandSpectralCutoffHz( - rgba: rgba, + final cutoff = estimateEffectiveSpectralCutoffHz( + intensity: intensity, width: width, height: height, maxFrequencyHz: nyquist, @@ -156,10 +179,45 @@ lavfi.r128.true_peak=0.907 expect(cutoff!, inInclusiveRange(21000, 23500)); }); - test('retains genuine broadband ultrasonic content', () { - final rgba = _blankRgba(width, height); + test('rejects a low noise floor above a 15.8 kHz bandwidth edge', () { + const cdNyquist = 22050.0; + final intensity = _blankIntensity(width, height, value: 42); _paintFrequencyBand( - rgba, + intensity, + width: width, + height: height, + maxFrequencyHz: cdNyquist, + lowHz: 0, + highHz: 15800, + intensity: 100, + ); + _paintFrequencyBand( + intensity, + width: width, + height: height, + maxFrequencyHz: cdNyquist, + lowHz: 15800, + highHz: cdNyquist, + intensity: 85, + startColumn: 0, + endColumn: 10, + ); + + final cutoff = estimateEffectiveSpectralCutoffHz( + intensity: intensity, + width: width, + height: height, + maxFrequencyHz: cdNyquist, + ); + + expect(cutoff, isNotNull); + expect(cutoff!, inInclusiveRange(15300, 16300)); + }); + + test('retains genuine broadband ultrasonic content', () { + final intensity = _blankIntensity(width, height, value: 10); + _paintFrequencyBand( + intensity, width: width, height: height, maxFrequencyHz: nyquist, @@ -168,8 +226,8 @@ lavfi.r128.true_peak=0.907 intensity: 180, ); - final cutoff = estimateBroadbandSpectralCutoffHz( - rgba: rgba, + final cutoff = estimateEffectiveSpectralCutoffHz( + intensity: intensity, width: width, height: height, maxFrequencyHz: nyquist, @@ -179,10 +237,23 @@ lavfi.r128.true_peak=0.907 expect(cutoff!, inInclusiveRange(47000, 49500)); }); + test('reports Nyquist for full-bandwidth content', () { + final intensity = _blankIntensity(width, height, value: 180); + + final cutoff = estimateEffectiveSpectralCutoffHz( + intensity: intensity, + width: width, + height: height, + maxFrequencyHz: nyquist, + ); + + expect(cutoff, nyquist); + }); + test('does not report an isolated line as a broadband cutoff', () { - final rgba = _blankRgba(width, height); + final intensity = _blankIntensity(width, height); _paintFrequencyBand( - rgba, + intensity, width: width, height: height, maxFrequencyHz: nyquist, @@ -191,8 +262,8 @@ lavfi.r128.true_peak=0.907 intensity: 255, ); - final cutoff = estimateBroadbandSpectralCutoffHz( - rgba: rgba, + final cutoff = estimateEffectiveSpectralCutoffHz( + intensity: intensity, width: width, height: height, maxFrequencyHz: nyquist, @@ -203,8 +274,8 @@ lavfi.r128.true_peak=0.907 test('rejects invalid or incomplete images', () { expect( - estimateBroadbandSpectralCutoffHz( - rgba: Uint8List(3), + estimateEffectiveSpectralCutoffHz( + intensity: Uint8List(0), width: 1, height: 1, maxFrequencyHz: nyquist, @@ -215,31 +286,29 @@ lavfi.r128.true_peak=0.907 }); } -Uint8List _blankRgba(int width, int height) { - final rgba = Uint8List(width * height * 4); - for (var offset = 3; offset < rgba.length; offset += 4) { - rgba[offset] = 255; - } - return rgba; +Uint8List _blankIntensity(int width, int height, {int value = 0}) { + final intensity = Uint8List(width * height); + if (value > 0) intensity.fillRange(0, intensity.length, value); + return intensity; } void _paintFrequencyBand( - Uint8List rgba, { + Uint8List values, { required int width, required int height, required double maxFrequencyHz, required double lowHz, required double highHz, required int intensity, + int startColumn = 0, + int? endColumn, }) { + final columnEnd = endColumn ?? width; for (var y = 0; y < height; y++) { final frequency = (height - y - 0.5) / height * maxFrequencyHz; if (frequency < lowHz || frequency > highHz) continue; - for (var x = 0; x < width; x++) { - final offset = (y * width + x) * 4; - rgba[offset] = intensity; - rgba[offset + 1] = intensity; - rgba[offset + 2] = intensity; + for (var x = startColumn; x < columnEnd; x++) { + values[y * width + x] = intensity; } } }