mirror of
https://github.com/zarzet/SpotiFLAC-Mobile.git
synced 2026-11-21 20:00:49 +01:00
feat(rust): describe how songs sound from decoded windows
Add an audio_features native data job that reads one or two decoded mono windows and returns a compact, versioned vector of 42 numbers: tempo and its confidence, loudness and its spread, brightness, rolloff, flatness, onset rate and flux, low/mid/high band shares, key, mode and key strength, and 13 MFCC means and spreads. Silent or too short audio yields no features. The job reuses the hi-res FFT and the AutoMix tempo detector and checks for cancellation as it goes.
This commit is contained in:
1 parent
c1413461bb
commit
1780693f6d
5 files changed
+764
-2
No files matched your search
@@ -19,6 +19,7 @@ pub mod matching;
|
||||
pub mod media;
|
||||
pub mod metadata;
|
||||
pub mod native_audio_analysis;
|
||||
pub mod native_audio_features;
|
||||
pub mod native_palette;
|
||||
pub mod progress;
|
||||
pub mod resolver;
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
//! [`HiResCheckError::Unsupported`], which callers treat as "check skipped".
|
||||
|
||||
mod evidence;
|
||||
mod fft;
|
||||
pub(crate) mod fft;
|
||||
mod replacement;
|
||||
pub use replacement::replacement_preserves_audio;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -8,7 +8,7 @@ use std::io::Read;
|
||||
const MAX_PCM_BYTES: usize = 11_025 * 2 * 25;
|
||||
const MAX_SPECTRAL_BYTES: usize = 4 << 20;
|
||||
|
||||
fn read_bounded(
|
||||
pub(crate) fn read_bounded(
|
||||
path: &str,
|
||||
maximum: usize,
|
||||
check: &dyn Fn() -> Result<(), String>,
|
||||
|
||||
@@ -0,0 +1,714 @@
|
||||
//! A compact description of how a song sounds, so Autoplay can pick songs
|
||||
//! that sound like the listening session and flow into each other. Flutter
|
||||
//! decodes one or two short mono windows; whole files never come here, and
|
||||
//! no model or network is involved.
|
||||
|
||||
use crate::media::hires::fft::Radix2Fft;
|
||||
use crate::native_audio_analysis::{analyze_automix_pcm, read_bounded};
|
||||
use serde::Serialize;
|
||||
|
||||
/// Changes whenever the vector's layout or meaning does, so stored vectors
|
||||
/// of another version are analyzed again.
|
||||
pub const AUDIO_FEATURES_VERSION: u32 = 1;
|
||||
/// The rate of the mono s16le windows Flutter decodes.
|
||||
pub const AUDIO_FEATURES_SAMPLE_RATE: usize = 22_050;
|
||||
pub const MAX_AUDIO_FEATURE_WINDOWS: usize = 2;
|
||||
pub const MFCC_COEFFICIENTS: usize = 13;
|
||||
const MAX_WINDOW_BYTES: usize = AUDIO_FEATURES_SAMPLE_RATE * 2 * 25;
|
||||
const FRAME: usize = 2048;
|
||||
const HOP: usize = 512;
|
||||
const BINS: usize = FRAME / 2 + 1;
|
||||
const BIN_HZ: f64 = AUDIO_FEATURES_SAMPLE_RATE as f64 / FRAME as f64;
|
||||
const MEL_BANDS: usize = 40;
|
||||
/// Frames quieter than this stay out of the spectral statistics.
|
||||
const SILENT_DB: f64 = -60.0;
|
||||
/// Less audible audio than this describes nothing reliably.
|
||||
const MIN_AUDIBLE_SECONDS: f64 = 3.0;
|
||||
|
||||
// Krumhansl-Kessler key profiles, tonic first.
|
||||
const MAJOR_PROFILE: [f64; 12] = [
|
||||
6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88,
|
||||
];
|
||||
const MINOR_PROFILE: [f64; 12] = [
|
||||
6.33, 2.68, 3.52, 5.38, 2.60, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17,
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize)]
|
||||
pub struct AudioFeatures {
|
||||
/// Beats per minute, 60 to 180, or 0 when no beat was found.
|
||||
pub tempo: f64,
|
||||
pub tempo_confidence: f64,
|
||||
/// Mean and standard deviation of the audible frames' RMS, in dBFS.
|
||||
pub loudness: f64,
|
||||
pub loudness_spread: f64,
|
||||
/// Spectral centroid mean and standard deviation, in Hz (brightness).
|
||||
pub centroid: f64,
|
||||
pub centroid_spread: f64,
|
||||
/// The frequency below which 85% of the spectral magnitude lies, in Hz.
|
||||
pub rolloff: f64,
|
||||
/// Spectral flatness, near 0 for tones and about 0.56 for white noise.
|
||||
pub flatness: f64,
|
||||
/// Onsets per second and mean spectral flux (rhythmic density).
|
||||
pub onset_rate: f64,
|
||||
pub flux: f64,
|
||||
/// Energy shares below 250 Hz, up to 4 kHz and above.
|
||||
pub bands: [f64; 3],
|
||||
/// The key's tonic as a pitch class (C = 0), its mode, and how well the
|
||||
/// chroma correlates with that key's profile (-1 to 1).
|
||||
pub key: usize,
|
||||
pub major: bool,
|
||||
pub key_strength: f64,
|
||||
/// Means and standard deviations of MFCC 1 to 13 (timbre).
|
||||
pub mfcc: [f64; MFCC_COEFFICIENTS],
|
||||
pub mfcc_spread: [f64; MFCC_COEFFICIENTS],
|
||||
}
|
||||
|
||||
impl AudioFeatures {
|
||||
/// The length of [`AudioFeatures::to_vector`].
|
||||
pub const LEN: usize = 16 + 2 * MFCC_COEFFICIENTS;
|
||||
|
||||
/// The stored layout: tempo, tempo confidence, loudness, loudness
|
||||
/// spread, centroid, centroid spread, rolloff, flatness, onset rate,
|
||||
/// flux, the three band shares, key, mode (1 major), key strength, the
|
||||
/// MFCC means, then their spreads.
|
||||
pub fn to_vector(&self) -> Vec<f64> {
|
||||
let mut vector = vec![
|
||||
self.tempo,
|
||||
self.tempo_confidence,
|
||||
self.loudness,
|
||||
self.loudness_spread,
|
||||
self.centroid,
|
||||
self.centroid_spread,
|
||||
self.rolloff,
|
||||
self.flatness,
|
||||
self.onset_rate,
|
||||
self.flux,
|
||||
];
|
||||
vector.extend(self.bands);
|
||||
vector.extend([
|
||||
self.key as f64,
|
||||
if self.major { 1.0 } else { 0.0 },
|
||||
self.key_strength,
|
||||
]);
|
||||
vector.extend(self.mfcc);
|
||||
vector.extend(self.mfcc_spread);
|
||||
vector
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads one or two decoded windows, each at most 25 seconds.
|
||||
pub fn analyze_audio_features_files(
|
||||
paths: &[&str],
|
||||
check: &dyn Fn() -> Result<(), String>,
|
||||
) -> Result<Option<AudioFeatures>, String> {
|
||||
if paths.is_empty() || paths.len() > MAX_AUDIO_FEATURE_WINDOWS {
|
||||
return Err("audio features take one or two windows".into());
|
||||
}
|
||||
let windows = paths
|
||||
.iter()
|
||||
.map(|path| read_bounded(path, MAX_WINDOW_BYTES, check))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let windows: Vec<&[u8]> = windows.iter().map(Vec::as_slice).collect();
|
||||
analyze_audio_features(&windows, check)
|
||||
}
|
||||
|
||||
/// Describes mono s16le windows at [`AUDIO_FEATURES_SAMPLE_RATE`]. None
|
||||
/// means too little audible audio to describe.
|
||||
pub fn analyze_audio_features(
|
||||
windows: &[&[u8]],
|
||||
check: &dyn Fn() -> Result<(), String>,
|
||||
) -> Result<Option<AudioFeatures>, String> {
|
||||
check()?;
|
||||
if windows.is_empty() || windows.len() > MAX_AUDIO_FEATURE_WINDOWS {
|
||||
return Err("audio features take one or two windows".into());
|
||||
}
|
||||
if windows.iter().any(|window| window.len() > MAX_WINDOW_BYTES) {
|
||||
return Err("audio feature window exceeds the size limit".into());
|
||||
}
|
||||
let mut spectra = Spectra::new();
|
||||
let mut tempos = Vec::new();
|
||||
for window in windows {
|
||||
let samples: Vec<f64> = window
|
||||
.as_chunks::<2>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|bytes| f64::from(i16::from_le_bytes(*bytes)) / 32768.0)
|
||||
.collect();
|
||||
spectra.add_window(&samples, check)?;
|
||||
// The AutoMix beat grid runs at half this rate.
|
||||
let grid = analyze_automix_pcm(&decimate(&samples), check)?;
|
||||
if grid.bpm > 0.0 {
|
||||
tempos.push((grid.bpm, grid.confidence));
|
||||
}
|
||||
}
|
||||
Ok(spectra.finish(&tempos, windows.len()))
|
||||
}
|
||||
|
||||
/// Averages sample pairs, a gentle low-pass before halving the rate.
|
||||
fn decimate(samples: &[f64]) -> Vec<u8> {
|
||||
samples
|
||||
.as_chunks::<2>()
|
||||
.0
|
||||
.iter()
|
||||
.flat_map(|[first, second]| {
|
||||
let value = ((first + second) * 0.5 * 32768.0)
|
||||
.round()
|
||||
.clamp(-32768.0, 32767.0) as i16;
|
||||
value.to_le_bytes()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn mel(hz: f64) -> f64 {
|
||||
2595.0 * (1.0 + hz / 700.0).log10()
|
||||
}
|
||||
|
||||
fn hz(mel: f64) -> f64 {
|
||||
700.0 * (10f64.powf(mel / 2595.0) - 1.0)
|
||||
}
|
||||
|
||||
/// Triangular mel filters from 20 Hz to Nyquist, each as its first bin and
|
||||
/// the weights from there.
|
||||
fn mel_filters() -> Vec<(usize, Vec<f64>)> {
|
||||
let low = mel(20.0);
|
||||
let high = mel(AUDIO_FEATURES_SAMPLE_RATE as f64 / 2.0);
|
||||
let points: Vec<f64> = (0..MEL_BANDS + 2)
|
||||
.map(|index| hz(low + (high - low) * index as f64 / (MEL_BANDS + 1) as f64))
|
||||
.collect();
|
||||
points
|
||||
.windows(3)
|
||||
.map(|edges| {
|
||||
let (left, center, right) = (edges[0], edges[1], edges[2]);
|
||||
let first = ((left / BIN_HZ).ceil() as usize).max(1);
|
||||
let last = ((right / BIN_HZ).floor() as usize).min(BINS - 1);
|
||||
let weights: Vec<f64> = (first..=last.max(first))
|
||||
.map(|bin| {
|
||||
let frequency = bin as f64 * BIN_HZ;
|
||||
if frequency <= center {
|
||||
(frequency - left) / (center - left)
|
||||
} else {
|
||||
(right - frequency) / (right - center)
|
||||
}
|
||||
.max(0.0)
|
||||
})
|
||||
.collect();
|
||||
// A filter narrower than a bin still takes its nearest bin.
|
||||
if weights.iter().all(|weight| *weight == 0.0) {
|
||||
((center / BIN_HZ).round() as usize, vec![1.0])
|
||||
} else {
|
||||
(first, weights)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Statistics accumulated over the audible frames of every window.
|
||||
struct Spectra {
|
||||
fft: Radix2Fft,
|
||||
hann: Vec<f64>,
|
||||
mel: Vec<(usize, Vec<f64>)>,
|
||||
dct: Vec<[f64; MEL_BANDS]>,
|
||||
seconds: f64,
|
||||
onsets: usize,
|
||||
loudness: Vec<f64>,
|
||||
centroid: Vec<f64>,
|
||||
rolloff: f64,
|
||||
flatness: f64,
|
||||
flux: f64,
|
||||
bands: [f64; 3],
|
||||
mfcc: [f64; MFCC_COEFFICIENTS],
|
||||
mfcc_squares: [f64; MFCC_COEFFICIENTS],
|
||||
chroma: [f64; 12],
|
||||
}
|
||||
|
||||
impl Spectra {
|
||||
fn new() -> Self {
|
||||
let hann = (0..FRAME)
|
||||
.map(|index| {
|
||||
0.5 - 0.5 * (2.0 * std::f64::consts::PI * index as f64 / FRAME as f64).cos()
|
||||
})
|
||||
.collect();
|
||||
// Orthonormal DCT-II rows 1 to 13; row 0 is overall level.
|
||||
let dct = (1..=MFCC_COEFFICIENTS)
|
||||
.map(|row| {
|
||||
let mut coefficients = [0.0; MEL_BANDS];
|
||||
for (band, value) in coefficients.iter_mut().enumerate() {
|
||||
*value = (2.0 / MEL_BANDS as f64).sqrt()
|
||||
* (std::f64::consts::PI * row as f64 * (band as f64 + 0.5)
|
||||
/ MEL_BANDS as f64)
|
||||
.cos();
|
||||
}
|
||||
coefficients
|
||||
})
|
||||
.collect();
|
||||
Self {
|
||||
fft: Radix2Fft::new(FRAME),
|
||||
hann,
|
||||
mel: mel_filters(),
|
||||
dct,
|
||||
seconds: 0.0,
|
||||
onsets: 0,
|
||||
loudness: Vec::new(),
|
||||
centroid: Vec::new(),
|
||||
rolloff: 0.0,
|
||||
flatness: 0.0,
|
||||
flux: 0.0,
|
||||
bands: [0.0; 3],
|
||||
mfcc: [0.0; MFCC_COEFFICIENTS],
|
||||
mfcc_squares: [0.0; MFCC_COEFFICIENTS],
|
||||
chroma: [0.0; 12],
|
||||
}
|
||||
}
|
||||
|
||||
fn add_window(
|
||||
&mut self,
|
||||
samples: &[f64],
|
||||
check: &dyn Fn() -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
self.seconds += samples.len() as f64 / AUDIO_FEATURES_SAMPLE_RATE as f64;
|
||||
if samples.len() < FRAME {
|
||||
return Ok(());
|
||||
}
|
||||
let (mut re, mut im) = (vec![0.0; FRAME], vec![0.0; FRAME]);
|
||||
let mut power = vec![0.0; BINS];
|
||||
// Log-compressed magnitudes of this frame and the one before.
|
||||
let (mut current, mut previous) = (vec![0.0; BINS], vec![0.0; BINS]);
|
||||
let mut flux = Vec::new();
|
||||
for (index, start) in (0..=samples.len() - FRAME).step_by(HOP).enumerate() {
|
||||
if index % 32 == 0 {
|
||||
check()?;
|
||||
}
|
||||
let frame = &samples[start..start + FRAME];
|
||||
let rms = (frame.iter().map(|value| value * value).sum::<f64>() / FRAME as f64).sqrt();
|
||||
let db = 20.0 * rms.max(1e-9).log10();
|
||||
for ((real, imaginary), (value, weight)) in re
|
||||
.iter_mut()
|
||||
.zip(im.iter_mut())
|
||||
.zip(frame.iter().zip(&self.hann))
|
||||
{
|
||||
*real = value * weight;
|
||||
*imaginary = 0.0;
|
||||
}
|
||||
self.fft.transform(&mut re, &mut im);
|
||||
for (((bin, magnitude), real), imaginary) in
|
||||
power.iter_mut().zip(current.iter_mut()).zip(&re).zip(&im)
|
||||
{
|
||||
*bin = real * real + imaginary * imaginary;
|
||||
// About the amplitude of a sine at this bin, compressed.
|
||||
*magnitude = (1.0 + 100.0 * bin.sqrt() * 4.0 / FRAME as f64).ln();
|
||||
}
|
||||
let rise = current
|
||||
.iter()
|
||||
.zip(&previous)
|
||||
.map(|(now, before)| (now - before).max(0.0))
|
||||
.sum::<f64>()
|
||||
/ BINS as f64;
|
||||
std::mem::swap(&mut current, &mut previous);
|
||||
// The first frame rises from nothing.
|
||||
flux.push(if index == 0 { 0.0 } else { rise });
|
||||
if db < SILENT_DB {
|
||||
continue;
|
||||
}
|
||||
self.loudness.push(db);
|
||||
self.flux += rise;
|
||||
self.describe(&power);
|
||||
}
|
||||
self.onsets += count_onsets(&flux);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn describe(&mut self, power: &[f64]) {
|
||||
let total: f64 = power[1..].iter().sum();
|
||||
if total <= 1e-12 {
|
||||
return;
|
||||
}
|
||||
// Brightness weighs magnitudes, as power would let the bass decide.
|
||||
let magnitudes: f64 = power[1..].iter().map(|value| value.sqrt()).sum();
|
||||
let mut weighted = 0.0;
|
||||
let mut cumulative = 0.0;
|
||||
let mut rolloff = None;
|
||||
for (bin, value) in power.iter().enumerate().skip(1) {
|
||||
let frequency = bin as f64 * BIN_HZ;
|
||||
let magnitude = value.sqrt();
|
||||
weighted += frequency * magnitude;
|
||||
cumulative += magnitude;
|
||||
if rolloff.is_none() && cumulative >= 0.85 * magnitudes {
|
||||
rolloff = Some(frequency);
|
||||
}
|
||||
let band = if frequency < 250.0 {
|
||||
0
|
||||
} else if frequency < 4000.0 {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
self.bands[band] += value;
|
||||
}
|
||||
self.centroid.push(weighted / magnitudes);
|
||||
self.rolloff += rolloff.unwrap_or(0.0);
|
||||
let mean = total / (BINS - 1) as f64;
|
||||
let floor = mean * 1e-10;
|
||||
let log_mean = power[1..]
|
||||
.iter()
|
||||
.map(|value| (value + floor).ln())
|
||||
.sum::<f64>()
|
||||
/ (BINS - 1) as f64;
|
||||
self.flatness += (log_mean.exp() / mean).min(1.0);
|
||||
|
||||
let mut log_mel = [0.0; MEL_BANDS];
|
||||
for (value, (first, weights)) in log_mel.iter_mut().zip(&self.mel) {
|
||||
let energy: f64 = weights
|
||||
.iter()
|
||||
.zip(&power[*first..])
|
||||
.map(|(weight, bin)| weight * bin)
|
||||
.sum();
|
||||
*value = (energy + 1e-10).ln();
|
||||
}
|
||||
for ((sum, squares), row) in self
|
||||
.mfcc
|
||||
.iter_mut()
|
||||
.zip(self.mfcc_squares.iter_mut())
|
||||
.zip(&self.dct)
|
||||
{
|
||||
let coefficient: f64 = row.iter().zip(&log_mel).map(|(a, b)| a * b).sum();
|
||||
*sum += coefficient;
|
||||
*squares += coefficient * coefficient;
|
||||
}
|
||||
|
||||
// Pitch classes of the spectral peaks, their frequencies refined by
|
||||
// parabolic interpolation, between 80 Hz and 5 kHz.
|
||||
let peak = power.iter().copied().fold(0.0, f64::max);
|
||||
let mut chroma = [0.0; 12];
|
||||
for bin in ((80.0 / BIN_HZ) as usize).max(1)..((5000.0 / BIN_HZ) as usize).min(BINS - 1) {
|
||||
let value = power[bin];
|
||||
if value < peak * 1e-4 || value < power[bin - 1] || value <= power[bin + 1] {
|
||||
continue;
|
||||
}
|
||||
let (left, center, right) = (
|
||||
(power[bin - 1] + 1e-20).ln(),
|
||||
(value + 1e-20).ln(),
|
||||
(power[bin + 1] + 1e-20).ln(),
|
||||
);
|
||||
let curvature = left - 2.0 * center + right;
|
||||
let offset = if curvature < 0.0 {
|
||||
(0.5 * (left - right) / curvature).clamp(-0.5, 0.5)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let frequency = (bin as f64 + offset) * BIN_HZ;
|
||||
let note = 69.0 + 12.0 * (frequency / 440.0).log2();
|
||||
chroma[(note.round() as i64).rem_euclid(12) as usize] += value.sqrt();
|
||||
}
|
||||
let strongest = chroma.iter().copied().fold(0.0, f64::max);
|
||||
if strongest > 0.0 {
|
||||
for (sum, value) in self.chroma.iter_mut().zip(chroma) {
|
||||
*sum += value / strongest;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(self, tempos: &[(f64, f64)], windows: usize) -> Option<AudioFeatures> {
|
||||
let frames = self.loudness.len();
|
||||
if (frames as f64) * (HOP as f64) / (AUDIO_FEATURES_SAMPLE_RATE as f64)
|
||||
< MIN_AUDIBLE_SECONDS
|
||||
|| self.centroid.is_empty()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let count = self.centroid.len() as f64;
|
||||
let (loudness, loudness_spread) = mean_spread(&self.loudness);
|
||||
let (centroid, centroid_spread) = mean_spread(&self.centroid);
|
||||
let band_total = self.bands.iter().sum::<f64>().max(1e-12);
|
||||
let mut mfcc = [0.0; MFCC_COEFFICIENTS];
|
||||
let mut mfcc_spread = [0.0; MFCC_COEFFICIENTS];
|
||||
for (index, (mean, spread)) in mfcc.iter_mut().zip(mfcc_spread.iter_mut()).enumerate() {
|
||||
*mean = self.mfcc[index] / count;
|
||||
*spread = (self.mfcc_squares[index] / count - *mean * *mean)
|
||||
.max(0.0)
|
||||
.sqrt();
|
||||
}
|
||||
let (key, major, key_strength) = estimate_key(&self.chroma);
|
||||
let (tempo, tempo_confidence) = combine_tempos(tempos, windows);
|
||||
Some(AudioFeatures {
|
||||
tempo,
|
||||
tempo_confidence,
|
||||
loudness,
|
||||
loudness_spread,
|
||||
centroid,
|
||||
centroid_spread,
|
||||
rolloff: self.rolloff / count,
|
||||
flatness: self.flatness / count,
|
||||
onset_rate: self.onsets as f64 / self.seconds.max(1.0),
|
||||
flux: self.flux / frames as f64,
|
||||
bands: self.bands.map(|band| band / band_total),
|
||||
key,
|
||||
major,
|
||||
key_strength,
|
||||
mfcc,
|
||||
mfcc_spread,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn mean_spread(values: &[f64]) -> (f64, f64) {
|
||||
let count = values.len().max(1) as f64;
|
||||
let mean = values.iter().sum::<f64>() / count;
|
||||
let variance = values
|
||||
.iter()
|
||||
.map(|value| (value - mean) * (value - mean))
|
||||
.sum::<f64>()
|
||||
/ count;
|
||||
(mean, variance.sqrt())
|
||||
}
|
||||
|
||||
/// Local spectral flux peaks that stand out from their surroundings, at
|
||||
/// least four frames (93 ms) apart.
|
||||
fn count_onsets(flux: &[f64]) -> usize {
|
||||
const RADIUS: usize = 3;
|
||||
const CONTEXT: usize = 16;
|
||||
const FLOOR: f64 = 0.002;
|
||||
let mut onsets = 0;
|
||||
let mut last: Option<usize> = None;
|
||||
for (index, value) in flux.iter().enumerate() {
|
||||
if *value < FLOOR {
|
||||
continue;
|
||||
}
|
||||
let begin = index.saturating_sub(RADIUS);
|
||||
let end = (index + RADIUS + 1).min(flux.len());
|
||||
if flux[begin..end].iter().any(|other| other > value) {
|
||||
continue;
|
||||
}
|
||||
let begin = index.saturating_sub(CONTEXT);
|
||||
let end = (index + CONTEXT + 1).min(flux.len());
|
||||
let local = flux[begin..end].iter().sum::<f64>() / (end - begin) as f64;
|
||||
if *value < 1.5 * local || last.is_some_and(|before| index - before <= RADIUS) {
|
||||
continue;
|
||||
}
|
||||
onsets += 1;
|
||||
last = Some(index);
|
||||
}
|
||||
onsets
|
||||
}
|
||||
|
||||
/// The key whose Krumhansl-Kessler profile correlates best with [chroma].
|
||||
fn estimate_key(chroma: &[f64; 12]) -> (usize, bool, f64) {
|
||||
let correlate = |profile: &[f64; 12], tonic: usize| {
|
||||
let mean_chroma = chroma.iter().sum::<f64>() / 12.0;
|
||||
let mean_profile = profile.iter().sum::<f64>() / 12.0;
|
||||
let (mut dot, mut left, mut right) = (0.0, 0.0, 0.0);
|
||||
for (pitch, value) in chroma.iter().enumerate() {
|
||||
let a = value - mean_chroma;
|
||||
let b = profile[(pitch + 12 - tonic) % 12] - mean_profile;
|
||||
dot += a * b;
|
||||
left += a * a;
|
||||
right += b * b;
|
||||
}
|
||||
dot / (left * right).sqrt().max(1e-12)
|
||||
};
|
||||
let mut best = (0, true, f64::MIN);
|
||||
for tonic in 0..12 {
|
||||
for (major, profile) in [(true, &MAJOR_PROFILE), (false, &MINOR_PROFILE)] {
|
||||
let strength = correlate(profile, tonic);
|
||||
if strength > best.2 {
|
||||
best = (tonic, major, strength);
|
||||
}
|
||||
}
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// One tempo for the song: windows that agree reinforce each other; a
|
||||
/// disagreement keeps the surer one at reduced confidence.
|
||||
fn combine_tempos(tempos: &[(f64, f64)], windows: usize) -> (f64, f64) {
|
||||
match tempos {
|
||||
[] => (0.0, 0.0),
|
||||
[(bpm, confidence)] if windows > 1 => (*bpm, confidence * 0.8),
|
||||
[(bpm, confidence)] => (*bpm, *confidence),
|
||||
[first, second, ..] => {
|
||||
let (best, other) = if first.1 >= second.1 {
|
||||
(first, second)
|
||||
} else {
|
||||
(second, first)
|
||||
};
|
||||
if (other.0 / best.0 - 1.0).abs() < 0.03 {
|
||||
let weight = (best.1 + other.1).max(1e-9);
|
||||
((best.0 * best.1 + other.0 * other.1) / weight, best.1)
|
||||
} else {
|
||||
(best.0, best.1 * 0.8)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::cell::Cell;
|
||||
use std::f64::consts::PI;
|
||||
|
||||
const SECONDS: usize = 24;
|
||||
|
||||
fn pcm(samples: impl Iterator<Item = f64>) -> Vec<u8> {
|
||||
samples
|
||||
.flat_map(|value| (((value * 32767.0).round()) as i16).to_le_bytes())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn signal(mut value: impl FnMut(f64) -> f64) -> Vec<u8> {
|
||||
pcm((0..AUDIO_FEATURES_SAMPLE_RATE * SECONDS)
|
||||
.map(|index| value(index as f64 / AUDIO_FEATURES_SAMPLE_RATE as f64)))
|
||||
}
|
||||
|
||||
fn noise(seed: u32) -> impl FnMut() -> f64 {
|
||||
let mut state = seed;
|
||||
move || {
|
||||
state ^= state << 13;
|
||||
state ^= state >> 17;
|
||||
state ^= state << 5;
|
||||
f64::from(state) / f64::from(u32::MAX) * 2.0 - 1.0
|
||||
}
|
||||
}
|
||||
|
||||
fn describe(window: &[u8]) -> AudioFeatures {
|
||||
analyze_audio_features(&[window, window], &|| Ok(()))
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn sine(frequency: f64, amplitude: f64) -> Vec<u8> {
|
||||
signal(|time| amplitude * (2.0 * PI * frequency * time).sin())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn click_track_gives_its_tempo_and_onsets() {
|
||||
let mut next = noise(7);
|
||||
let bursts: Vec<f64> = (0..AUDIO_FEATURES_SAMPLE_RATE * SECONDS)
|
||||
.map(|_| next())
|
||||
.collect();
|
||||
let clicks = pcm(bursts.iter().enumerate().map(|(index, value)| {
|
||||
let time = index as f64 / AUDIO_FEATURES_SAMPLE_RATE as f64;
|
||||
let since = (time - 0.1).rem_euclid(0.5);
|
||||
if time < 0.1 {
|
||||
0.0
|
||||
} else {
|
||||
0.8 * (-since * 60.0).exp() * value
|
||||
}
|
||||
}));
|
||||
let features = describe(&clicks);
|
||||
assert!((features.tempo - 120.0).abs() < 1.5, "{}", features.tempo);
|
||||
assert!(
|
||||
features.tempo_confidence > 0.5,
|
||||
"{}",
|
||||
features.tempo_confidence
|
||||
);
|
||||
assert!(
|
||||
(features.onset_rate - 2.0).abs() < 0.4,
|
||||
"{}",
|
||||
features.onset_rate
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tone_and_noise_differ_in_brightness_and_flatness() {
|
||||
let tone = describe(&sine(1000.0, 0.5));
|
||||
let mut next = noise(3);
|
||||
let hiss = describe(&signal(|_| 0.5 * next()));
|
||||
assert!((tone.centroid - 1000.0).abs() < 100.0, "{}", tone.centroid);
|
||||
assert!(hiss.centroid > 4000.0, "{}", hiss.centroid);
|
||||
assert!(hiss.rolloff > tone.rolloff + 4000.0);
|
||||
assert!(tone.flatness < 0.05, "{}", tone.flatness);
|
||||
assert!(hiss.flatness > 0.3, "{}", hiss.flatness);
|
||||
assert!(tone.bands[1] > 0.95);
|
||||
assert!(hiss.bands[2] > 0.5);
|
||||
assert!(tone.onset_rate < 0.2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quiet_and_loud_differ_in_loudness() {
|
||||
let tone = describe(&sine(440.0, 0.5));
|
||||
assert!((tone.loudness + 9.03).abs() < 0.5, "{}", tone.loudness);
|
||||
assert!(tone.loudness_spread < 0.5);
|
||||
let mut next = noise(5);
|
||||
let hiss: Vec<f64> = (0..AUDIO_FEATURES_SAMPLE_RATE * SECONDS)
|
||||
.map(|_| next())
|
||||
.collect();
|
||||
let quiet = describe(&pcm(hiss.iter().map(|value| 0.05 * value)));
|
||||
let loud = describe(&pcm(hiss.iter().map(|value| 0.5 * value)));
|
||||
assert!((loud.loudness - quiet.loudness - 20.0).abs() < 0.5);
|
||||
// Timbre does not depend on level.
|
||||
for (a, b) in quiet.mfcc.iter().zip(&loud.mfcc) {
|
||||
assert!((a - b).abs() < 0.1, "{a} {b}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn triads_give_their_keys() {
|
||||
let triad = |notes: [f64; 3]| {
|
||||
describe(&signal(|time| {
|
||||
notes
|
||||
.iter()
|
||||
.map(|note| {
|
||||
let frequency = 440.0 * 2f64.powf((note - 69.0) / 12.0);
|
||||
0.2 * (2.0 * PI * frequency * time).sin()
|
||||
})
|
||||
.sum()
|
||||
}))
|
||||
};
|
||||
let c_major = triad([60.0, 64.0, 67.0]);
|
||||
assert_eq!((c_major.key, c_major.major), (0, true));
|
||||
assert!(c_major.key_strength > 0.5);
|
||||
let a_minor = triad([57.0, 60.0, 64.0]);
|
||||
assert_eq!((a_minor.key, a_minor.major), (9, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn silence_and_short_windows_describe_nothing() {
|
||||
let silence = vec![0; AUDIO_FEATURES_SAMPLE_RATE * 2 * SECONDS];
|
||||
assert_eq!(
|
||||
analyze_audio_features(&[&silence], &|| Ok(())).unwrap(),
|
||||
None
|
||||
);
|
||||
let short = &sine(440.0, 0.5)[..AUDIO_FEATURES_SAMPLE_RATE * 2];
|
||||
assert_eq!(analyze_audio_features(&[short], &|| Ok(())).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_job_is_bounded_and_cancelable() {
|
||||
let tone = sine(440.0, 0.5);
|
||||
let features = analyze_audio_features(&[&tone], &|| Ok(()))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let vector = features.to_vector();
|
||||
assert_eq!(vector.len(), AudioFeatures::LEN);
|
||||
assert!(vector.iter().all(|value| value.is_finite()));
|
||||
|
||||
assert!(analyze_audio_features(&[], &|| Ok(())).is_err());
|
||||
assert!(analyze_audio_features(&[&tone, &tone, &tone], &|| Ok(())).is_err());
|
||||
let oversized = vec![0; MAX_WINDOW_BYTES + 2];
|
||||
assert!(analyze_audio_features(&[&oversized], &|| Ok(())).is_err());
|
||||
assert!(analyze_audio_features_files(&[], &|| Ok(())).is_err());
|
||||
|
||||
let calls = Cell::new(0);
|
||||
let result = analyze_audio_features(&[&tone, &tone], &|| {
|
||||
calls.set(calls.get() + 1);
|
||||
if calls.get() >= 20 {
|
||||
Err("cancelled".into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
});
|
||||
assert_eq!(result.unwrap_err(), "cancelled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn windows_that_disagree_on_tempo_lower_its_confidence() {
|
||||
assert_eq!(combine_tempos(&[], 2), (0.0, 0.0));
|
||||
assert_eq!(combine_tempos(&[(120.0, 0.8)], 1), (120.0, 0.8));
|
||||
let (tempo, confidence) = combine_tempos(&[(120.0, 0.8), (121.0, 0.4)], 2);
|
||||
assert!((tempo - 120.0 - 1.0 / 3.0).abs() < 1e-9);
|
||||
assert_eq!(confidence, 0.8);
|
||||
let (tempo, confidence) = combine_tempos(&[(90.0, 0.4), (128.0, 0.6)], 2);
|
||||
assert_eq!(tempo, 128.0);
|
||||
assert!((confidence - 0.48).abs() < 1e-9);
|
||||
}
|
||||
}
|
||||
@@ -98,6 +98,21 @@ fn execute(
|
||||
)?;
|
||||
serde_json::to_value(grid).map_err(|error| error.to_string())
|
||||
}
|
||||
"audio_features" => {
|
||||
use mornye_core::native_audio_features as features;
|
||||
let paths = request
|
||||
.get("paths")
|
||||
.and_then(Value::as_array)
|
||||
.ok_or("Missing native data field: paths")?
|
||||
.iter()
|
||||
.map(|path| path.as_str().ok_or("Invalid audio feature window path"))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let described = features::analyze_audio_features_files(&paths, check)?;
|
||||
Ok(json!({
|
||||
"version": features::AUDIO_FEATURES_VERSION,
|
||||
"features": described.map(|value| value.to_vector()),
|
||||
}))
|
||||
}
|
||||
"spectral_cutoff" => {
|
||||
let width = request
|
||||
.get("width")
|
||||
@@ -197,4 +212,36 @@ mod tests {
|
||||
);
|
||||
assert!(hash_file(file.path().to_str().unwrap(), &|| Err("cancelled".into())).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn describes_decoded_windows_and_rejects_bad_requests() {
|
||||
let mut tone = tempfile::NamedTempFile::new().unwrap();
|
||||
for index in 0..22_050 * 8 {
|
||||
let value = 0.5 * (index as f64 * 440.0 * std::f64::consts::TAU / 22_050.0).sin();
|
||||
tone.write_all(&(((value * 32767.0) as i16).to_le_bytes()))
|
||||
.unwrap();
|
||||
}
|
||||
let mut silence = tempfile::NamedTempFile::new().unwrap();
|
||||
silence.write_all(&[0; 22_050 * 2 * 8]).unwrap();
|
||||
let run = |paths: Value| {
|
||||
execute(
|
||||
&json!({"operation": "audio_features", "paths": paths}),
|
||||
&[],
|
||||
&|| Ok(()),
|
||||
None,
|
||||
)
|
||||
};
|
||||
let tone_path = tone.path().to_str().unwrap();
|
||||
let described = run(json!([tone_path, tone_path])).unwrap();
|
||||
assert_eq!(described["version"], 1);
|
||||
assert_eq!(
|
||||
described["features"].as_array().unwrap().len(),
|
||||
mornye_core::native_audio_features::AudioFeatures::LEN
|
||||
);
|
||||
let silent = run(json!([silence.path().to_str().unwrap()])).unwrap();
|
||||
assert!(silent["features"].is_null());
|
||||
assert!(run(json!([])).is_err());
|
||||
assert!(run(json!([1])).is_err());
|
||||
assert!(run(json!(["/missing/window.pcm"])).is_err());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user