perf(media): shrink FFmpeg while preserving HTTPS motion artwork

This commit is contained in:
zarzet committed 2026-10-04 01:35:48 +07:00
1 parent e8c635eda0
commit 541bfc4002
17 files changed
+999 -20

No files matched your search

+1 -1
View File
@@ -271,7 +271,7 @@ jobs:
- name: Normalize ffmpeg plugin shell scripts (strip CRLF)
run: |
find "$HOME/.pub-cache/hosted" -path "*ffmpeg_kit_flutter_new_full*/scripts/*.sh" -type f -print0 |
find "$HOME/.pub-cache/hosted" -path "*ffmpeg_kit_flutter_new_audio*/scripts/*.sh" -type f -print0 |
while IFS= read -r -d '' f; do
perl -pi -e 's/\r$//' "$f"
chmod +x "$f"
+2
View File
@@ -67,6 +67,8 @@ Gradle builds the Rust native artifacts automatically. Tool versions are pinned
Start with the [Contributing Guide](CONTRIBUTING.md) for the development setup,
project boundaries, validation commands, and pull request checklist.
For native iOS codec checks, see the [FFmpeg capability probe](scripts/README_ios_ffmpeg_capabilities.md).
---
## Related Projects
+1 -1
View File
@@ -183,7 +183,7 @@ dependencies {
implementation("androidx.activity:activity-ktx:1.13.0")
// NativeDownloadFinalizer imports FFmpegKit APIs directly. The Flutter
// plugin owns the runtime AAR; compileOnly avoids packaging it twice here.
compileOnly("com.antonkarpenko:ffmpeg-kit-full:2.2.1")
compileOnly("com.antonkarpenko:ffmpeg-kit-audio:2.2.2")
testImplementation("junit:junit:4.13.2")
testImplementation("org.json:json:20260814")
@@ -8,3 +8,20 @@ The DSF input stores those bytes with reversed bits (LSB-first).
This tests lossless WavPack DSD decoding, channel order, seek alignment, native
LE/BE packing and DoP packing against known bytes. It is packaged only in the
instrumentation test APK.
## Synthetic motion artwork fixtures
`motion-probe-h264.mp4` and `motion-probe-hevc.mp4` contain one second of the
FFmpeg `testsrc2` pattern at 32x32 pixels and two frames per second. They are
original generated test data and are packaged only in the instrumentation APK.
The capability test validates decoding and MP4 stream-copy after reducing the
shipped FFmpeg dependency. Generate with `-f lavfi -i testsrc2=size=32x32:rate=2
-t 1 -an -pix_fmt yuv420p -movflags +faststart`, using `-c:v libx264` or
`-c:v libx265 -x265-params log-level=error:pools=1 -tag:v hvc1` respectively.
For a live encrypted HLS gate through `MotionArtworkProxy`, forward its local
port with `adb reverse tcp:PORT tcp:PORT` and pass
`-e motionProxyUrl http://127.0.0.1:PORT/TOKEN/0.m3u8` to the instrumentation
runner for `com.zarz.spotiflac.FFmpegCapabilitiesTest`. The optional gate decodes
the input, remuxes it, then decodes the resulting MP4 using the shipped native
library. Normal fixture tests run without a server or this argument.
@@ -0,0 +1,75 @@
package com.zarz.spotiflac
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import java.io.File
import org.junit.Assert.assertTrue
import org.junit.Test
import org.junit.runner.RunWith
/** Exercise shipped libraries rather than the host's unrelated FFmpeg build. */
@RunWith(AndroidJUnit4::class)
class FFmpegCapabilitiesTest {
private fun execute(vararg arguments: String) {
val result = NativeDownloadFinalizer.runFFmpegArguments(arrayOf("-v", "error", "-y", *arguments))
assertTrue(result.second, result.first)
}
@Test
fun audioEncodersAnalysisAndCoverFiltersRemainAvailable() {
val context = InstrumentationRegistry.getInstrumentation().targetContext
val root = File(context.cacheDir, "ffmpeg-capabilities-${System.nanoTime()}").apply { mkdirs() }
try {
for ((extension, encoder) in listOf(
"flac" to "flac", "m4a" to "alac", "wav" to "pcm_s24le",
"aiff" to "pcm_s24be", "mp3" to "libmp3lame", "opus" to "libopus", "aac" to "aac",
)) {
val output = File(root, "track.$extension")
execute("-f", "lavfi", "-i", "sine=frequency=997:sample_rate=48000", "-t", "0.5", "-c:a", encoder, output.path)
execute("-i", output.path, "-map", "0:a:0", "-f", "null", "-")
assertTrue(output.length() > 0)
}
val audio = File(root, "track.flac")
val stats = File(root, "analysis.txt")
execute("-i", audio.path, "-af", "astats=metadata=1:reset=0,ebur128=peak=true:metadata=1,ametadata=print:file='${stats.path}'", "-f", "null", "-")
assertTrue(stats.readText().contains("lavfi.astats"))
val spectrum = File(root, "spectrum.png")
execute("-i", audio.path, "-lavfi", "showspectrumpic=s=64x64:legend=0:mode=combined:color=channel:scale=log:fscale=lin:win_func=hann", "-frames:v", "1", spectrum.path)
val cover = File(root, "cover.jpg")
execute("-i", spectrum.path, "-vf", "scale=32:32", "-frames:v", "1", cover.path)
assertTrue(cover.length() > 0)
} finally { root.deleteRecursively() }
}
@Test
fun motionArtworkDecodingAndRemuxRemainAvailable() {
val instrumentation = InstrumentationRegistry.getInstrumentation()
val root = File(instrumentation.targetContext.cacheDir, "ffmpeg-motion-${System.nanoTime()}").apply { mkdirs() }
try {
for (codec in listOf("h264", "hevc")) {
val input = File(root, "$codec.mp4")
instrumentation.context.assets.open("motion-probe-$codec.mp4").use { source ->
input.outputStream().use { source.copyTo(it) }
}
execute("-xerror", "-err_detect", "explode", "-i", input.path, "-map", "0:v:0", "-an", "-sn", "-dn", "-f", "null", "-")
val output = File(root, "$codec-remux.mp4")
execute("-i", input.path, "-map", "0:v:0", "-an", "-c:v", "copy", "-movflags", "+faststart", output.path)
assertTrue(output.length() > 0)
}
// Optional live Dart HTTPS/HLS proxy gate, forwarded with adb reverse.
// The local codec fixtures above remain independent of this server.
InstrumentationRegistry.getArguments().getString("motionProxyUrl")?.let { source ->
val output = File(root, "proxy-remux.mp4")
val input = arrayOf(
"-protocol_whitelist", "file,http,https,tcp,tls,crypto",
"-rw_timeout", "10000000", "-http_multiple", "0",
"-http_persistent", "0", "-extension_picky", "0", "-i", source,
)
execute("-xerror", "-err_detect", "explode", *input, "-map", "0:v:0", "-an", "-f", "null", "-")
execute(*input, "-map", "0:v:0", "-an", "-c:v", "copy", "-movflags", "+faststart", output.path)
assertTrue(output.length() > 0)
execute("-xerror", "-err_detect", "explode", "-i", output.path, "-map", "0:v:0", "-an", "-f", "null", "-")
}
} finally { root.deleteRecursively() }
}
}
+2 -2
View File
@@ -1,8 +1,8 @@
import 'dart:async';
import 'dart:io';
import 'package:ffmpeg_kit_flutter_new_full/ffmpeg_kit.dart';
import 'package:ffmpeg_kit_flutter_new_full/return_code.dart';
import 'package:ffmpeg_kit_flutter_new_audio/ffmpeg_kit.dart';
import 'package:ffmpeg_kit_flutter_new_audio/return_code.dart';
import 'package:flutter/foundation.dart';
import 'package:path_provider/path_provider.dart';
import 'package:spotiflac_android/services/audio_analysis_jobs.dart';
+3 -3
View File
@@ -2,9 +2,9 @@ import 'dart:convert';
import 'dart:io';
import 'dart:math' as math;
import 'package:flutter/foundation.dart';
import 'package:ffmpeg_kit_flutter_new_full/ffmpeg_kit.dart';
import 'package:ffmpeg_kit_flutter_new_full/ffprobe_kit.dart';
import 'package:ffmpeg_kit_flutter_new_full/return_code.dart';
import 'package:ffmpeg_kit_flutter_new_audio/ffmpeg_kit.dart';
import 'package:ffmpeg_kit_flutter_new_audio/ffprobe_kit.dart';
import 'package:ffmpeg_kit_flutter_new_audio/return_code.dart';
import 'package:path_provider/path_provider.dart';
import 'package:spotiflac_android/services/audio_metadata_mapper.dart';
import 'package:spotiflac_android/services/ffmpeg_models.dart';
+293
View File
@@ -0,0 +1,293 @@
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:math';
/// Dart handles upstream TLS; FFmpeg sees only a short-lived loopback URL.
/// Only the initial resource and URLs discovered in its HLS playlists can be
/// requested, with a shared transfer budget across playlists, keys and ranges.
class MotionArtworkProxy {
MotionArtworkProxy._(this._server, this._client, this._maxBytes);
final HttpServer _server;
final HttpClient _client;
final int _maxBytes;
final _resources = <String, Uri>{};
final _paths = <Uri, String>{};
final String _token = List.generate(
24,
(_) => Random.secure().nextInt(256).toRadixString(16).padLeft(2, '0'),
).join();
late final Timer _deadline;
late final Uri source;
int _transferred = 0;
bool _closed = false;
bool _hlsDetected = false;
Future<void>? _closing;
bool get isClosed => _closed;
bool get isHls => _hlsDetected || source.path.endsWith('.m3u8');
static Future<MotionArtworkProxy> start(
Uri source, {
HttpClient? client,
int maxBytes = 24 << 20,
Duration lifetime = const Duration(seconds: 30),
}) async {
_checkRemote(source);
final upstream = client ?? HttpClient();
upstream
..connectionTimeout = const Duration(seconds: 8)
..autoUncompress = false;
try {
final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
final proxy = MotionArtworkProxy._(server, upstream, maxBytes);
proxy.source = proxy._register(source);
proxy._deadline = Timer(lifetime, () => unawaited(proxy.close()));
server.listen(proxy._handle, onError: (Object _) {});
return proxy;
} catch (_) {
upstream.close(force: true);
rethrow;
}
}
Future<void> close() => _closing ??= _close();
Future<void> _close() async {
_closed = true;
_deadline.cancel();
_client.close(force: true);
await _server.close(force: true);
}
static void _checkRemote(Uri uri) {
if (!['http', 'https'].contains(uri.scheme) ||
uri.host.isEmpty ||
uri.toString().length > 8192) {
throw const FormatException('Artwork resource requires HTTP or HTTPS');
}
}
Uri _register(Uri remote) {
_checkRemote(remote);
remote = remote.removeFragment();
final existing = _paths[remote];
if (existing != null) return _local(existing);
if (_resources.length >= 4096 || remote.toString().length > 8192) {
throw const FormatException('Too many artwork resources');
}
// HLS validates segment extensions before opening them; retain the
// upstream suffix while hiding its URL/query behind the resource token.
final extension =
RegExp(r'\.[a-z0-9]{1,10}$').stringMatch(remote.path.toLowerCase()) ??
'.bin';
final path = '/$_token/${_resources.length}$extension';
_resources[path] = remote;
_paths[remote] = path;
return _local(path);
}
Uri _local(String path) => Uri(
scheme: 'http',
host: InternetAddress.loopbackIPv4.address,
port: _server.port,
path: path,
);
Future<({HttpClientResponse response, Uri uri})> _request(
Uri uri,
HttpRequest incoming,
) async {
for (var redirects = 0; redirects <= 5; redirects++) {
_checkRemote(uri);
final request = await _client
.openUrl(incoming.method, uri)
.timeout(const Duration(seconds: 8));
request.followRedirects = false;
request.headers.set(HttpHeaders.acceptEncodingHeader, 'identity');
for (final name in [HttpHeaders.rangeHeader, HttpHeaders.ifRangeHeader]) {
final value = incoming.headers.value(name);
if (value != null) request.headers.set(name, value);
}
final response = await request.close().timeout(
const Duration(seconds: 8),
);
if (!response.isRedirect) return (response: response, uri: uri);
final location = response.headers.value(HttpHeaders.locationHeader);
await response.listen(null).cancel();
if (location == null) throw const HttpException('Missing redirect');
uri = uri.resolve(location);
}
throw const HttpException('Too many artwork redirects');
}
Stream<List<int>> _body(HttpClientResponse response) async* {
await for (final chunk in response.timeout(const Duration(seconds: 8))) {
if (_closed || _transferred + chunk.length > _maxBytes) {
unawaited(close());
throw const HttpException('Artwork transfer budget exceeded');
}
_transferred += chunk.length;
yield chunk;
}
}
Future<void> _handle(HttpRequest request) async {
final outgoing = request.response;
outgoing.persistentConnection = false;
outgoing.bufferOutput = false;
try {
final remote = _resources[request.uri.path];
if (_closed || remote == null || request.uri.hasQuery) {
outgoing.statusCode = HttpStatus.notFound;
return;
}
if (request.method != 'GET' && request.method != 'HEAD') {
outgoing.statusCode = HttpStatus.methodNotAllowed;
return;
}
final upstream = await _request(remote, request);
final response = upstream.response;
outgoing.statusCode = response.statusCode;
for (final name in [
HttpHeaders.contentTypeHeader,
HttpHeaders.contentRangeHeader,
HttpHeaders.acceptRangesHeader,
HttpHeaders.contentEncodingHeader,
]) {
final value = response.headers.value(name);
if (value != null) outgoing.headers.set(name, value);
}
if (request.method == 'HEAD') {
if (response.contentLength >= 0) {
outgoing.contentLength = response.contentLength;
}
await response.drain<void>();
return;
}
final body = StreamIterator(_body(response));
try {
final prefix = <int>[];
while (prefix.length < 7 && await body.moveNext()) {
prefix.addAll(body.current);
}
final playlist =
response.statusCode >= 200 &&
response.statusCode < 300 &&
(upstream.uri.path.toLowerCase().endsWith('.m3u8') ||
(response.headers.contentType?.mimeType.contains('mpegurl') ??
false) ||
utf8
.decode(prefix.take(32).toList(), allowMalformed: true)
.trimLeft()
.startsWith('#EXTM3U'));
if (playlist) {
_hlsDetected = true;
final bytes = <int>[...prefix];
while (await body.moveNext()) {
if (bytes.length + body.current.length > 512 << 10) {
throw const FormatException(
'Artwork playlist exceeds size limit',
);
}
bytes.addAll(body.current);
}
if (bytes.length > 512 << 10) {
throw const FormatException('Artwork playlist exceeds size limit');
}
final text = await _playlistText(bytes, response.headers);
final rewritten = utf8.encode(_rewrite(text, upstream.uri));
outgoing
..statusCode = HttpStatus.ok
..headers.removeAll(HttpHeaders.contentRangeHeader)
..headers.removeAll(HttpHeaders.contentEncodingHeader)
..headers.contentType = ContentType(
'application',
'vnd.apple.mpegurl',
)
..contentLength = rewritten.length
..add(rewritten);
} else {
if (response.contentLength >= 0) {
outgoing.contentLength = response.contentLength;
}
outgoing.add(prefix);
await outgoing.flush();
while (await body.moveNext()) {
outgoing.add(body.current);
await outgoing.flush();
}
}
} finally {
await body.cancel();
}
} catch (_) {
try {
outgoing.statusCode = HttpStatus.badGateway;
} catch (_) {
// A stream failure can happen after binary response headers were sent.
}
} finally {
try {
await outgoing.close();
} catch (_) {}
}
}
Future<String> _playlistText(List<int> bytes, HttpHeaders headers) async {
Stream<List<int>> stream = Stream.value(bytes);
switch (headers.value(HttpHeaders.contentEncodingHeader)?.toLowerCase()) {
case 'gzip':
stream = gzip.decoder.bind(stream);
case 'deflate':
stream = zlib.decoder.bind(stream);
case null || 'identity':
break;
default:
throw const FormatException('Unsupported artwork playlist encoding');
}
final decoded = <int>[];
await for (final chunk in stream) {
if (decoded.length + chunk.length > 512 << 10) {
throw const FormatException('Artwork playlist exceeds size limit');
}
decoded.addAll(chunk);
}
return utf8.decode(decoded);
}
String _rewrite(String text, Uri base) => const LineSplitter()
.convert(text)
.map((raw) {
final line = raw.trim();
if (line.isEmpty) return raw;
if (!line.startsWith('#')) {
return _register(base.resolve(line)).toString();
}
return raw.replaceAllMapped(RegExp(r'([:,])URI="([^"]+)"'), (match) {
return '${match[1]}URI="${_register(base.resolve(match[2]!))}"';
});
})
.join('\n');
}
Future<T> withMotionArtworkProxy<T>(
Uri source,
Future<T> Function(Uri source) action, {
HttpClient? client,
int maxBytes = 24 << 20,
Duration lifetime = const Duration(seconds: 30),
}) async {
final proxy = await MotionArtworkProxy.start(
source,
client: client,
maxBytes: maxBytes,
lifetime: lifetime,
);
try {
return await action(proxy.source);
} finally {
await proxy.close();
}
}
+36 -5
View File
@@ -3,11 +3,12 @@ import 'dart:convert';
import 'dart:io';
import 'package:crypto/crypto.dart';
import 'package:ffmpeg_kit_flutter_new_full/ffmpeg_kit.dart';
import 'package:ffmpeg_kit_flutter_new_full/return_code.dart';
import 'package:ffmpeg_kit_flutter_new_audio/ffmpeg_kit.dart';
import 'package:ffmpeg_kit_flutter_new_audio/return_code.dart';
import 'package:path_provider/path_provider.dart';
import 'package:spotiflac_android/services/ffmpeg_service.dart';
import 'package:spotiflac_android/services/motion_artwork_download.dart';
import 'package:spotiflac_android/services/motion_artwork_proxy.dart';
import 'package:spotiflac_android/utils/logger.dart';
import 'package:spotiflac_android/utils/string_utils.dart';
@@ -183,13 +184,39 @@ class MotionArtworkStore {
'$output.source.mp4',
);
try {
return await _remuxVideo(original?.path ?? source, output);
if (original != null) return await _remuxVideo(original.path, output);
final proxy = await MotionArtworkProxy.start(Uri.parse(source));
try {
final hls = proxy.isHls;
final ratio = await _remuxVideo(
proxy.source.toString(),
output,
hlsInput: hls,
);
// Opaque or redirected URLs reveal HLS only after the first request.
// Retry failed HLS input with its scoped demuxer options, without
// passing HLS-only flags to ordinary MP4 inputs or buffering video.
if (ratio == null && !hls && proxy.isHls && !proxy.isClosed) {
return await _remuxVideo(
proxy.source.toString(),
output,
hlsInput: true,
);
}
return ratio;
} finally {
await proxy.close();
}
} finally {
if (original != null && await original.exists()) await original.delete();
}
}
static Future<double?> _remuxVideo(String source, String output) async {
static Future<double?> _remuxVideo(
String source,
String output, {
bool hlsInput = false,
}) async {
// Remux public video/HLS into a self-contained, silent MP4. Bound both
// transfer time and output size; never let optional artwork block audio.
final success = await _run([
@@ -200,7 +227,11 @@ class MotionArtworkStore {
'10000000',
// HLS byte ranges can share one MP4 URL. Reusing/prefetching HTTP
// connections can splice the next range into an unfinished segment.
if (Uri.parse(source).path.toLowerCase().endsWith('.m3u8')) ...[
if (hlsInput) ...[
// Tokenized URLs retain normal suffixes, but public HLS sources can
// use opaque segment paths. Only discovered HTTP resources are served.
'-extension_picky',
'0',
'-http_multiple',
'0',
'-http_persistent',
+4 -4
View File
@@ -4,10 +4,10 @@ import 'dart:io';
import 'dart:math' as math;
import 'dart:typed_data';
import 'dart:ui' as ui;
import 'package:ffmpeg_kit_flutter_new_full/ffmpeg_kit.dart';
import 'package:ffmpeg_kit_flutter_new_full/ffmpeg_session.dart';
import 'package:ffmpeg_kit_flutter_new_full/ffprobe_kit.dart';
import 'package:ffmpeg_kit_flutter_new_full/return_code.dart';
import 'package:ffmpeg_kit_flutter_new_audio/ffmpeg_kit.dart';
import 'package:ffmpeg_kit_flutter_new_audio/ffmpeg_session.dart';
import 'package:ffmpeg_kit_flutter_new_audio/ffprobe_kit.dart';
import 'package:ffmpeg_kit_flutter_new_audio/return_code.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:spotiflac_android/theme/mornye_theme.dart';
+3 -3
View File
@@ -431,11 +431,11 @@ packages:
url: "https://pub.dev"
source: hosted
version: "0.1.2"
ffmpeg_kit_flutter_new_full:
ffmpeg_kit_flutter_new_audio:
dependency: "direct main"
description:
name: ffmpeg_kit_flutter_new_full
sha256: "0f610062616b70b963a9f3b47675243969a044c570db1620f1fd9f4f0b85eb7b"
name: ffmpeg_kit_flutter_new_audio
sha256: "79e51732fa9bb5e88992ffdc84cbb78babbb44a773cd1f75f11290b295959c71"
url: "https://pub.dev"
source: hosted
version: "2.5.2"
+1 -1
View File
@@ -68,7 +68,7 @@ dependencies:
logger: ^2.7.0
# FFmpeg for audio conversion
ffmpeg_kit_flutter_new_full: ^2.5.2
ffmpeg_kit_flutter_new_audio: 2.5.2
open_filex: ^4.7.0
video_player: ^2.14.0
+58
View File
@@ -0,0 +1,58 @@
# iOS FFmpeg capability probe
`scripts/ios_ffmpeg_capabilities.m` runs the shipped simulator frameworks without
building the Flutter app. It checks FLAC, ALAC, 24-bit WAV/AIFF, MP3, Opus and AAC
encoding/decoding; analysis filters; spectrum PNG/JPEG conversion; and H.264/HEVC
motion decoding and stream-copy remuxing. This complements application tests: it
does not prove Flutter routing, download finalization or the Dart TLS proxy.
Prepare the audio plugin's XCFrameworks with its provided `scripts/setup_ios.sh`
from the plugin's `ios` directory. Boot an iOS simulator, then run the following
from the repository root. Framework copies are signed in a temporary directory;
the downloaded vendor artifacts remain unchanged.
```bash
probe_plugin="$HOME/.pub-cache/hosted/pub.dev/ffmpeg_kit_flutter_new_audio-2.5.2"
probe_work=$(mktemp -d /tmp/spotiflac-ios-ffmpeg.XXXXXX)
mkdir -p "$probe_work/Frameworks"
for framework in "$probe_plugin"/ios/Frameworks/*.xcframework; do
name=$(basename "$framework" .xcframework)
cp -R "$framework/ios-arm64_x86_64-simulator/$name.framework" "$probe_work/Frameworks/"
codesign --force --sign - "$probe_work/Frameworks/$name.framework"
done
xcrun --sdk iphonesimulator clang -fobjc-arc -target arm64-apple-ios16.0-simulator \
-isysroot "$(xcrun --sdk iphonesimulator --show-sdk-path)" \
-F "$probe_work/Frameworks" -framework Foundation -framework ffmpegkit \
-Wl,-rpath,"$probe_work/Frameworks" scripts/ios_ffmpeg_capabilities.m \
-o "$probe_work/probe"
codesign --force --sign - "$probe_work/probe"
xcrun simctl spawn booted "$probe_work/probe" "$probe_work/results" \
"$PWD/android/app/src/androidTest/assets"
```
The audio package exposes HTTP but has no native HTTPS/TLS support. Remote motion
input requires the app's scoped Dart TLS proxy. To exercise remote input, append
a fixture URL as the last probe argument; use the proxy's HTTP loopback URL when
testing the application route. Passing a direct HTTPS URL intentionally reports
a capability failure on this package. Run the proxy's Dart tests and Android/iOS
application checks before claiming that remote motion playback is preserved.
Remote checks decode the complete fixture, stream-copy it to MP4 with faststart,
and decode the resulting MP4 with strict error handling. The probe also prints
the shipped HLS demuxer's options. For HLS fixtures with opaque segment URLs,
append `--disable-hls-extension-check` after the proxy URL; this passes
`-extension_picky 0` only for that input. The other HLS flags in the usage message
are diagnostic modes for comparing the native extension checks.
Verified on the shipped iOS simulator audio frameworks:
| HTTPS-origin fixture through the Dart proxy | Input options | Decode/remux/decode |
| --- | --- | --- |
| Redirected master playlist, AES-128 key, three TS segments | Defaults | Pass |
| Same content with `.bin`, extensionless and `.ts` segment URLs | Defaults | Rejects segment extension |
| Same opaque segment URLs | `-allowed_extensions ALL` | Rejects segment extension |
| Same opaque segment URLs | Both extension lists set to `ALL` | Rejects format/extension mismatch |
| Same opaque segment URLs | `-extension_picky 0` | Pass |
The app uses the final option for scoped HLS proxy input. The proxy still owns
the upstream URL allowlist, TLS requests, transfer budget and cancellation.
+118
View File
@@ -0,0 +1,118 @@
#import <Foundation/Foundation.h>
#import <ffmpegkit/FFmpegKit.h>
#import <ffmpegkit/FFmpegKitConfig.h>
#import <ffmpegkit/ReturnCode.h>
// Standalone simulator probe for the shipped audio frameworks. Pass a writable
// output directory, android/app/src/androidTest/assets, and optionally a remote
// motion fixture URL. The audio package requires the app's Dart TLS proxy for
// HTTPS inputs; pass its HTTP loopback URL to verify that application route.
static BOOL execute(NSArray<NSString *> *arguments) {
NSArray *command = [@[@"-v", @"error", @"-y"] arrayByAddingObjectsFromArray:arguments];
FFmpegSession *session = [FFmpegKit executeWithArguments:command];
if (![ReturnCode isSuccess:[session getReturnCode]]) {
fprintf(stderr, "FFmpeg failed: %s\n%s\n", [[command description] UTF8String],
[[session getAllLogsAsString] UTF8String]);
return NO;
}
return YES;
}
static BOOL nonempty(NSString *path) {
NSDictionary *attributes = [[NSFileManager defaultManager] attributesOfItemAtPath:path error:nil];
return [attributes[NSFileSize] unsignedLongLongValue] > 0;
}
int main(int argc, const char *argv[]) {
@autoreleasepool {
NSString *hlsMode = argc == 5 ? [NSString stringWithUTF8String:argv[4]] : @"";
if (argc < 3 || argc > 5 || (argc == 5 && ![@[@"--allow-hls-extensions", @"--allow-hls-segments", @"--relax-hls-extensions", @"--disable-hls-extension-check"] containsObject:hlsMode])) {
fprintf(stderr, "Usage: ios_ffmpeg_capabilities OUTPUT_DIR MOTION_FIXTURE_DIR [REMOTE_FIXTURE_URL [--allow-hls-extensions|--allow-hls-segments|--relax-hls-extensions|--disable-hls-extension-check]]\n");
return 2;
}
[FFmpegKitConfig setLogRedirectionStrategy:LogRedirectionStrategyNeverPrintLogs];
FFmpegSession *protocols = [FFmpegKit executeWithArguments:@[@"-hide_banner", @"-protocols"]];
NSString *protocolList = [protocols getAllLogsAsString];
printf("FFmpeg protocols:\n%s\n", [protocolList UTF8String]);
BOOL supportsHTTPS = NO;
BOOL supportsHTTP = NO;
for (NSString *line in [protocolList componentsSeparatedByCharactersInSet:[NSCharacterSet newlineCharacterSet]]) {
NSString *protocol = [line stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
if ([protocol isEqualToString:@"http"]) supportsHTTP = YES;
if ([protocol isEqualToString:@"https"]) {
supportsHTTPS = YES;
}
}
if (!supportsHTTP) {
fprintf(stderr, "FAIL HTTP protocol unavailable: app TLS proxy cannot work\n");
return 1;
}
printf("PASS HTTP protocol; native HTTPS %s\n", supportsHTTPS ? "available" : "unavailable (requires app TLS proxy)");
FFmpegSession *hlsOptions = [FFmpegKit executeWithArguments:@[@"-hide_banner", @"-h", @"demuxer=hls"]];
printf("HLS demuxer options:\n%s\n", [[hlsOptions getAllLogsAsString] UTF8String]);
NSString *root = [NSString stringWithUTF8String:argv[1]];
NSString *fixtures = [NSString stringWithUTF8String:argv[2]];
if (![[NSFileManager defaultManager] createDirectoryAtPath:root
withIntermediateDirectories:YES attributes:nil error:nil]) return 2;
for (NSArray<NSString *> *format in @[
@[@"flac", @"flac"], @[@"m4a", @"alac"], @[@"wav", @"pcm_s24le"],
@[@"aiff", @"pcm_s24be"], @[@"mp3", @"libmp3lame"], @[@"opus", @"libopus"], @[@"aac", @"aac"]
]) {
NSString *output = [root stringByAppendingPathComponent:[@"track." stringByAppendingString:format[0]]];
if (!execute(@[@"-f", @"lavfi", @"-i", @"sine=frequency=997:sample_rate=48000",
@"-t", @"0.5", @"-c:a", format[1], output]) || !nonempty(output)) return 1;
if (!execute(@[@"-i", output, @"-map", @"0:a:0", @"-f", @"null", @"-"])) return 1;
printf("PASS encode/decode %s\n", [format[1] UTF8String]);
}
NSString *audio = [root stringByAppendingPathComponent:@"track.flac"];
NSString *stats = [root stringByAppendingPathComponent:@"analysis.txt"];
NSString *filter = [NSString stringWithFormat:
@"astats=metadata=1:reset=0,ebur128=peak=true:metadata=1,ametadata=print:file='%@'", stats];
if (!execute(@[@"-i", audio, @"-af", filter, @"-f", @"null", @"-"])) return 1;
NSString *metadata = [NSString stringWithContentsOfFile:stats encoding:NSUTF8StringEncoding error:nil];
if (![metadata containsString:@"lavfi.astats"] || ![metadata containsString:@"lavfi.r128.I"]) return 1;
printf("PASS astats/ebur128/ametadata\n");
NSString *spectrum = [root stringByAppendingPathComponent:@"spectrum.png"];
if (!execute(@[@"-i", audio, @"-lavfi",
@"showspectrumpic=s=64x64:legend=0:mode=combined:color=channel:scale=log:fscale=lin:win_func=hann",
@"-frames:v", @"1", spectrum]) || !nonempty(spectrum)) return 1;
NSString *cover = [root stringByAppendingPathComponent:@"cover.jpg"];
if (!execute(@[@"-i", spectrum, @"-vf", @"scale=32:32", @"-frames:v", @"1", cover]) || !nonempty(cover)) return 1;
printf("PASS spectrum PNG/JPEG scale\n");
for (NSString *codec in @[@"h264", @"hevc"]) {
NSString *input = [fixtures stringByAppendingPathComponent:
[NSString stringWithFormat:@"motion-probe-%@.mp4", codec]];
if (!execute(@[@"-xerror", @"-err_detect", @"explode", @"-i", input, @"-map", @"0:v:0",
@"-an", @"-sn", @"-dn", @"-f", @"null", @"-"])) return 1;
NSString *output = [root stringByAppendingPathComponent:[codec stringByAppendingString:@"-remux.mp4"]];
if (!execute(@[@"-i", input, @"-map", @"0:v:0", @"-an", @"-c:v", @"copy",
@"-movflags", @"+faststart", output]) || !nonempty(output)) return 1;
printf("PASS motion decode/remux %s\n", [codec UTF8String]);
}
if (argc >= 4) {
NSString *remote = [NSString stringWithUTF8String:argv[3]];
NSMutableArray<NSString *> *input = [NSMutableArray arrayWithArray:@[@"-rw_timeout", @"5000000"]];
if (argc == 5 && ![hlsMode isEqualToString:@"--disable-hls-extension-check"]) {
[input addObjectsFromArray:@[@"-allowed_extensions", @"ALL"]];
}
if ([@[@"--allow-hls-segments", @"--relax-hls-extensions"] containsObject:hlsMode]) {
[input addObjectsFromArray:@[@"-allowed_segment_extensions", @"ALL"]];
}
if ([@[@"--relax-hls-extensions", @"--disable-hls-extension-check"] containsObject:hlsMode]) {
[input addObjectsFromArray:@[@"-extension_picky", @"0"]];
}
[input addObjectsFromArray:@[@"-i", remote]];
if (!execute([input arrayByAddingObjectsFromArray:@[@"-map", @"0:v:0",
@"-an", @"-f", @"null", @"-"]])) return 1;
NSString *remuxed = [root stringByAppendingPathComponent:@"remote-remux.mp4"];
if (!execute([input arrayByAddingObjectsFromArray:@[@"-map", @"0:v:0", @"-an", @"-c:v", @"copy",
@"-movflags", @"+faststart", remuxed]]) || !nonempty(remuxed)) return 1;
if (!execute(@[@"-xerror", @"-err_detect", @"explode", @"-i", remuxed, @"-map", @"0:v:0",
@"-an", @"-f", @"null", @"-"])) return 1;
printf("PASS remote motion input %s\n", [remote UTF8String]);
printf("PASS remote stream-copy remux/decode\n");
}
printf("PASS iOS audio FFmpeg capabilities\n");
return 0;
}
}
+385
View File
@@ -0,0 +1,385 @@
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:flutter_test/flutter_test.dart';
import 'package:spotiflac_android/services/motion_artwork_proxy.dart';
void main() {
late HttpClient reader;
setUp(() => reader = HttpClient());
tearDown(() => reader.close(force: true));
Future<({int status, HttpHeaders headers, List<int> bytes})> fetch(
Uri uri, {
String method = 'GET',
String? range,
}) async {
final request = await reader.openUrl(method, uri);
if (range != null) request.headers.set('range', range);
final response = await request.close();
return (
status: response.statusCode,
headers: response.headers,
bytes: await response.expand((chunk) => chunk).toList(),
);
}
Future<HttpServer> tlsServer() async {
final context = SecurityContext()
..useCertificateChain('test/fixtures/network_tls/cert.pem')
..usePrivateKey('test/fixtures/network_tls/key.pem');
final server = await HttpServer.bindSecure(
InternetAddress.loopbackIPv4,
0,
context,
);
addTearDown(() => server.close(force: true));
return server;
}
HttpClient trustedFixtureClient(HttpServer server) =>
HttpClient()
..badCertificateCallback = (_, host, port) =>
host == '127.0.0.1' && port == server.port;
test('TLS MP4 relays GET, HEAD, ranges, status and content type', () async {
final server = await tlsServer();
final requests = <(String, String?)>[];
server.listen((request) async {
requests.add((request.method, request.headers.value('range')));
request.response.headers.contentType = ContentType('video', 'mp4');
request.response.headers.set('accept-ranges', 'bytes');
if (request.headers.value('range') == 'bytes=1-2') {
request.response
..statusCode = 206
..headers.set('content-range', 'bytes 1-2/4')
..contentLength = 2
..add([20, 30]);
} else {
request.response.contentLength = 4;
if (request.method != 'HEAD') request.response.add([10, 20, 30, 40]);
}
await request.response.close();
}, onError: (Object _) {});
final proxy = await MotionArtworkProxy.start(
Uri.parse('https://127.0.0.1:${server.port}/cover.mp4'),
client: trustedFixtureClient(server),
);
addTearDown(proxy.close);
final full = await fetch(proxy.source);
expect(full.bytes, [10, 20, 30, 40]);
expect(full.headers.contentType?.mimeType, 'video/mp4');
final range = await fetch(proxy.source, range: 'bytes=1-2');
expect(range.status, 206);
expect(range.bytes, [20, 30]);
expect(range.headers.value('content-range'), 'bytes 1-2/4');
final head = await fetch(proxy.source, method: 'HEAD');
expect(head.status, 200);
expect(head.bytes, isEmpty);
expect(head.headers.contentLength, 4);
final count = requests.length;
expect(
(await fetch(proxy.source.replace(query: 'url=http://other'))).status,
404,
);
expect(
(await fetch(proxy.source.replace(path: '/unknown/0.bin'))).status,
404,
);
expect((await fetch(proxy.source, method: 'POST')).status, 405);
expect(requests.length, count);
expect(requests, [('GET', null), ('GET', 'bytes=1-2'), ('HEAD', null)]);
});
test(
'TLS stays verified unless the test explicitly trusts its fixture',
() async {
final server = await tlsServer();
var requests = 0;
server.listen((request) async {
requests++;
await request.response.close();
}, onError: (Object _) {});
final proxy = await MotionArtworkProxy.start(
Uri.parse('https://127.0.0.1:${server.port}/cover.mp4'),
);
addTearDown(proxy.close);
expect((await fetch(proxy.source)).status, 502);
expect(requests, 0);
},
);
test(
'HLS rewrites master, redirected media, key, map and byte ranges',
() async {
final server = await tlsServer();
final paths = <String>[];
server.listen((request) async {
paths.add(request.uri.toString());
final response = request.response;
switch (request.uri.path) {
case '/start.m3u8':
response.statusCode = 302;
response.headers.set('location', '/cdn/master.m3u8');
case '/cdn/master.m3u8':
response.write(
'#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=100,CODECS="avc1.4"\nsub/media.m3u8\n',
);
case '/cdn/sub/media.m3u8':
response.statusCode = 302;
response.headers.set('location', '/final/media/list.m3u8');
case '/final/media/list.m3u8':
response.write('''#EXTM3U
#EXT-X-KEY:METHOD=AES-128,URI="../key.bin?token=secret"
#EXT-X-MAP:URI="../init.mp4",BYTERANGE="2@0"
#EXTINF:1,
one.ts
#EXT-X-BYTERANGE:2@1
../../shared.mp4
#EXTINF:1,
two.ts
#EXT-X-ENDLIST
''');
case '/final/key.bin':
response.add(List.filled(16, 42));
case '/final/init.mp4':
expect(request.headers.value('range'), 'bytes=0-1');
response
..statusCode = 206
..headers.set('content-range', 'bytes 0-1/4')
..add([1, 2]);
case '/shared.mp4':
expect(request.headers.value('range'), 'bytes=1-2');
response
..statusCode = 206
..headers.set('content-range', 'bytes 1-2/4')
..add([20, 30]);
case '/final/media/one.ts':
response.add([4, 5]);
case '/final/media/two.ts':
response.add([6, 7]);
default:
response.statusCode = 404;
}
await response.close();
}, onError: (Object _) {});
final proxy = await MotionArtworkProxy.start(
Uri.parse('https://127.0.0.1:${server.port}/start.m3u8'),
client: trustedFixtureClient(server),
);
addTearDown(proxy.close);
final master = utf8.decode((await fetch(proxy.source)).bytes);
final child = Uri.parse(master.split('\n').last);
expect(child.scheme, 'http');
expect(child.host, '127.0.0.1');
final media = utf8.decode((await fetch(child)).bytes);
expect(media, isNot(contains('https:')));
expect(media, contains('#EXT-X-BYTERANGE:2@1'));
expect(media, contains('BYTERANGE="2@0"'));
final attributes = RegExp(
r'URI="([^"]+)"',
).allMatches(media).map((match) => Uri.parse(match[1]!)).toList();
final segments = const LineSplitter()
.convert(media)
.where((line) => line.isNotEmpty && !line.startsWith('#'))
.map(Uri.parse)
.toList();
expect((await fetch(attributes[0])).bytes, List.filled(16, 42));
expect((await fetch(attributes[1], range: 'bytes=0-1')).bytes, [1, 2]);
expect((await fetch(segments[0])).bytes, [4, 5]);
expect((await fetch(segments[1], range: 'bytes=1-2')).bytes, [20, 30]);
expect((await fetch(segments[2])).bytes, [6, 7]);
expect(paths, contains('/final/key.bin?token=secret'));
expect(paths, contains('/cdn/sub/media.m3u8'));
expect(paths, contains('/final/media/list.m3u8'));
},
);
test(
'binary responses start streaming before the upstream finishes',
() async {
final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
addTearDown(() => server.close(force: true));
final continueBody = Completer<void>();
server.listen((request) async {
request.response.add(List.filled(16384, 1));
await request.response.flush();
await continueBody.future;
request.response.add(List.filled(1024 * 1024, 2));
await request.response.close();
});
final proxy = await MotionArtworkProxy.start(
Uri.parse('http://127.0.0.1:${server.port}/cover.mp4'),
);
addTearDown(proxy.close);
final response = await (await reader.getUrl(proxy.source)).close();
var bytes = 0;
await for (final chunk in response.timeout(const Duration(seconds: 2))) {
bytes += chunk.length;
if (!continueBody.isCompleted) continueBody.complete();
}
expect(bytes, 16384 + 1024 * 1024);
},
);
test(
'large declared MP4 supports partial reads and consumer cancellation',
() async {
final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
addTearDown(() => server.close(force: true));
final stop = Completer<void>();
server.listen((request) async {
expect(request.headers.value('range'), 'bytes=0-');
request.response
..bufferOutput = false
..statusCode = 206
..headers.set('content-range', 'bytes 0-67108863/67108864')
..contentLength = 64 << 20
..add(List.filled(1024, 1));
await request.response.flush();
await stop.future;
try {
await request.response.close();
} catch (_) {
// FFmpeg stops after reading its clip instead of downloading 64 MiB.
}
});
final proxy = await MotionArtworkProxy.start(
Uri.parse('http://127.0.0.1:${server.port}/cover.mp4'),
maxBytes: 4096,
);
addTearDown(proxy.close);
final request = await reader.getUrl(proxy.source);
request.headers.set('range', 'bytes=0-');
final response = await request.close();
expect(response.statusCode, 206);
expect(response.contentLength, 64 << 20);
final body = StreamIterator(response);
expect(await body.moveNext(), isTrue);
expect(body.current, everyElement(1));
expect(proxy.isClosed, isFalse);
await body.cancel();
stop.complete();
},
);
test(
'gzip playlists rewrite URI attributes and preserve unrelated names',
() async {
final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
addTearDown(() => server.close(force: true));
server.listen((request) async {
request.response.headers.contentType = ContentType(
'application',
'vnd.apple.mpegurl',
);
request.response.headers.set('content-encoding', 'gzip');
request.response.add(
gzip.encode(
utf8.encode('''#EXTM3U
#EXT-X-UNKNOWN:SERVER-URI="urn:ignored",X-FOO-URI="file:///ignored"
#EXT-X-KEY:METHOD=AES-128,URI="key.bin"
#EXTINF:1,
one.ts
#EXT-X-ENDLIST
'''),
),
);
await request.response.close();
});
final proxy = await MotionArtworkProxy.start(
Uri.parse('http://127.0.0.1:${server.port}/opaque'),
);
addTearDown(proxy.close);
expect(proxy.isHls, isFalse);
final response = await fetch(proxy.source);
expect(proxy.isHls, isTrue);
expect(response.status, 200);
expect(response.headers.value('content-encoding'), isNull);
final text = utf8.decode(response.bytes);
expect(
text,
contains('SERVER-URI="urn:ignored",X-FOO-URI="file:///ignored"'),
);
expect(text, contains('METHOD=AES-128,URI="http://127.0.0.1:'));
expect(
text.split('\n').where((line) => line.startsWith('http')).single,
endsWith('.ts'),
);
},
);
test(
'cumulative byte limit closes the proxy instead of retaining video',
() async {
final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
addTearDown(() => server.close(force: true));
server.listen((request) async {
request.response
..contentLength = 5
..add([1, 2, 3, 4, 5]);
await request.response.close();
});
final proxy = await MotionArtworkProxy.start(
Uri.parse('http://127.0.0.1:${server.port}/cover.mp4'),
maxBytes: 8,
);
addTearDown(proxy.close);
expect((await fetch(proxy.source)).bytes, [1, 2, 3, 4, 5]);
await expectLater(fetch(proxy.source), throwsA(isA<HttpException>()));
expect(proxy.isClosed, isTrue);
},
);
test(
'deadline and failed consumers close their client and loopback server',
() async {
final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
addTearDown(() => server.close(force: true));
server.listen((request) async {
request.response.write('video');
await request.response.close();
});
final source = Uri.parse('http://127.0.0.1:${server.port}/cover.mp4');
Uri? expired;
final client = HttpClient();
await expectLater(
withMotionArtworkProxy<void>(source, (local) async {
expired = local;
throw StateError('remux failed');
}, client: client),
throwsStateError,
);
await expectLater(fetch(expired!), throwsA(isA<SocketException>()));
expect(() => client.getUrl(source), throwsStateError);
final proxy = await MotionArtworkProxy.start(
source,
lifetime: const Duration(milliseconds: 30),
);
addTearDown(proxy.close);
await Future<void>.delayed(const Duration(milliseconds: 60));
expect(proxy.isClosed, isTrue);
await expectLater(fetch(proxy.source), throwsA(isA<SocketException>()));
},
);
test(
'playlist references cannot enable local file or non-HTTP resources',
() async {
final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
addTearDown(() => server.close(force: true));
server.listen((request) async {
request.response.write(
'#EXTM3U\n#EXT-X-KEY:URI="file:///private/key"\n',
);
await request.response.close();
});
final proxy = await MotionArtworkProxy.start(
Uri.parse('http://127.0.0.1:${server.port}/cover.m3u8'),
);
addTearDown(proxy.close);
expect((await fetch(proxy.source)).status, 502);
},
);
}