mirror of
https://github.com/zarzet/SpotiFLAC-Mobile.git
synced 2026-10-20 06:28:27 +02:00
perf(backup): stream history backup codecs and restore transactions natively
Export, import, and split legacy history backups in Rust while preserving schema 15, all history columns, path keys, and atomic restore behavior. Keep provider refresh and pending history edits intact.
This commit is contained in:
1 parent
d1de833184
commit
ad674a55ad
11 files changed
+2434
-26
No files matched your search
@@ -1131,6 +1131,14 @@ class DownloadHistoryNotifier extends Notifier<DownloadHistoryState> {
|
||||
Future<void> restoreFromBackup(List<Map<String, dynamic>> items) =>
|
||||
restoreFromBackupStream(Stream.fromIterable(items));
|
||||
|
||||
/// Serializes a native backup replacement with existing history writes, then
|
||||
/// publishes the same bounded initial state as a streamed restore.
|
||||
Future<void> restoreFromBackupOperation(Future<void> Function() restore) =>
|
||||
_enqueueHistoryWrite(() async {
|
||||
await restore();
|
||||
await _loadFromDatabase();
|
||||
});
|
||||
|
||||
/// Restores a large v2 backup without retaining the complete history in
|
||||
/// Dart memory. SQLite writes are grouped to keep JNI/channel overhead low.
|
||||
Future<void> restoreFromBackupStream(Stream<Map<String, dynamic>> items) =>
|
||||
|
||||
@@ -142,7 +142,9 @@ class _BackupRestorePageState extends ConsumerState<BackupRestorePage> {
|
||||
if (bundle.hasHistory) {
|
||||
await ref
|
||||
.read(downloadHistoryProvider.notifier)
|
||||
.restoreFromBackupStream(bundle.streamHistory());
|
||||
.restoreFromBackupOperation(
|
||||
() => BackupService.restoreHistory(bundle!),
|
||||
);
|
||||
}
|
||||
if (bundle.hasCollections) {
|
||||
await ref
|
||||
|
||||
@@ -3,10 +3,12 @@ import 'dart:io';
|
||||
import 'dart:isolate';
|
||||
|
||||
import 'package:archive/archive_io.dart';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/foundation.dart' show visibleForTesting;
|
||||
import 'package:path/path.dart' as p;
|
||||
import 'package:path_provider/path_provider.dart';
|
||||
import 'package:spotiflac_android/constants/app_info.dart';
|
||||
import 'package:spotiflac_android/services/history_database.dart';
|
||||
import 'package:spotiflac_android/services/platform_bridge.dart';
|
||||
import 'package:spotiflac_android/services/user_profile_store.dart';
|
||||
import 'package:spotiflac_android/utils/logger.dart';
|
||||
|
||||
@@ -171,22 +173,38 @@ class BackupService {
|
||||
|
||||
try {
|
||||
var historyCount = 0;
|
||||
var offset = 0;
|
||||
final historySink = historyFile.openWrite();
|
||||
try {
|
||||
while (includeHistory) {
|
||||
final page = await loadHistoryPage(_historyPageSize, offset);
|
||||
for (final item in page) {
|
||||
historySink.writeln(jsonEncode(item));
|
||||
if (includeHistory && PlatformBridge.supportsCoreBackend) {
|
||||
final database = await HistoryDatabase.instance.database;
|
||||
final result = await PlatformBridge.runNativeDataJob({
|
||||
'operation': 'backup_history_export',
|
||||
'history_path': database.path,
|
||||
'ndjson_path': historyFile.path,
|
||||
'platform': Platform.isAndroid ? 'android' : 'ios',
|
||||
if (Platform.isIOS)
|
||||
'documents_path': (await getApplicationDocumentsDirectory()).path,
|
||||
});
|
||||
if (result['published'] != true || result['count'] is! int) {
|
||||
throw const FormatException('Incomplete native history backup');
|
||||
}
|
||||
historyCount = result['count'] as int;
|
||||
} else {
|
||||
var offset = 0;
|
||||
final historySink = historyFile.openWrite();
|
||||
try {
|
||||
while (includeHistory) {
|
||||
final page = await loadHistoryPage(_historyPageSize, offset);
|
||||
for (final item in page) {
|
||||
historySink.writeln(jsonEncode(item));
|
||||
}
|
||||
await historySink.flush();
|
||||
historyCount += page.length;
|
||||
offset += page.length;
|
||||
if (page.length < _historyPageSize) break;
|
||||
}
|
||||
await historySink.flush();
|
||||
historyCount += page.length;
|
||||
offset += page.length;
|
||||
if (page.length < _historyPageSize) break;
|
||||
} finally {
|
||||
await historySink.close();
|
||||
}
|
||||
await historySink.flush();
|
||||
} finally {
|
||||
await historySink.close();
|
||||
}
|
||||
|
||||
final coverManifest = <String, Map<String, String>>{};
|
||||
@@ -300,13 +318,104 @@ class BackupService {
|
||||
Directory? temporaryDirectory,
|
||||
}) async {
|
||||
final tempRoot = temporaryDirectory ?? await getTemporaryDirectory();
|
||||
final bundle = await _parseFileInBackground(path, tempRoot.path);
|
||||
final bundle =
|
||||
PlatformBridge.supportsCoreBackend && !await _hasZipHeader(File(path))
|
||||
? await _parseLegacyNative(path, tempRoot)
|
||||
: await _parseFileInBackground(path, tempRoot.path);
|
||||
if (bundle == null) {
|
||||
_log.w('Backup file could not be read: invalid or unsupported contents');
|
||||
}
|
||||
return bundle;
|
||||
}
|
||||
|
||||
/// Called inside the history notifier's write queue. The native importer
|
||||
/// validates and stages the complete input before replacing the live index.
|
||||
static Future<void> restoreHistory(BackupBundle bundle) async {
|
||||
if (!bundle.hasHistory) return;
|
||||
final database = await HistoryDatabase.instance.database;
|
||||
if (PlatformBridge.supportsCoreBackend &&
|
||||
bundle._historyNdjsonPath != null) {
|
||||
final result = await PlatformBridge.runNativeDataJob({
|
||||
'operation': 'backup_history_import',
|
||||
'history_path': database.path,
|
||||
'ndjson_path': bundle._historyNdjsonPath,
|
||||
if (bundle._historyCount != null)
|
||||
'expected_count': bundle._historyCount,
|
||||
'platform': Platform.isAndroid ? 'android' : 'ios',
|
||||
'mode': 'replace',
|
||||
});
|
||||
if (result['committed'] != true) {
|
||||
throw const FormatException('Incomplete native history restore');
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Desktop and legacy in-memory callers retain the established codec.
|
||||
await HistoryDatabase.instance.clearAll();
|
||||
var batch = <Map<String, dynamic>>[];
|
||||
await for (final item in bundle.streamHistory()) {
|
||||
batch.add(item);
|
||||
if (batch.length < _historyPageSize) continue;
|
||||
await HistoryDatabase.instance.upsertBatch(batch);
|
||||
batch = <Map<String, dynamic>>[];
|
||||
}
|
||||
if (batch.isNotEmpty) await HistoryDatabase.instance.upsertBatch(batch);
|
||||
}
|
||||
|
||||
static Future<bool> _hasZipHeader(File file) async {
|
||||
final header = await file
|
||||
.openRead(0, 4)
|
||||
.fold<List<int>>(<int>[], (bytes, chunk) => bytes..addAll(chunk));
|
||||
return header.length == 4 &&
|
||||
header[0] == 0x50 &&
|
||||
header[1] == 0x4b &&
|
||||
header[2] == 0x03 &&
|
||||
header[3] == 0x04;
|
||||
}
|
||||
|
||||
static Future<BackupBundle?> _parseLegacyNative(
|
||||
String path,
|
||||
Directory temporary,
|
||||
) async {
|
||||
await temporary.create(recursive: true);
|
||||
final staging = await temporary.createTemp('spotiflac_legacy_backup_');
|
||||
try {
|
||||
final historyPath = p.join(staging.path, 'history.ndjson');
|
||||
final metadataPath = p.join(staging.path, 'metadata.json');
|
||||
final result = await PlatformBridge.runNativeDataJob({
|
||||
'operation': 'backup_split_legacy',
|
||||
'input_path': path,
|
||||
'ndjson_path': historyPath,
|
||||
'metadata_path': metadataPath,
|
||||
});
|
||||
if (result['published'] != true || result['count'] is! int) {
|
||||
throw const FormatException('Incomplete legacy backup staging');
|
||||
}
|
||||
final parsed = await Isolate.run(
|
||||
() => parse(File(metadataPath).readAsStringSync()),
|
||||
);
|
||||
if (parsed == null) throw const FormatException('Invalid backup');
|
||||
return BackupBundle(
|
||||
formatVersion: parsed.formatVersion,
|
||||
appVersion: parsed.appVersion,
|
||||
createdAt: parsed.createdAt,
|
||||
settings: parsed.settings,
|
||||
profile: parsed.profile,
|
||||
history: const [],
|
||||
hasHistory: parsed.hasHistory,
|
||||
historyCount: result['count'] as int,
|
||||
historyNdjsonPath: parsed.hasHistory ? historyPath : null,
|
||||
collections: parsed.collections,
|
||||
playlistCovers: parsed.playlistCovers,
|
||||
extensions: parsed.extensions,
|
||||
temporaryDirectoryPath: staging.path,
|
||||
);
|
||||
} catch (error) {
|
||||
await staging.delete(recursive: true);
|
||||
_log.w('Legacy backup native parse failed: $error');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
static Future<BackupBundle?> _parseFileInBackground(
|
||||
String path,
|
||||
String temporaryPath,
|
||||
@@ -317,15 +426,7 @@ class BackupService {
|
||||
String temporaryPath,
|
||||
) async {
|
||||
final file = File(path);
|
||||
final header = await file
|
||||
.openRead(0, 4)
|
||||
.fold<List<int>>(<int>[], (bytes, chunk) => bytes..addAll(chunk));
|
||||
final isZip =
|
||||
header.length == 4 &&
|
||||
header[0] == 0x50 &&
|
||||
header[1] == 0x4b &&
|
||||
header[2] == 0x03 &&
|
||||
header[3] == 0x04;
|
||||
final isZip = await _hasZipHeader(file);
|
||||
return isZip
|
||||
? _parseArchive(file, temporaryDirectory: Directory(temporaryPath))
|
||||
: parse(await file.readAsString());
|
||||
@@ -442,7 +543,7 @@ class BackupService {
|
||||
history: const [],
|
||||
hasHistory: historyEntry != null,
|
||||
historyNdjsonPath: historyPath,
|
||||
historyCount: (root['history_count'] as num?)?.toInt() ?? 0,
|
||||
historyCount: (root['history_count'] as num?)?.toInt(),
|
||||
collections: _mapOrEmpty(data['collections']),
|
||||
playlistCovers: restoredCovers,
|
||||
extensions: _mapOrEmpty(data['extensions']),
|
||||
|
||||
@@ -128,6 +128,9 @@ fn execute(
|
||||
operation if operation.starts_with("library_") => {
|
||||
crate::native_library::execute(request, check)
|
||||
}
|
||||
operation if operation.starts_with("backup_") => {
|
||||
crate::native_backup::execute(request, check)
|
||||
}
|
||||
_ => Err(format!("Unknown native data operation: {operation}")),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ mod logging;
|
||||
mod lyrics;
|
||||
mod manager;
|
||||
mod metadata;
|
||||
mod native_backup;
|
||||
mod native_id3;
|
||||
mod native_library;
|
||||
mod native_listing;
|
||||
|
||||
@@ -0,0 +1,551 @@
|
||||
//! History backup codecs stay on the native worker. Other backup categories,
|
||||
//! archive layout and presentation remain owned by Flutter.
|
||||
mod legacy;
|
||||
|
||||
use crate::native_library::dart_date;
|
||||
use crate::native_library::path_keys::{
|
||||
build, dart_lower, dart_trim, dart_upper, regex_whitespace,
|
||||
};
|
||||
use rusqlite::{Connection, OpenFlags, TransactionBehavior, params, params_from_iter};
|
||||
use serde_json::{Map, Value, json};
|
||||
use std::{
|
||||
fs::{self, File, OpenOptions},
|
||||
io::{BufRead, BufReader, BufWriter, Read, Write},
|
||||
path::Path,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
type Check<'a> = &'a dyn Fn() -> Result<(), String>;
|
||||
const MAX_HISTORY_BYTES: u64 = 512 << 20;
|
||||
const MAX_ROW_BYTES: u64 = 16 << 20;
|
||||
const BATCH_SIZE: usize = 500;
|
||||
const COLUMNS: &[(&str, &str)] = &[
|
||||
("id", "id"),
|
||||
("trackName", "track_name"),
|
||||
("artistName", "artist_name"),
|
||||
("albumName", "album_name"),
|
||||
("albumArtist", "album_artist"),
|
||||
("coverUrl", "cover_url"),
|
||||
("filePath", "file_path"),
|
||||
("storageMode", "storage_mode"),
|
||||
("downloadTreeUri", "download_tree_uri"),
|
||||
("safRelativeDir", "saf_relative_dir"),
|
||||
("safFileName", "saf_file_name"),
|
||||
("service", "service"),
|
||||
("downloadedAt", "downloaded_at"),
|
||||
("isrc", "isrc"),
|
||||
("spotifyId", "spotify_id"),
|
||||
("trackNumber", "track_number"),
|
||||
("totalTracks", "total_tracks"),
|
||||
("discNumber", "disc_number"),
|
||||
("totalDiscs", "total_discs"),
|
||||
("duration", "duration"),
|
||||
("releaseDate", "release_date"),
|
||||
("quality", "quality"),
|
||||
("bitDepth", "bit_depth"),
|
||||
("sampleRate", "sample_rate"),
|
||||
("bitrate", "bitrate"),
|
||||
("format", "format"),
|
||||
("genre", "genre"),
|
||||
("composer", "composer"),
|
||||
("label", "label"),
|
||||
("copyright", "copyright"),
|
||||
];
|
||||
|
||||
fn err(context: &str, error: impl std::fmt::Display) -> String {
|
||||
format!("{context}: {error}")
|
||||
}
|
||||
fn required<'a>(request: &'a Value, field: &str) -> Result<&'a str, String> {
|
||||
request[field]
|
||||
.as_str()
|
||||
.filter(|v| !v.is_empty())
|
||||
.ok_or_else(|| format!("Backup requires {field}"))
|
||||
}
|
||||
fn output_path<'a>(request: &'a Value, field: &str) -> Result<&'a str, String> {
|
||||
let path = required(request, field)?;
|
||||
if !Path::new(path).is_absolute() {
|
||||
return Err("Backup output path must be absolute".into());
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
fn open(request: &Value, writable: bool) -> Result<Connection, String> {
|
||||
let path = required(request, "history_path")?;
|
||||
if !Path::new(path).is_absolute()
|
||||
|| Path::new(path).file_name().and_then(|v| v.to_str()) != Some("history.db")
|
||||
{
|
||||
return Err("Backup database must reference existing history.db".into());
|
||||
}
|
||||
let db = Connection::open_with_flags(
|
||||
path,
|
||||
if writable {
|
||||
OpenFlags::SQLITE_OPEN_READ_WRITE
|
||||
} else {
|
||||
OpenFlags::SQLITE_OPEN_READ_ONLY
|
||||
},
|
||||
)
|
||||
.map_err(|e| err("open History backup", e))?;
|
||||
db.busy_timeout(Duration::from_secs(5))
|
||||
.map_err(|e| err("History timeout", e))?;
|
||||
let version: i64 = db
|
||||
.pragma_query_value(None, "user_version", |row| row.get(0))
|
||||
.map_err(|e| err("History schema", e))?;
|
||||
if version != 15 {
|
||||
return Err(format!(
|
||||
"History schema v{version} is incompatible with native v15"
|
||||
));
|
||||
}
|
||||
db.execute_batch("PRAGMA recursive_triggers=ON; PRAGMA synchronous=NORMAL;")
|
||||
.map_err(|e| err("configure History backup", e))?;
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
pub(crate) fn execute(request: &Value, check: Check<'_>) -> Result<Value, String> {
|
||||
check()?;
|
||||
match required(request, "operation")? {
|
||||
"backup_history_export" => export(request, check),
|
||||
"backup_history_import" => import(request, check),
|
||||
"backup_split_legacy" => legacy::split(request, check),
|
||||
_ => Err("Unknown native backup operation".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn scalar(value: &Value) -> Result<rusqlite::types::Value, String> {
|
||||
use rusqlite::types::Value as Sql;
|
||||
match value {
|
||||
Value::Null => Ok(Sql::Null),
|
||||
Value::String(s) => Ok(Sql::Text(s.clone())),
|
||||
Value::Bool(v) => Ok(Sql::Integer(i64::from(*v))),
|
||||
Value::Number(n) => n
|
||||
.as_i64()
|
||||
.map(Sql::Integer)
|
||||
.or_else(|| n.as_f64().map(Sql::Real))
|
||||
.ok_or("Invalid backup number".into()),
|
||||
_ => Err("Backup storage field cannot contain an object or array".into()),
|
||||
}
|
||||
}
|
||||
fn text<'a>(row: &'a Value, key: &str) -> Result<&'a str, String> {
|
||||
match &row[key] {
|
||||
Value::Null => Ok(""),
|
||||
Value::String(s) => Ok(s),
|
||||
_ => Err(format!("Backup {key} must be a string")),
|
||||
}
|
||||
}
|
||||
fn number(value: &Value) -> Option<i64> {
|
||||
value
|
||||
.as_i64()
|
||||
.or_else(|| value.as_f64().filter(|v| v.is_finite()).map(|v| v as i64))
|
||||
}
|
||||
fn normalized(value: &str) -> String {
|
||||
dart_lower(dart_trim(value))
|
||||
}
|
||||
fn date(value: &str) -> Option<(String, i64)> {
|
||||
let parsed = dart_date::parse(value)?;
|
||||
Some((
|
||||
dart_date::iso_utc(&parsed.instant),
|
||||
parsed.instant.timestamp_millis(),
|
||||
))
|
||||
}
|
||||
fn replaygain(row: &Value) -> bool {
|
||||
if row["hasReplayGain"] == true || row["hasReplayGain"].as_f64() == Some(1.0) {
|
||||
return true;
|
||||
}
|
||||
["replaygain_track_gain", "replaygain_album_gain"]
|
||||
.iter()
|
||||
.any(|key| {
|
||||
let s = match &row[*key] {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Number(n) => n.to_string(),
|
||||
_ => return false,
|
||||
};
|
||||
let trimmed = dart_trim(&s);
|
||||
let value = if trimmed.to_ascii_lowercase().ends_with("db") {
|
||||
dart_trim(&trimmed[..trimmed.len() - 2])
|
||||
} else {
|
||||
trimmed
|
||||
};
|
||||
value.parse::<f64>().is_ok_and(f64::is_finite)
|
||||
})
|
||||
}
|
||||
|
||||
fn encode(row: &Value) -> Result<Map<String, Value>, String> {
|
||||
if !row.is_object() {
|
||||
return Err("Backup history row must be an object".into());
|
||||
}
|
||||
for key in [
|
||||
"id",
|
||||
"trackName",
|
||||
"artistName",
|
||||
"albumName",
|
||||
"filePath",
|
||||
"service",
|
||||
"downloadedAt",
|
||||
] {
|
||||
if !row[key].is_string() {
|
||||
return Err(format!("Backup history requires string {key}"));
|
||||
}
|
||||
}
|
||||
for key in ["lyricsMetadataScanVersion", "replayGainMetadataScanVersion"] {
|
||||
if !row[key].is_null() && !row[key].is_number() {
|
||||
return Err(format!("Backup {key} must be numeric"));
|
||||
}
|
||||
}
|
||||
let mut values = Map::new();
|
||||
for (key, column) in COLUMNS {
|
||||
values.insert((*column).into(), row[*key].clone());
|
||||
}
|
||||
let parsed_date = date(text(row, "downloadedAt")?);
|
||||
if let Some((date, _)) = &parsed_date {
|
||||
values.insert("downloaded_at".into(), date.clone().into());
|
||||
}
|
||||
for (key, column) in [
|
||||
("safRepaired", "saf_repaired"),
|
||||
("explicit", "explicit"),
|
||||
("hasLyrics", "has_lyrics"),
|
||||
] {
|
||||
values.insert(column.into(), i64::from(row[key] == true).into());
|
||||
}
|
||||
values.insert("has_replaygain".into(), i64::from(replaygain(row)).into());
|
||||
values.insert(
|
||||
"lyrics_metadata_scan_version".into(),
|
||||
number(&row["lyricsMetadataScanVersion"])
|
||||
.unwrap_or(i64::from(row.get("hasLyrics").is_some()))
|
||||
.into(),
|
||||
);
|
||||
values.insert(
|
||||
"replaygain_metadata_scan_version".into(),
|
||||
number(&row["replayGainMetadataScanVersion"])
|
||||
.unwrap_or(0)
|
||||
.into(),
|
||||
);
|
||||
let track = normalized(text(row, "trackName")?);
|
||||
let artist = normalized(text(row, "artistName")?);
|
||||
let album = normalized(text(row, "albumName")?);
|
||||
let album_artist = if dart_trim(text(row, "albumArtist")?).is_empty() {
|
||||
artist.clone()
|
||||
} else {
|
||||
normalized(text(row, "albumArtist")?)
|
||||
};
|
||||
let isrc = dart_upper(dart_trim(text(row, "isrc")?))
|
||||
.chars()
|
||||
.filter(|ch| *ch != '-' && !regex_whitespace(*ch))
|
||||
.collect::<String>();
|
||||
values.insert(
|
||||
"spotify_id_norm".into(),
|
||||
normalized(text(row, "spotifyId")?).into(),
|
||||
);
|
||||
values.insert("isrc_norm".into(), isrc.into());
|
||||
values.insert(
|
||||
"match_key".into(),
|
||||
if track.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{track}|{artist}")
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
values.insert("album_key".into(), format!("{album}|{album_artist}").into());
|
||||
values.insert(
|
||||
"search_text".into(),
|
||||
[&track, &artist, &album, &album_artist]
|
||||
.into_iter()
|
||||
.filter(|v| !v.is_empty())
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.into(),
|
||||
);
|
||||
for (column, value) in [
|
||||
("sort_track", track),
|
||||
("sort_artist", artist),
|
||||
("sort_album", album),
|
||||
("sort_album_artist", album_artist),
|
||||
("sort_genre", normalized(text(row, "genre")?)),
|
||||
(
|
||||
"sort_release",
|
||||
dart_trim(text(row, "releaseDate")?).to_string(),
|
||||
),
|
||||
] {
|
||||
values.insert(column.into(), value.into());
|
||||
}
|
||||
values.insert(
|
||||
"sort_added".into(),
|
||||
parsed_date.map(|(_, time)| time).unwrap_or(0).into(),
|
||||
);
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
fn export(request: &Value, check: Check<'_>) -> Result<Value, String> {
|
||||
let db = open(request, false)?;
|
||||
let output = output_path(request, "ndjson_path")?;
|
||||
let file = OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(output)
|
||||
.map_err(|e| err("create History export", e))?;
|
||||
let result = (|| {
|
||||
let mut writer = BufWriter::new(file);
|
||||
let mut statement = db
|
||||
.prepare("SELECT * FROM history ORDER BY sort_added DESC,id DESC")
|
||||
.map_err(|e| err("prepare History export", e))?;
|
||||
let columns = statement
|
||||
.column_names()
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect::<Vec<_>>();
|
||||
let mut rows = statement
|
||||
.query([])
|
||||
.map_err(|e| err("query History export", e))?;
|
||||
let mut count = 0_u64;
|
||||
let mut bytes = 0_u64;
|
||||
while let Some(row) = rows.next().map_err(|e| err("read History export", e))? {
|
||||
check()?;
|
||||
let mut stored = Map::new();
|
||||
for (index, column) in columns.iter().enumerate() {
|
||||
use rusqlite::types::ValueRef;
|
||||
let value = match row
|
||||
.get_ref(index)
|
||||
.map_err(|e| err("read History value", e))?
|
||||
{
|
||||
ValueRef::Null => Value::Null,
|
||||
ValueRef::Integer(n) => n.into(),
|
||||
ValueRef::Real(n) => json!(n),
|
||||
ValueRef::Text(v) => String::from_utf8(v.to_vec())
|
||||
.map_err(|e| err("History UTF-8", e))?
|
||||
.into(),
|
||||
ValueRef::Blob(_) => {
|
||||
return Err("History backup cannot encode binary column".into());
|
||||
}
|
||||
};
|
||||
stored.insert(column.clone(), value);
|
||||
}
|
||||
let json = decode(&stored, request);
|
||||
let encoded = serde_json::to_vec(&json).map_err(|e| err("encode History export", e))?;
|
||||
bytes += encoded.len() as u64 + 1;
|
||||
if bytes > MAX_HISTORY_BYTES {
|
||||
return Err("History backup exceeds size limit".into());
|
||||
}
|
||||
writer
|
||||
.write_all(&encoded)
|
||||
.and_then(|_| writer.write_all(b"\n"))
|
||||
.map_err(|e| err("write History export", e))?;
|
||||
count += 1;
|
||||
}
|
||||
writer
|
||||
.flush()
|
||||
.and_then(|_| writer.get_ref().sync_all())
|
||||
.map_err(|e| err("publish History export", e))?;
|
||||
check()?;
|
||||
Ok(json!({"published":true,"count":count,"path":output}))
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_file(output);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn decode(stored: &Map<String, Value>, request: &Value) -> Map<String, Value> {
|
||||
let mut json = Map::new();
|
||||
for (key, column) in COLUMNS {
|
||||
json.insert(
|
||||
(*key).into(),
|
||||
stored.get(*column).cloned().unwrap_or(Value::Null),
|
||||
);
|
||||
}
|
||||
if request["platform"] == "ios"
|
||||
&& let Some(documents) = request["documents_path"].as_str()
|
||||
&& let Some(path) = json["filePath"].as_str()
|
||||
{
|
||||
json.insert("filePath".into(), rebase_ios(path, documents).into());
|
||||
}
|
||||
for (key, column) in [
|
||||
("safRepaired", "saf_repaired"),
|
||||
("explicit", "explicit"),
|
||||
("hasLyrics", "has_lyrics"),
|
||||
("hasReplayGain", "has_replaygain"),
|
||||
] {
|
||||
json.insert(
|
||||
key.into(),
|
||||
(stored.get(column).and_then(Value::as_i64) == Some(1)).into(),
|
||||
);
|
||||
}
|
||||
for (key, column) in [
|
||||
("lyricsMetadataScanVersion", "lyrics_metadata_scan_version"),
|
||||
(
|
||||
"replayGainMetadataScanVersion",
|
||||
"replaygain_metadata_scan_version",
|
||||
),
|
||||
] {
|
||||
json.insert(
|
||||
key.into(),
|
||||
stored
|
||||
.get(column)
|
||||
.filter(|v| !v.is_null())
|
||||
.cloned()
|
||||
.unwrap_or(json!(0)),
|
||||
);
|
||||
}
|
||||
json
|
||||
}
|
||||
|
||||
fn rebase_ios(path: &str, documents: &str) -> String {
|
||||
use std::sync::OnceLock;
|
||||
static ROOT: OnceLock<regex::Regex> = OnceLock::new();
|
||||
let regex=ROOT.get_or_init(||regex::Regex::new(r"(?i)^(?:/private)?/var/mobile/Containers/Data/Application/[A-F0-9\-]+|^/[^\n]+/Library/Developer/CoreSimulator/Devices/[A-F0-9\-]+/data/Containers/Data/Application/[A-F0-9\-]+").expect("iOS container pattern"));
|
||||
let Some(current) = regex.find(documents) else {
|
||||
return path.into();
|
||||
};
|
||||
let Some(previous) = regex.find(path) else {
|
||||
return path.into();
|
||||
};
|
||||
if &documents[current.end()..] != "/Documents" || !path[previous.end()..].starts_with('/') {
|
||||
return path.into();
|
||||
}
|
||||
let suffix = &path[previous.end()..];
|
||||
if suffix.split('/').any(|v| v == "..")
|
||||
|| ![
|
||||
"/Documents",
|
||||
"/Library/Application Support",
|
||||
"/Library/Caches",
|
||||
]
|
||||
.iter()
|
||||
.any(|root| suffix == *root || suffix.starts_with(&format!("{root}/")))
|
||||
{
|
||||
return path.into();
|
||||
}
|
||||
format!("{}{suffix}", &documents[..current.end()])
|
||||
}
|
||||
|
||||
fn import(request: &Value, check: Check<'_>) -> Result<Value, String> {
|
||||
let mut db = open(request, true)?;
|
||||
let input = required(request, "ndjson_path")?;
|
||||
if fs::metadata(input)
|
||||
.map_err(|e| err("History backup metadata", e))?
|
||||
.len()
|
||||
> MAX_HISTORY_BYTES
|
||||
{
|
||||
return Err("History backup exceeds size limit".into());
|
||||
}
|
||||
let columns = db
|
||||
.prepare("PRAGMA table_info(history)")
|
||||
.map_err(|e| err("History columns", e))?
|
||||
.query_map([], |row| row.get::<_, String>(1))
|
||||
.map_err(|e| err("History columns", e))?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| err("History columns", e))?;
|
||||
let column_list = columns
|
||||
.iter()
|
||||
.map(|v| format!("\"{}\"", v.replace('"', "\"\"")))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
db.execute_batch("CREATE TEMP TABLE native_backup_stage AS SELECT * FROM history WHERE 0;
|
||||
CREATE UNIQUE INDEX native_backup_stage_id ON native_backup_stage(id);
|
||||
CREATE TEMP TABLE native_backup_keys(item_id TEXT NOT NULL,path_key TEXT NOT NULL,PRIMARY KEY(item_id,path_key));").map_err(|e|err("stage History restore",e))?;
|
||||
let insert = format!(
|
||||
"INSERT OR REPLACE INTO native_backup_stage({column_list}) VALUES({})",
|
||||
vec!["?"; columns.len()].join(",")
|
||||
);
|
||||
let android = request["platform"] == "android";
|
||||
let mut count = 0_u64;
|
||||
let mut batch = Vec::with_capacity(BATCH_SIZE);
|
||||
let mut flush = |batch: &mut Vec<Value>| -> Result<(), String> {
|
||||
check()?;
|
||||
let transaction = db
|
||||
.transaction()
|
||||
.map_err(|e| err("stage restore transaction", e))?;
|
||||
for row in batch.drain(..) {
|
||||
check()?;
|
||||
let values = encode(&row)?;
|
||||
let fields = columns
|
||||
.iter()
|
||||
.map(|column| scalar(values.get(column).unwrap_or(&Value::Null)))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
transaction
|
||||
.prepare_cached(&insert)
|
||||
.map_err(|e| err("prepare restore row", e))?
|
||||
.execute(params_from_iter(fields))
|
||||
.map_err(|e| err("stage restore row", e))?;
|
||||
let id = text(&row, "id")?;
|
||||
transaction
|
||||
.execute("DELETE FROM native_backup_keys WHERE item_id=?", [id])
|
||||
.map_err(|e| err("replace restore keys", e))?;
|
||||
for key in build(text(&row, "filePath")?, android)? {
|
||||
transaction
|
||||
.prepare_cached("INSERT OR IGNORE INTO native_backup_keys VALUES(?,?)")
|
||||
.map_err(|e| err("prepare restore keys", e))?
|
||||
.execute(params![id, key])
|
||||
.map_err(|e| err("stage restore keys", e))?;
|
||||
}
|
||||
}
|
||||
transaction
|
||||
.commit()
|
||||
.map_err(|e| err("commit restore staging", e))
|
||||
};
|
||||
let mut reader = BufReader::new(File::open(input).map_err(|e| err("open History backup", e))?);
|
||||
loop {
|
||||
check()?;
|
||||
let mut line = String::new();
|
||||
let bytes = reader
|
||||
.by_ref()
|
||||
.take(MAX_ROW_BYTES + 1)
|
||||
.read_line(&mut line)
|
||||
.map_err(|e| err("read History backup", e))?;
|
||||
if bytes == 0 {
|
||||
break;
|
||||
}
|
||||
if bytes as u64 > MAX_ROW_BYTES {
|
||||
return Err("History backup row exceeds size limit".into());
|
||||
}
|
||||
if dart_trim(&line).is_empty() {
|
||||
continue;
|
||||
}
|
||||
let row: Value =
|
||||
serde_json::from_str(&line).map_err(|e| err("decode History backup", e))?;
|
||||
// Legacy archive restore skipped JSON values that were not objects.
|
||||
if !row.is_object() {
|
||||
continue;
|
||||
}
|
||||
batch.push(row);
|
||||
count += 1;
|
||||
if batch.len() == BATCH_SIZE {
|
||||
flush(&mut batch)?;
|
||||
}
|
||||
}
|
||||
flush(&mut batch)?;
|
||||
if let Some(expected) = request["expected_count"].as_u64()
|
||||
&& count != expected
|
||||
{
|
||||
return Err(format!(
|
||||
"History backup count mismatch: {count}, expected {expected}"
|
||||
));
|
||||
}
|
||||
check()?;
|
||||
let transaction = db
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.map_err(|e| err("begin History restore", e))?;
|
||||
let replace = request["mode"].as_str().unwrap_or("replace") == "replace";
|
||||
if replace {
|
||||
transaction
|
||||
.execute_batch("DELETE FROM history_path_keys; DELETE FROM history;")
|
||||
.map_err(|e| err("clear restored History", e))?;
|
||||
} else if request["mode"] != "merge" {
|
||||
return Err("Invalid History restore mode".into());
|
||||
} else {
|
||||
transaction.execute("DELETE FROM history_path_keys WHERE item_id IN(SELECT id FROM native_backup_stage)",[]).map_err(|e|err("replace restored History keys",e))?;
|
||||
}
|
||||
check()?;
|
||||
transaction.execute(&format!("INSERT OR REPLACE INTO history({column_list}) SELECT {column_list} FROM native_backup_stage"),[]).map_err(|e|err("swap restored History",e))?;
|
||||
transaction.execute("INSERT OR IGNORE INTO history_path_keys SELECT item_id,path_key FROM native_backup_keys",[]).map_err(|e|err("swap restored keys",e))?;
|
||||
let imported: i64 = transaction
|
||||
.query_row("SELECT COUNT(*) FROM native_backup_stage", [], |row| {
|
||||
row.get(0)
|
||||
})
|
||||
.map_err(|e| err("count restored History", e))?;
|
||||
check()?;
|
||||
transaction
|
||||
.commit()
|
||||
.map_err(|e| err("commit History restore", e))?;
|
||||
Ok(json!({"committed":true,"count":count,"imported":imported}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,232 @@
|
||||
//! Deserialize the legacy envelope one history object at a time. Large history
|
||||
//! arrays never become a Value/List or cross the platform bridge.
|
||||
use super::*;
|
||||
use serde::de::{self, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor};
|
||||
use std::{
|
||||
fmt,
|
||||
io::{Seek, SeekFrom},
|
||||
};
|
||||
|
||||
struct History<'a> {
|
||||
writer: &'a mut BufWriter<File>,
|
||||
count: u64,
|
||||
bytes: u64,
|
||||
check: Check<'a>,
|
||||
}
|
||||
impl History<'_> {
|
||||
fn reset(&mut self) -> Result<(), String> {
|
||||
self.writer
|
||||
.flush()
|
||||
.map_err(|e| err("flush legacy history", e))?;
|
||||
self.writer
|
||||
.get_mut()
|
||||
.set_len(0)
|
||||
.map_err(|e| err("reset legacy history", e))?;
|
||||
self.writer
|
||||
.seek(SeekFrom::Start(0))
|
||||
.map_err(|e| err("seek legacy history", e))?;
|
||||
self.count = 0;
|
||||
self.bytes = 0;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
struct RootSeed<'a, 'b>(&'a mut History<'b>);
|
||||
impl<'de> DeserializeSeed<'de> for RootSeed<'_, '_> {
|
||||
type Value = Value;
|
||||
fn deserialize<D: de::Deserializer<'de>>(self, deserializer: D) -> Result<Value, D::Error> {
|
||||
deserializer.deserialize_map(RootVisitor(self.0))
|
||||
}
|
||||
}
|
||||
struct RootVisitor<'a, 'b>(&'a mut History<'b>);
|
||||
impl<'de> Visitor<'de> for RootVisitor<'_, '_> {
|
||||
type Value = Value;
|
||||
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str("backup envelope")
|
||||
}
|
||||
fn visit_map<M: MapAccess<'de>>(self, mut map: M) -> Result<Value, M::Error> {
|
||||
let mut root = Map::new();
|
||||
while let Some(key) = map.next_key::<String>()? {
|
||||
(self.0.check)().map_err(de::Error::custom)?;
|
||||
let value = if key == "data" {
|
||||
map.next_value_seed(DataSeed(self.0))?
|
||||
} else {
|
||||
map.next_value::<Value>()?
|
||||
};
|
||||
root.insert(key, value);
|
||||
}
|
||||
Ok(root.into())
|
||||
}
|
||||
}
|
||||
struct DataSeed<'a, 'b>(&'a mut History<'b>);
|
||||
impl<'de> DeserializeSeed<'de> for DataSeed<'_, '_> {
|
||||
type Value = Value;
|
||||
fn deserialize<D: de::Deserializer<'de>>(self, deserializer: D) -> Result<Value, D::Error> {
|
||||
self.0.reset().map_err(de::Error::custom)?;
|
||||
deserializer.deserialize_map(DataVisitor(self.0))
|
||||
}
|
||||
}
|
||||
struct DataVisitor<'a, 'b>(&'a mut History<'b>);
|
||||
impl<'de> Visitor<'de> for DataVisitor<'_, '_> {
|
||||
type Value = Value;
|
||||
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str("backup data")
|
||||
}
|
||||
fn visit_map<M: MapAccess<'de>>(self, mut map: M) -> Result<Value, M::Error> {
|
||||
let mut data = Map::new();
|
||||
while let Some(key) = map.next_key::<String>()? {
|
||||
(self.0.check)().map_err(de::Error::custom)?;
|
||||
let value = if key == "history" {
|
||||
self.0.reset().map_err(de::Error::custom)?;
|
||||
if map.next_value_seed(HistorySeed(self.0))? {
|
||||
Value::Array(Vec::new())
|
||||
} else {
|
||||
Value::Null
|
||||
}
|
||||
} else {
|
||||
map.next_value::<Value>()?
|
||||
};
|
||||
data.insert(key, value);
|
||||
}
|
||||
Ok(data.into())
|
||||
}
|
||||
}
|
||||
struct HistorySeed<'a, 'b>(&'a mut History<'b>);
|
||||
impl<'de> DeserializeSeed<'de> for HistorySeed<'_, '_> {
|
||||
type Value = bool;
|
||||
fn deserialize<D: de::Deserializer<'de>>(self, deserializer: D) -> Result<bool, D::Error> {
|
||||
deserializer.deserialize_any(HistoryVisitor(self.0))
|
||||
}
|
||||
}
|
||||
struct HistoryVisitor<'a, 'b>(&'a mut History<'b>);
|
||||
impl<'de> Visitor<'de> for HistoryVisitor<'_, '_> {
|
||||
type Value = bool;
|
||||
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str("optional history array")
|
||||
}
|
||||
fn visit_seq<S: SeqAccess<'de>>(self, mut seq: S) -> Result<bool, S::Error> {
|
||||
while let Some(row) = seq.next_element::<Value>()? {
|
||||
(self.0.check)().map_err(de::Error::custom)?;
|
||||
if !row.is_object() {
|
||||
continue;
|
||||
}
|
||||
let encoded = serde_json::to_vec(&row).map_err(de::Error::custom)?;
|
||||
if encoded.len() as u64 > MAX_ROW_BYTES {
|
||||
return Err(de::Error::custom(
|
||||
"Legacy backup history row exceeds size limit",
|
||||
));
|
||||
}
|
||||
self.0.bytes += encoded.len() as u64 + 1;
|
||||
if self.0.bytes > MAX_HISTORY_BYTES {
|
||||
return Err(de::Error::custom(
|
||||
"Legacy backup history exceeds size limit",
|
||||
));
|
||||
}
|
||||
self.0
|
||||
.writer
|
||||
.write_all(&encoded)
|
||||
.and_then(|_| self.0.writer.write_all(b"\n"))
|
||||
.map_err(de::Error::custom)?;
|
||||
self.0.count += 1;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
fn visit_map<M: MapAccess<'de>>(self, mut map: M) -> Result<bool, M::Error> {
|
||||
while map.next_entry::<IgnoredAny, IgnoredAny>()?.is_some() {}
|
||||
Ok(false)
|
||||
}
|
||||
fn visit_bool<E: de::Error>(self, _: bool) -> Result<bool, E> {
|
||||
Ok(false)
|
||||
}
|
||||
fn visit_i64<E: de::Error>(self, _: i64) -> Result<bool, E> {
|
||||
Ok(false)
|
||||
}
|
||||
fn visit_u64<E: de::Error>(self, _: u64) -> Result<bool, E> {
|
||||
Ok(false)
|
||||
}
|
||||
fn visit_f64<E: de::Error>(self, _: f64) -> Result<bool, E> {
|
||||
Ok(false)
|
||||
}
|
||||
fn visit_str<E: de::Error>(self, _: &str) -> Result<bool, E> {
|
||||
Ok(false)
|
||||
}
|
||||
fn visit_none<E: de::Error>(self) -> Result<bool, E> {
|
||||
Ok(false)
|
||||
}
|
||||
fn visit_unit<E: de::Error>(self) -> Result<bool, E> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn split(request: &Value, check: Check<'_>) -> Result<Value, String> {
|
||||
let input = required(request, "input_path")?;
|
||||
if fs::metadata(input)
|
||||
.map_err(|e| err("legacy backup metadata", e))?
|
||||
.len()
|
||||
> MAX_HISTORY_BYTES + (8 << 20)
|
||||
{
|
||||
return Err("Legacy backup exceeds size limit".into());
|
||||
}
|
||||
let history_path = output_path(request, "ndjson_path")?;
|
||||
let metadata_path = output_path(request, "metadata_path")?;
|
||||
if history_path == metadata_path || history_path == input || metadata_path == input {
|
||||
return Err("Backup staging paths must be distinct".into());
|
||||
}
|
||||
let file = OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(history_path)
|
||||
.map_err(|e| err("create legacy history", e))?;
|
||||
let mut metadata_created = false;
|
||||
let result = (|| {
|
||||
let mut writer = BufWriter::new(file);
|
||||
let mut context = History {
|
||||
writer: &mut writer,
|
||||
count: 0,
|
||||
bytes: 0,
|
||||
check,
|
||||
};
|
||||
let input = File::open(input).map_err(|e| err("open legacy backup", e))?;
|
||||
let mut deserializer = serde_json::Deserializer::from_reader(BufReader::new(input));
|
||||
let root = RootSeed(&mut context)
|
||||
.deserialize(&mut deserializer)
|
||||
.map_err(|e| err("decode legacy backup", e))?;
|
||||
deserializer
|
||||
.end()
|
||||
.map_err(|e| err("legacy backup trailing data", e))?;
|
||||
if root["magic"] != "spotiflac-backup" || !root["data"].is_object() {
|
||||
return Err("Invalid legacy backup envelope".into());
|
||||
}
|
||||
let count = context.count;
|
||||
writer
|
||||
.flush()
|
||||
.and_then(|_| writer.get_ref().sync_all())
|
||||
.map_err(|e| err("publish legacy history", e))?;
|
||||
let encoded = serde_json::to_vec(&root).map_err(|e| err("encode legacy metadata", e))?;
|
||||
if encoded.len() > 8 << 20 {
|
||||
return Err("Legacy backup metadata exceeds size limit".into());
|
||||
}
|
||||
check()?;
|
||||
let mut metadata = OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(metadata_path)
|
||||
.map_err(|e| err("create legacy metadata", e))?;
|
||||
metadata_created = true;
|
||||
metadata
|
||||
.write_all(&encoded)
|
||||
.and_then(|_| metadata.sync_all())
|
||||
.map_err(|e| err("publish legacy metadata", e))?;
|
||||
check()?;
|
||||
Ok(
|
||||
json!({"published":true,"count":count,"history_path":history_path,"metadata_path":metadata_path}),
|
||||
)
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_file(history_path);
|
||||
if metadata_created {
|
||||
let _ = fs::remove_file(metadata_path);
|
||||
}
|
||||
}
|
||||
// A create_new conflict must never remove the owner's preexisting file.
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
use super::*;
|
||||
use std::cell::Cell;
|
||||
use tempfile::TempDir;
|
||||
|
||||
struct Fixture {
|
||||
dir: TempDir,
|
||||
db: Connection,
|
||||
request: Value,
|
||||
}
|
||||
fn track(id: &str) -> Value {
|
||||
json!({"id":id,"trackName":format!("Track {id}"),"artistName":"Artist","albumName":"Album","filePath":format!("/music/{id}.flac"),"service":"provider","downloadedAt":"2026-10-06T19:00:00.000+07:00","hasLyrics":true,"hasReplayGain":true})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actual_complete_dart_history_codec_and_path_identity_parity() {
|
||||
let fixture: Value = serde_json::from_str(include_str!(
|
||||
"../../../../../test/fixtures/native_backup_contract.json"
|
||||
))
|
||||
.unwrap();
|
||||
for row in fixture.as_array().unwrap() {
|
||||
let encoded = encode(&row["input"]).unwrap();
|
||||
assert_eq!(
|
||||
Value::Object(encoded.clone()),
|
||||
row["encoded"],
|
||||
"encoding {:?}",
|
||||
row["input"]["id"]
|
||||
);
|
||||
assert_eq!(
|
||||
Value::Object(decode(&encoded, &json!({}))),
|
||||
row["decoded"],
|
||||
"decoding {:?}",
|
||||
row["input"]["id"]
|
||||
);
|
||||
let expected = row["path_keys"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap().to_string())
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
build(row["input"]["filePath"].as_str().unwrap(), false).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ios_export_rebases_only_app_owned_paths_and_keeps_import_verbatim() {
|
||||
let old = "/var/mobile/Containers/Data/Application/AAAA/Documents/Music/track.flac";
|
||||
let current = "/var/mobile/Containers/Data/Application/BBBB/Documents";
|
||||
assert_eq!(
|
||||
rebase_ios(old, current),
|
||||
"/var/mobile/Containers/Data/Application/BBBB/Documents/Music/track.flac"
|
||||
);
|
||||
for path in [
|
||||
"/var/mobile/Containers/Data/Application/AAAA/tmp/a.flac",
|
||||
"/var/mobile/Containers/Data/Application/AAAA/Documents/../tmp/a.flac",
|
||||
"/external/Music/a.flac",
|
||||
] {
|
||||
assert_eq!(rebase_ios(path, current), path);
|
||||
}
|
||||
let mut row = track("old");
|
||||
row["filePath"] = old.into();
|
||||
assert_eq!(encode(&row).unwrap()["file_path"], old);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelling_restore_after_swap_keeps_old_rows_and_path_keys() {
|
||||
let fixture = Fixture::new();
|
||||
fixture.restore(&[track("old")], 1, &|| Ok(())).unwrap();
|
||||
let calls = Cell::new(0);
|
||||
let check = || {
|
||||
calls.set(calls.get() + 1);
|
||||
if calls.get() == 8 {
|
||||
Err("cancelled during restore".into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
assert!(fixture.restore(&[track("new")], 1, &check).is_err());
|
||||
assert_eq!(calls.get(), 8);
|
||||
assert_eq!(fixture.ids(), ["old"]);
|
||||
assert!(
|
||||
fixture
|
||||
.db
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM history_path_keys WHERE item_id='old'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0)
|
||||
)
|
||||
.unwrap()
|
||||
> 0
|
||||
);
|
||||
}
|
||||
impl Fixture {
|
||||
fn new() -> Self {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("history.db");
|
||||
let db = Connection::open(&path).unwrap();
|
||||
let values = encode(&track("schema")).unwrap();
|
||||
let columns = values
|
||||
.keys()
|
||||
.map(|key| {
|
||||
format!(
|
||||
"{key} {}",
|
||||
if key == "id" {
|
||||
"TEXT PRIMARY KEY"
|
||||
} else if key == "sort_added"
|
||||
|| key.ends_with("scan_version")
|
||||
|| ["saf_repaired", "explicit", "has_lyrics", "has_replaygain"]
|
||||
.contains(&key.as_str())
|
||||
{
|
||||
"INTEGER"
|
||||
} else {
|
||||
"TEXT"
|
||||
}
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
db.execute_batch(&format!("PRAGMA journal_mode=WAL;PRAGMA user_version=15;PRAGMA recursive_triggers=ON;CREATE TABLE history({columns});CREATE TABLE history_path_keys(item_id TEXT,path_key TEXT,PRIMARY KEY(item_id,path_key));CREATE VIRTUAL TABLE history_search_fts USING fts5(search_text,content='history',content_rowid='rowid',tokenize='trigram');CREATE TRIGGER history_fts_ai AFTER INSERT ON history BEGIN INSERT INTO history_search_fts(rowid,search_text) VALUES(new.rowid,new.search_text);END;CREATE TRIGGER history_fts_ad AFTER DELETE ON history BEGIN INSERT INTO history_search_fts(history_search_fts,rowid,search_text) VALUES('delete',old.rowid,old.search_text);END;")).unwrap();
|
||||
let request = json!({"history_path":path.to_str().unwrap(),"platform":"ios"});
|
||||
Self { dir, db, request }
|
||||
}
|
||||
fn restore(&self, rows: &[Value], expected: u64, check: Check<'_>) -> Result<Value, String> {
|
||||
let path = self.dir.path().join("input.ndjson");
|
||||
let mut file = File::create(&path).unwrap();
|
||||
for row in rows {
|
||||
writeln!(file, "{row}").unwrap();
|
||||
}
|
||||
let mut request = self.request.clone();
|
||||
request["operation"] = "backup_history_import".into();
|
||||
request["ndjson_path"] = path.to_str().unwrap().into();
|
||||
request["expected_count"] = expected.into();
|
||||
execute(&request, check)
|
||||
}
|
||||
fn ids(&self) -> Vec<String> {
|
||||
self.db
|
||||
.prepare("SELECT id FROM history ORDER BY id")
|
||||
.unwrap()
|
||||
.query_map([], |row| row.get(0))
|
||||
.unwrap()
|
||||
.map(Result::unwrap)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backup_replace_round_trip_preserves_flags_dates_keys_order_and_fts() {
|
||||
let fixture = Fixture::new();
|
||||
fixture.restore(&[track("old")], 1, &|| Ok(())).unwrap();
|
||||
fixture
|
||||
.restore(&[track("b"), track("a")], 2, &|| Ok(()))
|
||||
.unwrap();
|
||||
assert_eq!(fixture.ids(), ["a", "b"]);
|
||||
assert!(
|
||||
fixture
|
||||
.db
|
||||
.query_row("SELECT COUNT(*) FROM history_path_keys", [], |r| r
|
||||
.get::<_, i64>(0))
|
||||
.unwrap()
|
||||
> 0
|
||||
);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.db
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM history_search_fts WHERE history_search_fts MATCH ?",
|
||||
["old"],
|
||||
|r| r.get::<_, i64>(0)
|
||||
)
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
let output = fixture.dir.path().join("export.ndjson");
|
||||
let mut request = fixture.request.clone();
|
||||
request["operation"] = "backup_history_export".into();
|
||||
request["ndjson_path"] = output.to_str().unwrap().into();
|
||||
let result = execute(&request, &|| Ok(())).unwrap();
|
||||
assert_eq!(result["count"], 2);
|
||||
assert_eq!(result["published"], true);
|
||||
let rows = fs::read_to_string(output)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str::<Value>(line).unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(rows[0]["id"], "b");
|
||||
assert_eq!(rows[0]["downloadedAt"], "2026-10-06T12:00:00.000Z");
|
||||
assert_eq!(rows[0]["hasLyrics"], true);
|
||||
assert_eq!(rows[0]["hasReplayGain"], true);
|
||||
fixture.restore(&[], 0, &|| Ok(())).unwrap();
|
||||
assert!(fixture.ids().is_empty());
|
||||
fixture.restore(&rows, 2, &|| Ok(())).unwrap();
|
||||
assert_eq!(fixture.ids(), ["a", "b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_count_cancel_and_invalid_rows_keep_history_and_keys() {
|
||||
for failure in ["count", "invalid", "cancel"] {
|
||||
let fixture = Fixture::new();
|
||||
fixture.restore(&[track("old")], 1, &|| Ok(())).unwrap();
|
||||
let old_keys = fixture
|
||||
.db
|
||||
.query_row("SELECT COUNT(*) FROM history_path_keys", [], |r| {
|
||||
r.get::<_, i64>(0)
|
||||
})
|
||||
.unwrap();
|
||||
let mut rows = (0..550)
|
||||
.map(|i| track(&format!("new-{i}")))
|
||||
.collect::<Vec<_>>();
|
||||
if failure == "invalid" {
|
||||
rows[540]["filePath"] = Value::Null;
|
||||
}
|
||||
let calls = Cell::new(0);
|
||||
let check = || {
|
||||
calls.set(calls.get() + 1);
|
||||
if failure == "cancel" && calls.get() > 520 {
|
||||
Err("cancelled".into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
assert!(
|
||||
fixture
|
||||
.restore(&rows, if failure == "count" { 549 } else { 550 }, &check)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(fixture.ids(), ["old"]);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.db
|
||||
.query_row("SELECT COUNT(*) FROM history_path_keys", [], |r| r
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
old_keys
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_replaces_ids_without_clearing_unrelated_downloads() {
|
||||
let fixture = Fixture::new();
|
||||
fixture.restore(&[track("old")], 1, &|| Ok(())).unwrap();
|
||||
let input = fixture.dir.path().join("merge.ndjson");
|
||||
fs::write(&input, format!("{}\n", track("new"))).unwrap();
|
||||
let mut request = fixture.request.clone();
|
||||
request["operation"] = "backup_history_import".into();
|
||||
request["ndjson_path"] = input.to_str().unwrap().into();
|
||||
request["mode"] = "merge".into();
|
||||
execute(&request, &|| Ok(())).unwrap();
|
||||
assert_eq!(fixture.ids(), ["new", "old"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_split_streams_history_and_retains_category_semantics() {
|
||||
let fixture = Fixture::new();
|
||||
let input = fixture.dir.path().join("backup.json");
|
||||
let history = fixture.dir.path().join("legacy.ndjson");
|
||||
let metadata = fixture.dir.path().join("legacy.json");
|
||||
let original = json!({"magic":"spotiflac-backup","format_version":1,"app_version":"5.0","data":{"settings":{"theme":"light"},"history":[track("one"),12,track("two")],"collections":{"loved":["one"]}}});
|
||||
fs::write(&input, original.to_string()).unwrap();
|
||||
let request = json!({"operation":"backup_split_legacy","input_path":input.to_str().unwrap(),"ndjson_path":history.to_str().unwrap(),"metadata_path":metadata.to_str().unwrap()});
|
||||
let result = execute(&request, &|| Ok(())).unwrap();
|
||||
assert_eq!(result["count"], 2);
|
||||
assert_eq!(result["published"], true);
|
||||
let parsed: Value = serde_json::from_reader(File::open(metadata).unwrap()).unwrap();
|
||||
assert_eq!(parsed["data"]["history"], json!([]));
|
||||
assert_eq!(parsed["data"]["settings"], original["data"]["settings"]);
|
||||
assert_eq!(fs::read_to_string(history).unwrap().lines().count(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_duplicate_history_uses_last_field_and_invalid_backup_cleans_output() {
|
||||
let fixture = Fixture::new();
|
||||
let input = fixture.dir.path().join("backup.json");
|
||||
let history = fixture.dir.path().join("legacy.ndjson");
|
||||
let metadata = fixture.dir.path().join("legacy.json");
|
||||
let request = json!({"operation":"backup_split_legacy","input_path":input.to_str().unwrap(),"ndjson_path":history.to_str().unwrap(),"metadata_path":metadata.to_str().unwrap()});
|
||||
fs::write(
|
||||
&input,
|
||||
format!(
|
||||
"{{\"magic\":\"spotiflac-backup\",\"data\":{{\"history\":[{}],\"history\":[]}}}}",
|
||||
track("old")
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(execute(&request, &|| Ok(())).unwrap()["count"], 0);
|
||||
fs::remove_file(&history).unwrap();
|
||||
fs::remove_file(&metadata).unwrap();
|
||||
fs::write(&input, "{\"magic\":\"wrong\",\"data\":{\"history\":[]}}").unwrap();
|
||||
assert!(execute(&request, &|| Ok(())).is_err());
|
||||
assert!(!history.exists());
|
||||
assert!(!metadata.exists());
|
||||
}
|
||||
@@ -28,6 +28,21 @@ pub(crate) fn dart_lower(value: &str) -> String {
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn dart_upper(value: &str) -> String {
|
||||
value
|
||||
.chars()
|
||||
.map(|ch| {
|
||||
let mut upper = ch.to_uppercase();
|
||||
let first = upper.next().unwrap_or(ch);
|
||||
if upper.next().is_some() { ch } else { first }
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn regex_whitespace(ch: char) -> bool {
|
||||
matches!(ch,'\u{0009}'..='\u{000d}'|'\u{0020}'|'\u{00a0}'|'\u{1680}'|'\u{2000}'..='\u{200a}'|'\u{2028}'|'\u{2029}'|'\u{202f}'|'\u{205f}'|'\u{3000}'|'\u{feff}')
|
||||
}
|
||||
|
||||
fn decode(value: &str) -> Option<String> {
|
||||
let bytes = value.as_bytes();
|
||||
let mut decoded = Vec::with_capacity(bytes.len());
|
||||
|
||||
+1006
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,195 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:shared_preferences/shared_preferences.dart';
|
||||
import 'package:sqflite/sqflite.dart';
|
||||
import 'package:spotiflac_android/models/download_history.dart';
|
||||
import 'package:spotiflac_android/services/history_database.dart';
|
||||
import 'package:spotiflac_android/utils/path_match_keys.dart';
|
||||
|
||||
void main() {
|
||||
TestWidgetsFlutterBinding.ensureInitialized();
|
||||
test(
|
||||
'native backup fixture matches actual complete HistoryDatabase codecs',
|
||||
() async {
|
||||
SharedPreferences.setMockInitialValues({});
|
||||
final factory = databaseFactoryOrNull;
|
||||
databaseFactory = databaseFactorySqflitePlugin;
|
||||
addTearDown(() => databaseFactory = factory);
|
||||
final messenger =
|
||||
TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger;
|
||||
const paths = MethodChannel('plugins.flutter.io/path_provider');
|
||||
const sqlite = MethodChannel('com.tekartik.sqflite');
|
||||
messenger.setMockMethodCallHandler(paths, (_) async => '/test/documents');
|
||||
addTearDown(() => messenger.setMockMethodCallHandler(paths, null));
|
||||
addTearDown(() => messenger.setMockMethodCallHandler(sqlite, null));
|
||||
Map<String, dynamic>? encoded;
|
||||
messenger.setMockMethodCallHandler(sqlite, (call) async {
|
||||
final args = (call.arguments as Map?) ?? {};
|
||||
if (call.method == 'openDatabase') return {'id': 1};
|
||||
final sql = args['sql'] as String? ?? '';
|
||||
if (call.method == 'execute' && sql.contains('CREATE VIRTUAL TABLE')) {
|
||||
throw PlatformException(code: 'sqlite_error', message: 'no fts5');
|
||||
}
|
||||
if (call.method == 'batch') {
|
||||
for (final operation in args['operations'] as List) {
|
||||
final op = operation as Map;
|
||||
final statement = op['sql'] as String? ?? '';
|
||||
if (statement.startsWith('INSERT OR REPLACE INTO history (')) {
|
||||
final columnText = statement.substring(
|
||||
statement.indexOf('(') + 1,
|
||||
statement.indexOf(')'),
|
||||
);
|
||||
final columns = columnText
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.toList();
|
||||
final values = op['arguments'] as List;
|
||||
final placeholders = statement
|
||||
.substring(
|
||||
statement.lastIndexOf('(') + 1,
|
||||
statement.lastIndexOf(')'),
|
||||
)
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.toList();
|
||||
var argument = 0;
|
||||
encoded = {
|
||||
for (var i = 0; i < columns.length; i++)
|
||||
columns[i]: placeholders[i] == '?'
|
||||
? values[argument++]
|
||||
: null,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (call.method != 'query') return null;
|
||||
if (sql.toLowerCase().contains('pragma user_version')) {
|
||||
return {
|
||||
'columns': ['user_version'],
|
||||
'rows': [
|
||||
[HistoryDatabase.schemaVersion],
|
||||
],
|
||||
};
|
||||
}
|
||||
if (sql.startsWith('SELECT * FROM history ') && encoded != null) {
|
||||
return {
|
||||
'columns': encoded!.keys.toList(),
|
||||
'rows': [encoded!.values.toList()],
|
||||
};
|
||||
}
|
||||
return {'columns': <String>[], 'rows': <List<Object?>>[]};
|
||||
});
|
||||
|
||||
final complete = DownloadHistoryItem(
|
||||
id: 'complete',
|
||||
trackName: ' CAFÉ İ ΑΣ ß ',
|
||||
artistName: ' Artist ',
|
||||
albumName: ' Album ',
|
||||
albumArtist: ' Album Artist ',
|
||||
coverUrl: 'https://example.test/cover.jpg',
|
||||
filePath: 'content://nas.provider/tree/root/document/Ab%252FC.flac',
|
||||
storageMode: 'saf',
|
||||
downloadTreeUri: 'content://nas.provider/tree/root',
|
||||
safRelativeDir: 'Album',
|
||||
safFileName: 'Track.flac',
|
||||
safRepaired: true,
|
||||
service: 'provider',
|
||||
downloadedAt: DateTime.utc(2026, 10, 6, 12, 0, 0, 123, 456),
|
||||
isrc: ' us-aa-a26 00001 ',
|
||||
spotifyId: ' Spotify:Track:ABC ',
|
||||
trackNumber: 3,
|
||||
totalTracks: 10,
|
||||
discNumber: 2,
|
||||
totalDiscs: 3,
|
||||
duration: 240,
|
||||
releaseDate: ' 2026-10-01 ',
|
||||
quality: 'lossless',
|
||||
bitDepth: 24,
|
||||
sampleRate: 96000,
|
||||
bitrate: 3456,
|
||||
format: 'flac',
|
||||
genre: ' Rock ',
|
||||
composer: 'Composer',
|
||||
label: 'Label',
|
||||
copyright: 'Copyright',
|
||||
explicit: true,
|
||||
hasLyrics: true,
|
||||
lyricsMetadataScanVersion: 2,
|
||||
hasReplayGain: true,
|
||||
replayGainMetadataScanVersion: 1,
|
||||
).toJson();
|
||||
final inputs = <Map<String, dynamic>>[
|
||||
complete,
|
||||
{
|
||||
...complete,
|
||||
'id': 'empty-artist',
|
||||
'albumArtist': '',
|
||||
'isrc': 'ß-ffi\u00a0\ufeff',
|
||||
'spotifyId': null,
|
||||
},
|
||||
{
|
||||
...complete,
|
||||
'id': 'numeric-flags',
|
||||
'explicit': 1,
|
||||
'safRepaired': 1,
|
||||
'hasLyrics': 1,
|
||||
'hasReplayGain': false,
|
||||
'replaygain_track_gain': ' -2.5 dB ',
|
||||
'lyricsMetadataScanVersion': 1.9,
|
||||
},
|
||||
{
|
||||
...complete,
|
||||
'id': 'legacy',
|
||||
'hasLyrics': null,
|
||||
'lyricsMetadataScanVersion': null,
|
||||
'replayGainMetadataScanVersion': null,
|
||||
'downloadedAt': '20261006T120000.123456789z',
|
||||
},
|
||||
{
|
||||
...complete,
|
||||
'id': 'overflow-date',
|
||||
'downloadedAt': '2026-10-32T25:61:00Z',
|
||||
},
|
||||
{
|
||||
...complete,
|
||||
'id': 'short-time',
|
||||
'downloadedAt': '2026-10-06T00Z',
|
||||
'albumArtist': null,
|
||||
},
|
||||
{
|
||||
...complete,
|
||||
'id': 'invalid-date',
|
||||
'downloadedAt': 'invalid',
|
||||
'hasReplayGain': false,
|
||||
'replaygain_album_gain': 'NaN',
|
||||
'filePath': '/Music/a%zz.flac',
|
||||
},
|
||||
];
|
||||
final actual = <Map<String, dynamic>>[];
|
||||
for (final input in inputs) {
|
||||
encoded = null;
|
||||
await HistoryDatabase.instance.upsertBatch([input]);
|
||||
expect(encoded, isNotNull);
|
||||
final decoded = (await HistoryDatabase.instance.getAll()).single;
|
||||
actual.add({
|
||||
'input': input,
|
||||
'encoded': encoded,
|
||||
'decoded': decoded,
|
||||
'path_keys': buildPathMatchKeys(input['filePath'] as String).toList()
|
||||
..sort(),
|
||||
});
|
||||
}
|
||||
final file = File('test/fixtures/native_backup_contract.json');
|
||||
if (const bool.fromEnvironment('NATIVE_BACKUP_CAPTURE')) {
|
||||
file.writeAsStringSync(
|
||||
'${const JsonEncoder.withIndent(' ').convert(actual)}\n',
|
||||
);
|
||||
}
|
||||
expect(actual, jsonDecode(file.readAsStringSync()));
|
||||
},
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user