feat(backup): stream versioned ZIP archives

This commit is contained in:
zarzet
2026-08-30 23:19:09 +07:00
parent 2c9881944d
commit 897fb7d523
5 changed files with 449 additions and 105 deletions
@@ -1064,6 +1064,23 @@ class DownloadHistoryNotifier extends Notifier<DownloadHistoryState> {
}
await reloadFromStorage();
}
/// Restores a large v2 backup without retaining the complete history in
/// Dart memory. SQLite writes are grouped to keep JNI/channel overhead low.
Future<void> restoreFromBackupStream(
Stream<Map<String, dynamic>> items,
) async {
await _db.clearAll();
var batch = <Map<String, dynamic>>[];
await for (final item in items) {
batch.add(item);
if (batch.length < 500) continue;
await _db.upsertBatch(batch);
batch = <Map<String, dynamic>>[];
}
if (batch.isNotEmpty) await _db.upsertBatch(batch);
await reloadFromStorage();
}
}
final downloadHistoryProvider =
@@ -1145,6 +1145,34 @@ class LibraryCollectionsNotifier extends Notifier<LibraryCollectionsState> {
return covers;
}
/// Returns cover file references for the streaming ZIP backup format. The
/// backup writer reads each file directly instead of materializing every
/// image as base64 in memory.
Future<Map<String, Map<String, String>>> exportPlaylistCoverFiles() async {
await _ensureLoaded();
final covers = <String, Map<String, String>>{};
final playlists = List<UserPlaylistCollection>.of(state.playlists);
for (final playlist in playlists) {
var path = playlist.coverImagePath;
if (path == null || path.isEmpty) continue;
try {
if (await _playlistCoverNeedsNormalization(path)) {
await setPlaylistCover(playlist.id, path);
path = state.playlistById(playlist.id)?.coverImagePath ?? path;
}
final file = File(path);
if (!await file.exists() || await file.length() == 0) continue;
covers[playlist.id] = {
'ext': p.extension(path).toLowerCase(),
'path': path,
};
} catch (_) {
// Skip unreadable cover; the rest of the backup still succeeds.
}
}
return covers;
}
/// Replaces all collections (wishlist, loved, playlists, favorite artists)
/// with the contents of a backup. [collectionsJson] uses the
/// [LibraryCollectionsState.toJson] shape; [coverImages] is the map produced
@@ -1169,15 +1197,21 @@ class LibraryCollectionsNotifier extends Notifier<LibraryCollectionsState> {
: null;
if (id != null && coverEntry is Map) {
final data = coverEntry['data'] as String?;
final restoredPath = coverEntry['path'] as String?;
final ext = (coverEntry['ext'] as String?) ?? '.jpg';
if (data != null && data.isNotEmpty) {
if ((data != null && data.isNotEmpty) ||
(restoredPath != null && restoredPath.isNotEmpty)) {
try {
final sourcePath = p.join(
coversDir.path,
'.$id.restore${ext.startsWith('.') ? ext : '.$ext'}',
);
final source = File(sourcePath);
await source.writeAsBytes(base64Decode(data), flush: true);
if (restoredPath != null && restoredPath.isNotEmpty) {
await File(restoredPath).copy(source.path);
} else {
await source.writeAsBytes(base64Decode(data!), flush: true);
}
try {
newCoverPath = await _normalizePlaylistCoverFile(
id,
+20 -13
View File
@@ -38,24 +38,22 @@ class _BackupRestorePageState extends ConsumerState<BackupRestorePage> {
final messenger = ScaffoldMessenger.of(context);
try {
final settings = ref.read(settingsProvider).toJson();
final history = await HistoryDatabase.instance.getAll();
final collectionsNotifier = ref.read(libraryCollectionsProvider.notifier);
final collections = await collectionsNotifier.exportCollections();
final covers = await collectionsNotifier.exportPlaylistCovers();
final covers = await collectionsNotifier.exportPlaylistCoverFiles();
final extensions = await ref
.read(extensionProvider.notifier)
.exportBackup(includeSecrets: _includeSecrets);
final envelope = BackupService.buildEnvelope(
final file = await BackupService.writeBackupArchive(
settings: settings,
history: history,
loadHistoryPage: (limit, offset) =>
HistoryDatabase.instance.getAll(limit: limit, offset: offset),
collections: collections,
playlistCovers: covers,
playlistCoverFiles: covers,
extensions: extensions,
);
final file = await BackupService.writeBackupFile(envelope);
messenger.showSnackBar(SnackBar(content: Text(l10n.backupCreated)));
await SharePlus.instance.share(
@@ -74,29 +72,37 @@ class _BackupRestorePageState extends ConsumerState<BackupRestorePage> {
final l10n = context.l10n;
final messenger = ScaffoldMessenger.of(context);
String? content;
BackupBundle? bundle;
try {
final picked = await FilePicker.pickFile(
type: FileType.custom,
allowedExtensions: ['json', BackupService.fileExtension],
);
if (picked == null) return;
content = utf8.decode(await picked.readAsBytes());
final path = picked.path;
bundle = path != null
? await BackupService.parseFile(path)
: BackupService.parse(utf8.decode(await picked.readAsBytes()));
} catch (e) {
_log.e('Failed to read backup file: $e');
messenger.showSnackBar(SnackBar(content: Text(l10n.backupInvalidFile)));
return;
}
final bundle = BackupService.parse(content);
if (bundle == null) {
messenger.showSnackBar(SnackBar(content: Text(l10n.backupInvalidFile)));
return;
}
if (!mounted) return;
if (!mounted) {
await bundle.cleanup();
return;
}
final confirmed = await _confirmRestore(bundle);
if (confirmed != true || !mounted) return;
if (confirmed != true || !mounted) {
await bundle.cleanup();
return;
}
setState(() => _isImporting = true);
try {
@@ -107,7 +113,7 @@ class _BackupRestorePageState extends ConsumerState<BackupRestorePage> {
}
await ref
.read(downloadHistoryProvider.notifier)
.restoreFromBackup(bundle.history);
.restoreFromBackupStream(bundle.streamHistory());
await ref
.read(libraryCollectionsProvider.notifier)
.restoreFromBackup(
@@ -141,6 +147,7 @@ class _BackupRestorePageState extends ConsumerState<BackupRestorePage> {
_log.e('Failed to restore backup: $e', e, stack);
messenger.showSnackBar(SnackBar(content: Text(l10n.backupRestoreFailed)));
} finally {
await bundle.cleanup();
if (mounted) setState(() => _isImporting = false);
}
}
+302 -90
View File
@@ -1,32 +1,29 @@
import 'dart:convert';
import 'dart:io';
import 'package:archive/archive_io.dart';
import 'package:path/path.dart' as p;
import 'package:path_provider/path_provider.dart';
import 'package:spotiflac_android/constants/app_info.dart';
import 'package:spotiflac_android/utils/logger.dart';
/// Parsed contents of a backup file.
typedef BackupHistoryPageLoader =
Future<List<Map<String, dynamic>>> Function(int limit, int offset);
/// Parsed contents of a backup file. Version 2 keeps large history and cover
/// payloads on disk until restore consumes them.
class BackupBundle {
final int formatVersion;
final String appVersion;
final DateTime? createdAt;
/// Raw `AppSettings.toJson()` map, or null when not present.
final Map<String, dynamic>? settings;
/// History items in `DownloadHistoryItem.toJson()` shape.
final List<Map<String, dynamic>> history;
/// Collections in `LibraryCollectionsState.toJson()` shape
/// (wishlist / loved / playlists / favoriteArtists).
final Map<String, dynamic> collections;
/// Playlist cover images keyed by playlist id: `{ id: { ext, data } }`.
final Map<String, dynamic> playlistCovers;
/// Extensions section: `{ registry_url, items: [ {id, version, enabled, settings} ] }`.
final Map<String, dynamic> extensions;
final String? _historyNdjsonPath;
final int? _historyCount;
final String? _temporaryDirectoryPath;
const BackupBundle({
required this.formatVersion,
@@ -37,11 +34,38 @@ class BackupBundle {
required this.collections,
required this.playlistCovers,
required this.extensions,
});
String? historyNdjsonPath,
int? historyCount,
String? temporaryDirectoryPath,
}) : _historyNdjsonPath = historyNdjsonPath,
_historyCount = historyCount,
_temporaryDirectoryPath = temporaryDirectoryPath;
bool get hasSettings => settings != null && settings!.isNotEmpty;
int get historyCount => _historyCount ?? history.length;
int get historyCount => history.length;
Stream<Map<String, dynamic>> streamHistory() async* {
if (_historyNdjsonPath == null) {
yield* Stream.fromIterable(history);
return;
}
await for (final line in File(
_historyNdjsonPath,
).openRead().transform(utf8.decoder).transform(const LineSplitter())) {
if (line.trim().isEmpty) continue;
final decoded = jsonDecode(line);
if (decoded is Map) yield Map<String, dynamic>.from(decoded);
}
}
Future<void> cleanup() async {
final path = _temporaryDirectoryPath;
if (path == null || path.isEmpty) return;
try {
final directory = Directory(path);
if (await directory.exists()) await directory.delete(recursive: true);
} catch (_) {}
}
int _collectionListCount(String key) {
final value = collections[key];
@@ -59,7 +83,6 @@ class BackupBundle {
}
bool get hasExtensions => extensionCount > 0;
bool get isEmpty =>
!hasSettings &&
historyCount == 0 &&
@@ -70,19 +93,23 @@ class BackupBundle {
extensionCount == 0;
}
/// Builds and parses SpotiFLAC backup files (a single JSON document containing
/// settings, download history and the user library).
class BackupService {
static final _log = AppLogger('BackupService');
static const String magic = 'spotiflac-backup';
static const int formatVersion = 1;
static const String fileExtension = 'json';
static const int formatVersion = 2;
static const String fileExtension = 'sflbackup';
static const int _historyPageSize = 500;
static const int _maxMetadataBytes = 8 << 20;
static const int _maxHistoryBytes = 512 << 20;
static const int _maxCoverBytes = 20 << 20;
static const int _maxAllCoversBytes = 256 << 20;
static String encode(Map<String, dynamic> envelope) =>
const JsonEncoder.withIndent(' ').convert(envelope);
/// Builds the backup envelope written to disk.
/// Legacy JSON envelope retained so backups from earlier releases and unit
/// tests remain readable. New backups are written by [writeBackupArchive].
static Map<String, dynamic> buildEnvelope({
required Map<String, dynamic>? settings,
required List<Map<String, dynamic>> history,
@@ -92,7 +119,7 @@ class BackupService {
}) {
return {
'magic': magic,
'format_version': formatVersion,
'format_version': 1,
'app': 'SpotiFLAC Mobile',
'app_version': AppInfo.displayVersion,
'created_at': DateTime.now().toIso8601String(),
@@ -106,29 +133,248 @@ class BackupService {
};
}
/// Writes [envelope] to a timestamped file under the app documents directory
/// and returns the created file.
static Future<File> writeBackupArchive({
required Map<String, dynamic>? settings,
required BackupHistoryPageLoader loadHistoryPage,
required Map<String, dynamic> collections,
required Map<String, Map<String, String>> playlistCoverFiles,
required Map<String, dynamic> extensions,
Directory? outputDirectory,
Directory? temporaryDirectory,
}) async {
final output = await _newBackupFile(outputDirectory);
final tempRoot = temporaryDirectory ?? await getTemporaryDirectory();
final staging = await Directory(
p.join(
tempRoot.path,
'spotiflac_backup_${DateTime.now().microsecondsSinceEpoch}',
),
).create(recursive: true);
final historyFile = File(p.join(staging.path, 'history.ndjson'));
final metadataFile = File(p.join(staging.path, 'metadata.json'));
final partFile = File('${output.path}.part');
ZipFileEncoder? encoder;
try {
var historyCount = 0;
var offset = 0;
final historySink = historyFile.openWrite();
try {
while (true) {
final page = await loadHistoryPage(_historyPageSize, offset);
for (final item in page) {
historySink.writeln(jsonEncode(item));
}
historyCount += page.length;
offset += page.length;
if (page.length < _historyPageSize) break;
}
await historySink.flush();
} finally {
await historySink.close();
}
final coverManifest = <String, Map<String, String>>{};
var coverIndex = 0;
for (final entry in playlistCoverFiles.entries) {
final sourcePath = entry.value['path'] ?? '';
final source = File(sourcePath);
if (sourcePath.isEmpty || !await source.exists()) continue;
var ext = (entry.value['ext'] ?? p.extension(sourcePath)).toLowerCase();
if (!RegExp(r'^\.[a-z0-9]{1,8}$').hasMatch(ext)) ext = '.jpg';
final archiveName = 'covers/cover_${coverIndex++}$ext';
coverManifest[entry.key] = {'ext': ext, 'file': archiveName};
}
final metadata = {
'magic': magic,
'format_version': formatVersion,
'app': 'SpotiFLAC Mobile',
'app_version': AppInfo.displayVersion,
'created_at': DateTime.now().toIso8601String(),
'history_count': historyCount,
'data': {
'settings': settings,
'collections': collections,
'playlist_covers': coverManifest,
'extensions': extensions,
},
};
await metadataFile.writeAsString(jsonEncode(metadata), flush: true);
if (await partFile.exists()) await partFile.delete();
encoder = ZipFileEncoder()..create(partFile.path);
await encoder.addFile(metadataFile, 'metadata.json');
await encoder.addFile(historyFile, 'history.ndjson');
for (final entry in coverManifest.entries) {
final sourcePath = playlistCoverFiles[entry.key]?['path'];
if (sourcePath == null) continue;
await encoder.addFile(
File(sourcePath),
entry.value['file'],
ZipFileEncoder.store,
);
}
await encoder.close();
encoder = null;
if (await output.exists()) await output.delete();
await partFile.rename(output.path);
_log.i('Streaming backup written to ${output.path}');
return output;
} finally {
if (encoder != null) {
try {
await encoder.close();
} catch (_) {}
}
try {
if (await staging.exists()) await staging.delete(recursive: true);
} catch (_) {}
}
}
static Future<File> _newBackupFile([Directory? outputDirectory]) async {
final dir = outputDirectory ?? await getApplicationDocumentsDirectory();
final backupsDir =
outputDirectory ?? Directory(p.join(dir.path, 'backups'));
await backupsDir.create(recursive: true);
final now = DateTime.now();
String two(int value) => value.toString().padLeft(2, '0');
final stamp =
'${now.year}${two(now.month)}${two(now.day)}_'
'${two(now.hour)}${two(now.minute)}${two(now.second)}';
return File(
p.join(backupsDir.path, 'spotiflac_mobile_backup_$stamp.$fileExtension'),
);
}
/// Legacy JSON writer retained for compatibility with callers outside the
/// settings UI. It no longer defines the default backup format.
static Future<File> writeBackupFile(Map<String, dynamic> envelope) async {
final dir = await getApplicationDocumentsDirectory();
final backupsDir = Directory(p.join(dir.path, 'backups'));
if (!await backupsDir.exists()) {
await backupsDir.create(recursive: true);
final output = await _newBackupFile();
await output.writeAsString(encode(envelope), flush: true);
return output;
}
static Future<BackupBundle?> parseFile(
String path, {
Directory? temporaryDirectory,
}) async {
final file = File(path);
final header = await file
.openRead(0, 4)
.fold<List<int>>(<int>[], (bytes, chunk) => bytes..addAll(chunk));
final isZip =
header.length == 4 &&
header[0] == 0x50 &&
header[1] == 0x4b &&
header[2] == 0x03 &&
header[3] == 0x04;
return isZip
? _parseArchive(file, temporaryDirectory: temporaryDirectory)
: parse(await file.readAsString());
}
static Future<BackupBundle?> _parseArchive(
File file, {
Directory? temporaryDirectory,
}) async {
InputFileStream? input;
Archive? archive;
Directory? extractionDir;
try {
input = InputFileStream(file.path);
archive = ZipDecoder().decodeStream(input);
final metadataEntry = archive.find('metadata.json');
final historyEntry = archive.find('history.ndjson');
if (metadataEntry == null ||
metadataEntry.size > _maxMetadataBytes ||
historyEntry == null ||
historyEntry.size > _maxHistoryBytes) {
return null;
}
final rootRaw = jsonDecode(utf8.decode(metadataEntry.content));
if (rootRaw is! Map) return null;
final root = Map<String, dynamic>.from(rootRaw);
if (root['magic'] != magic || root['format_version'] != formatVersion) {
return null;
}
final dataRaw = root['data'];
if (dataRaw is! Map) return null;
final data = Map<String, dynamic>.from(dataRaw);
final tempRoot = temporaryDirectory ?? await getTemporaryDirectory();
extractionDir = await Directory(
p.join(
tempRoot.path,
'spotiflac_restore_${DateTime.now().microsecondsSinceEpoch}',
),
).create(recursive: true);
final historyPath = p.join(extractionDir.path, 'history.ndjson');
final historyOutput = OutputFileStream(historyPath);
historyEntry.writeContent(historyOutput);
historyOutput.closeSync();
final restoredCovers = <String, dynamic>{};
final coverManifest = data['playlist_covers'];
if (coverManifest is Map) {
var index = 0;
var extractedCoverBytes = 0;
for (final manifestEntry in coverManifest.entries) {
if (manifestEntry.value is! Map) continue;
final cover = Map<String, dynamic>.from(manifestEntry.value as Map);
final archiveName = cover['file']?.toString() ?? '';
if (!archiveName.startsWith('covers/') ||
archiveName.contains('..')) {
continue;
}
final archiveEntry = archive.find(archiveName);
if (archiveEntry == null || archiveEntry.size > _maxCoverBytes) {
continue;
}
if (extractedCoverBytes + archiveEntry.size > _maxAllCoversBytes) {
break;
}
var ext = cover['ext']?.toString() ?? '.jpg';
if (!RegExp(r'^\.[a-z0-9]{1,8}$').hasMatch(ext)) ext = '.jpg';
final coverPath = p.join(extractionDir.path, 'cover_${index++}$ext');
final output = OutputFileStream(coverPath);
archiveEntry.writeContent(output);
output.closeSync();
extractedCoverBytes += archiveEntry.size;
restoredCovers[manifestEntry.key.toString()] = {
'ext': ext,
'path': coverPath,
};
}
}
return BackupBundle(
formatVersion: formatVersion,
appVersion: root['app_version'] as String? ?? '',
createdAt: DateTime.tryParse(root['created_at'] as String? ?? ''),
settings: _mapOrNull(data['settings']),
history: const [],
historyNdjsonPath: historyPath,
historyCount: (root['history_count'] as num?)?.toInt() ?? 0,
collections: _mapOrEmpty(data['collections']),
playlistCovers: restoredCovers,
extensions: _mapOrEmpty(data['extensions']),
temporaryDirectoryPath: extractionDir.path,
);
} catch (e) {
_log.w('Backup archive parse failed: $e');
if (extractionDir != null) {
try {
await extractionDir.delete(recursive: true);
} catch (_) {}
}
return null;
} finally {
if (input != null) await input.close();
}
final now = DateTime.now();
String two(int v) => v.toString().padLeft(2, '0');
final stamp =
'${now.year}${two(now.month)}${two(now.day)}_${two(now.hour)}${two(now.minute)}${two(now.second)}';
final fileName = 'spotiflac_backup_$stamp.$fileExtension';
final file = File(p.join(backupsDir.path, fileName));
await file.writeAsString(encode(envelope), flush: true);
_log.i('Backup written to ${file.path}');
return file;
}
/// Parses and validates a backup file's contents. Returns null when the
/// content is not a recognizable SpotiFLAC backup.
static BackupBundle? parse(String content) {
dynamic decoded;
try {
@@ -137,65 +383,31 @@ class BackupService {
_log.w('Backup parse failed: not valid JSON ($e)');
return null;
}
if (decoded is! Map) {
_log.w('Backup parse failed: root is not an object');
return null;
}
if (decoded is! Map) return null;
final root = Map<String, dynamic>.from(decoded);
if (root['magic'] != magic) {
_log.w('Backup parse failed: magic marker missing');
return null;
}
final dataRaw = root['data'];
if (dataRaw is! Map) {
_log.w('Backup parse failed: missing data section');
return null;
}
final data = Map<String, dynamic>.from(dataRaw);
Map<String, dynamic>? settings;
final settingsRaw = data['settings'];
if (settingsRaw is Map) {
settings = Map<String, dynamic>.from(settingsRaw);
}
if (root['magic'] != magic || root['data'] is! Map) return null;
final data = Map<String, dynamic>.from(root['data'] as Map);
final history = <Map<String, dynamic>>[];
final historyRaw = data['history'];
if (historyRaw is List) {
for (final item in historyRaw) {
if (item is Map) {
history.add(Map<String, dynamic>.from(item));
}
if (data['history'] is List) {
for (final item in data['history'] as List) {
if (item is Map) history.add(Map<String, dynamic>.from(item));
}
}
final collectionsRaw = data['collections'];
final collections = collectionsRaw is Map
? Map<String, dynamic>.from(collectionsRaw)
: <String, dynamic>{};
final coversRaw = data['playlist_covers'];
final playlistCovers = coversRaw is Map
? Map<String, dynamic>.from(coversRaw)
: <String, dynamic>{};
final extensionsRaw = data['extensions'];
final extensions = extensionsRaw is Map
? Map<String, dynamic>.from(extensionsRaw)
: <String, dynamic>{};
return BackupBundle(
formatVersion: (root['format_version'] as num?)?.toInt() ?? 1,
appVersion: root['app_version'] as String? ?? '',
createdAt: DateTime.tryParse(root['created_at'] as String? ?? ''),
settings: settings,
settings: _mapOrNull(data['settings']),
history: history,
collections: collections,
playlistCovers: playlistCovers,
extensions: extensions,
collections: _mapOrEmpty(data['collections']),
playlistCovers: _mapOrEmpty(data['playlist_covers']),
extensions: _mapOrEmpty(data['extensions']),
);
}
static Map<String, dynamic>? _mapOrNull(dynamic value) =>
value is Map ? Map<String, dynamic>.from(value) : null;
static Map<String, dynamic> _mapOrEmpty(dynamic value) =>
value is Map ? Map<String, dynamic>.from(value) : <String, dynamic>{};
}
+74
View File
@@ -0,0 +1,74 @@
import 'dart:io';
import 'package:flutter_test/flutter_test.dart';
import 'package:path/path.dart' as p;
import 'package:spotiflac_android/services/backup_service.dart';
void main() {
test('ZIP v2 pages history and streams cover files during restore', () async {
final root = await Directory.systemTemp.createTemp(
'spotiflac-backup-test-',
);
addTearDown(() async {
if (await root.exists()) await root.delete(recursive: true);
});
final output = await Directory(p.join(root.path, 'output')).create();
final temporary = await Directory(p.join(root.path, 'temporary')).create();
final cover = File(p.join(root.path, 'cover.jpg'));
await cover.writeAsBytes([0xff, 0xd8, 0xff, 0xd9]);
final history = List.generate(
1203,
(index) => <String, dynamic>{'id': 'item-$index', 'value': index},
);
final offsets = <int>[];
final file = await BackupService.writeBackupArchive(
settings: const {'theme': 'dark'},
loadHistoryPage: (limit, offset) async {
offsets.add(offset);
if (offset >= history.length) {
return const <Map<String, dynamic>>[];
}
final end = (offset + limit).clamp(0, history.length);
return history.sublist(offset, end);
},
collections: const {
'loved': <dynamic>[],
'wishlist': <dynamic>[],
'playlists': <dynamic>[],
},
playlistCoverFiles: {
'playlist-1': {'ext': '.jpg', 'path': cover.path},
},
extensions: const {'items': <dynamic>[]},
outputDirectory: output,
temporaryDirectory: temporary,
);
expect(file.path, endsWith('.${BackupService.fileExtension}'));
expect(offsets, [0, 500, 1000]);
final bundle = await BackupService.parseFile(
file.path,
temporaryDirectory: temporary,
);
expect(bundle, isNotNull);
expect(bundle!.formatVersion, 2);
expect(bundle.historyCount, history.length);
final restoredHistory = await bundle.streamHistory().toList();
expect(restoredHistory, hasLength(history.length));
expect(restoredHistory.last['id'], 'item-1202');
final restoredCover = bundle.playlistCovers['playlist-1'];
expect(restoredCover, isA<Map<String, dynamic>>());
final restoredCoverPath =
(restoredCover as Map<String, dynamic>)['path'] as String;
expect(await File(restoredCoverPath).readAsBytes(), [
0xff,
0xd8,
0xff,
0xd9,
]);
await bundle.cleanup();
expect(await File(restoredCoverPath).exists(), isFalse);
});
}