From 897fb7d52387a83862d34c462a7ae174d9c5790a Mon Sep 17 00:00:00 2001 From: zarzet Date: Sun, 30 Aug 2026 23:19:09 +0700 Subject: [PATCH] feat(backup): stream versioned ZIP archives --- lib/providers/download_history_provider.dart | 17 + .../library_collections_provider.dart | 38 +- lib/screens/settings/backup_restore_page.dart | 33 +- lib/services/backup_service.dart | 392 ++++++++++++++---- test/backup_service_v2_test.dart | 74 ++++ 5 files changed, 449 insertions(+), 105 deletions(-) create mode 100644 test/backup_service_v2_test.dart diff --git a/lib/providers/download_history_provider.dart b/lib/providers/download_history_provider.dart index 5d0390ba..d9bf4640 100644 --- a/lib/providers/download_history_provider.dart +++ b/lib/providers/download_history_provider.dart @@ -1064,6 +1064,23 @@ class DownloadHistoryNotifier extends Notifier { } await reloadFromStorage(); } + + /// Restores a large v2 backup without retaining the complete history in + /// Dart memory. SQLite writes are grouped to keep JNI/channel overhead low. + Future restoreFromBackupStream( + Stream> items, + ) async { + await _db.clearAll(); + var batch = >[]; + await for (final item in items) { + batch.add(item); + if (batch.length < 500) continue; + await _db.upsertBatch(batch); + batch = >[]; + } + if (batch.isNotEmpty) await _db.upsertBatch(batch); + await reloadFromStorage(); + } } final downloadHistoryProvider = diff --git a/lib/providers/library_collections_provider.dart b/lib/providers/library_collections_provider.dart index a25b2637..d9278f60 100644 --- a/lib/providers/library_collections_provider.dart +++ b/lib/providers/library_collections_provider.dart @@ -1145,6 +1145,34 @@ class LibraryCollectionsNotifier extends Notifier { return covers; } + /// Returns cover file references for the streaming ZIP backup format. The + /// backup writer reads each file directly instead of materializing every + /// image as base64 in memory. + Future>> exportPlaylistCoverFiles() async { + await _ensureLoaded(); + final covers = >{}; + final playlists = List.of(state.playlists); + for (final playlist in playlists) { + var path = playlist.coverImagePath; + if (path == null || path.isEmpty) continue; + try { + if (await _playlistCoverNeedsNormalization(path)) { + await setPlaylistCover(playlist.id, path); + path = state.playlistById(playlist.id)?.coverImagePath ?? path; + } + final file = File(path); + if (!await file.exists() || await file.length() == 0) continue; + covers[playlist.id] = { + 'ext': p.extension(path).toLowerCase(), + 'path': path, + }; + } catch (_) { + // Skip unreadable cover; the rest of the backup still succeeds. + } + } + return covers; + } + /// Replaces all collections (wishlist, loved, playlists, favorite artists) /// with the contents of a backup. [collectionsJson] uses the /// [LibraryCollectionsState.toJson] shape; [coverImages] is the map produced @@ -1169,15 +1197,21 @@ class LibraryCollectionsNotifier extends Notifier { : null; if (id != null && coverEntry is Map) { final data = coverEntry['data'] as String?; + final restoredPath = coverEntry['path'] as String?; final ext = (coverEntry['ext'] as String?) ?? '.jpg'; - if (data != null && data.isNotEmpty) { + if ((data != null && data.isNotEmpty) || + (restoredPath != null && restoredPath.isNotEmpty)) { try { final sourcePath = p.join( coversDir.path, '.$id.restore${ext.startsWith('.') ? ext : '.$ext'}', ); final source = File(sourcePath); - await source.writeAsBytes(base64Decode(data), flush: true); + if (restoredPath != null && restoredPath.isNotEmpty) { + await File(restoredPath).copy(source.path); + } else { + await source.writeAsBytes(base64Decode(data!), flush: true); + } try { newCoverPath = await _normalizePlaylistCoverFile( id, diff --git a/lib/screens/settings/backup_restore_page.dart b/lib/screens/settings/backup_restore_page.dart index e9550be5..bc917e2d 100644 --- a/lib/screens/settings/backup_restore_page.dart +++ b/lib/screens/settings/backup_restore_page.dart @@ -38,24 +38,22 @@ class _BackupRestorePageState extends ConsumerState { final messenger = ScaffoldMessenger.of(context); try { final settings = ref.read(settingsProvider).toJson(); - final history = await HistoryDatabase.instance.getAll(); final collectionsNotifier = ref.read(libraryCollectionsProvider.notifier); final collections = await collectionsNotifier.exportCollections(); - final covers = await collectionsNotifier.exportPlaylistCovers(); + final covers = await collectionsNotifier.exportPlaylistCoverFiles(); final extensions = await ref .read(extensionProvider.notifier) .exportBackup(includeSecrets: _includeSecrets); - final envelope = BackupService.buildEnvelope( + final file = await BackupService.writeBackupArchive( settings: settings, - history: history, + loadHistoryPage: (limit, offset) => + HistoryDatabase.instance.getAll(limit: limit, offset: offset), collections: collections, - playlistCovers: covers, + playlistCoverFiles: covers, extensions: extensions, ); - final file = await BackupService.writeBackupFile(envelope); - messenger.showSnackBar(SnackBar(content: Text(l10n.backupCreated))); await SharePlus.instance.share( @@ -74,29 +72,37 @@ class _BackupRestorePageState extends ConsumerState { final l10n = context.l10n; final messenger = ScaffoldMessenger.of(context); - String? content; + BackupBundle? bundle; try { final picked = await FilePicker.pickFile( type: FileType.custom, allowedExtensions: ['json', BackupService.fileExtension], ); if (picked == null) return; - content = utf8.decode(await picked.readAsBytes()); + final path = picked.path; + bundle = path != null + ? await BackupService.parseFile(path) + : BackupService.parse(utf8.decode(await picked.readAsBytes())); } catch (e) { _log.e('Failed to read backup file: $e'); messenger.showSnackBar(SnackBar(content: Text(l10n.backupInvalidFile))); return; } - final bundle = BackupService.parse(content); if (bundle == null) { messenger.showSnackBar(SnackBar(content: Text(l10n.backupInvalidFile))); return; } - if (!mounted) return; + if (!mounted) { + await bundle.cleanup(); + return; + } final confirmed = await _confirmRestore(bundle); - if (confirmed != true || !mounted) return; + if (confirmed != true || !mounted) { + await bundle.cleanup(); + return; + } setState(() => _isImporting = true); try { @@ -107,7 +113,7 @@ class _BackupRestorePageState extends ConsumerState { } await ref .read(downloadHistoryProvider.notifier) - .restoreFromBackup(bundle.history); + .restoreFromBackupStream(bundle.streamHistory()); await ref .read(libraryCollectionsProvider.notifier) .restoreFromBackup( @@ -141,6 +147,7 @@ class _BackupRestorePageState extends ConsumerState { _log.e('Failed to restore backup: $e', e, stack); messenger.showSnackBar(SnackBar(content: Text(l10n.backupRestoreFailed))); } finally { + await bundle.cleanup(); if (mounted) setState(() => _isImporting = false); } } diff --git a/lib/services/backup_service.dart b/lib/services/backup_service.dart index eab8cbca..6d93df21 100644 --- a/lib/services/backup_service.dart +++ b/lib/services/backup_service.dart @@ -1,32 +1,29 @@ import 'dart:convert'; import 'dart:io'; +import 'package:archive/archive_io.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; import 'package:spotiflac_android/constants/app_info.dart'; import 'package:spotiflac_android/utils/logger.dart'; -/// Parsed contents of a backup file. +typedef BackupHistoryPageLoader = + Future>> Function(int limit, int offset); + +/// Parsed contents of a backup file. Version 2 keeps large history and cover +/// payloads on disk until restore consumes them. class BackupBundle { final int formatVersion; final String appVersion; final DateTime? createdAt; - - /// Raw `AppSettings.toJson()` map, or null when not present. final Map? settings; - - /// History items in `DownloadHistoryItem.toJson()` shape. final List> history; - - /// Collections in `LibraryCollectionsState.toJson()` shape - /// (wishlist / loved / playlists / favoriteArtists). final Map collections; - - /// Playlist cover images keyed by playlist id: `{ id: { ext, data } }`. final Map playlistCovers; - - /// Extensions section: `{ registry_url, items: [ {id, version, enabled, settings} ] }`. final Map extensions; + final String? _historyNdjsonPath; + final int? _historyCount; + final String? _temporaryDirectoryPath; const BackupBundle({ required this.formatVersion, @@ -37,11 +34,38 @@ class BackupBundle { required this.collections, required this.playlistCovers, required this.extensions, - }); + String? historyNdjsonPath, + int? historyCount, + String? temporaryDirectoryPath, + }) : _historyNdjsonPath = historyNdjsonPath, + _historyCount = historyCount, + _temporaryDirectoryPath = temporaryDirectoryPath; bool get hasSettings => settings != null && settings!.isNotEmpty; + int get historyCount => _historyCount ?? history.length; - int get historyCount => history.length; + Stream> streamHistory() async* { + if (_historyNdjsonPath == null) { + yield* Stream.fromIterable(history); + return; + } + await for (final line in File( + _historyNdjsonPath, + ).openRead().transform(utf8.decoder).transform(const LineSplitter())) { + if (line.trim().isEmpty) continue; + final decoded = jsonDecode(line); + if (decoded is Map) yield Map.from(decoded); + } + } + + Future cleanup() async { + final path = _temporaryDirectoryPath; + if (path == null || path.isEmpty) return; + try { + final directory = Directory(path); + if (await directory.exists()) await directory.delete(recursive: true); + } catch (_) {} + } int _collectionListCount(String key) { final value = collections[key]; @@ -59,7 +83,6 @@ class BackupBundle { } bool get hasExtensions => extensionCount > 0; - bool get isEmpty => !hasSettings && historyCount == 0 && @@ -70,19 +93,23 @@ class BackupBundle { extensionCount == 0; } -/// Builds and parses SpotiFLAC backup files (a single JSON document containing -/// settings, download history and the user library). class BackupService { static final _log = AppLogger('BackupService'); static const String magic = 'spotiflac-backup'; - static const int formatVersion = 1; - static const String fileExtension = 'json'; + static const int formatVersion = 2; + static const String fileExtension = 'sflbackup'; + static const int _historyPageSize = 500; + static const int _maxMetadataBytes = 8 << 20; + static const int _maxHistoryBytes = 512 << 20; + static const int _maxCoverBytes = 20 << 20; + static const int _maxAllCoversBytes = 256 << 20; static String encode(Map envelope) => const JsonEncoder.withIndent(' ').convert(envelope); - /// Builds the backup envelope written to disk. + /// Legacy JSON envelope retained so backups from earlier releases and unit + /// tests remain readable. New backups are written by [writeBackupArchive]. static Map buildEnvelope({ required Map? settings, required List> history, @@ -92,7 +119,7 @@ class BackupService { }) { return { 'magic': magic, - 'format_version': formatVersion, + 'format_version': 1, 'app': 'SpotiFLAC Mobile', 'app_version': AppInfo.displayVersion, 'created_at': DateTime.now().toIso8601String(), @@ -106,29 +133,248 @@ class BackupService { }; } - /// Writes [envelope] to a timestamped file under the app documents directory - /// and returns the created file. + static Future writeBackupArchive({ + required Map? settings, + required BackupHistoryPageLoader loadHistoryPage, + required Map collections, + required Map> playlistCoverFiles, + required Map extensions, + Directory? outputDirectory, + Directory? temporaryDirectory, + }) async { + final output = await _newBackupFile(outputDirectory); + final tempRoot = temporaryDirectory ?? await getTemporaryDirectory(); + final staging = await Directory( + p.join( + tempRoot.path, + 'spotiflac_backup_${DateTime.now().microsecondsSinceEpoch}', + ), + ).create(recursive: true); + final historyFile = File(p.join(staging.path, 'history.ndjson')); + final metadataFile = File(p.join(staging.path, 'metadata.json')); + final partFile = File('${output.path}.part'); + ZipFileEncoder? encoder; + + try { + var historyCount = 0; + var offset = 0; + final historySink = historyFile.openWrite(); + try { + while (true) { + final page = await loadHistoryPage(_historyPageSize, offset); + for (final item in page) { + historySink.writeln(jsonEncode(item)); + } + historyCount += page.length; + offset += page.length; + if (page.length < _historyPageSize) break; + } + await historySink.flush(); + } finally { + await historySink.close(); + } + + final coverManifest = >{}; + var coverIndex = 0; + for (final entry in playlistCoverFiles.entries) { + final sourcePath = entry.value['path'] ?? ''; + final source = File(sourcePath); + if (sourcePath.isEmpty || !await source.exists()) continue; + var ext = (entry.value['ext'] ?? p.extension(sourcePath)).toLowerCase(); + if (!RegExp(r'^\.[a-z0-9]{1,8}$').hasMatch(ext)) ext = '.jpg'; + final archiveName = 'covers/cover_${coverIndex++}$ext'; + coverManifest[entry.key] = {'ext': ext, 'file': archiveName}; + } + + final metadata = { + 'magic': magic, + 'format_version': formatVersion, + 'app': 'SpotiFLAC Mobile', + 'app_version': AppInfo.displayVersion, + 'created_at': DateTime.now().toIso8601String(), + 'history_count': historyCount, + 'data': { + 'settings': settings, + 'collections': collections, + 'playlist_covers': coverManifest, + 'extensions': extensions, + }, + }; + await metadataFile.writeAsString(jsonEncode(metadata), flush: true); + + if (await partFile.exists()) await partFile.delete(); + encoder = ZipFileEncoder()..create(partFile.path); + await encoder.addFile(metadataFile, 'metadata.json'); + await encoder.addFile(historyFile, 'history.ndjson'); + for (final entry in coverManifest.entries) { + final sourcePath = playlistCoverFiles[entry.key]?['path']; + if (sourcePath == null) continue; + await encoder.addFile( + File(sourcePath), + entry.value['file'], + ZipFileEncoder.store, + ); + } + await encoder.close(); + encoder = null; + if (await output.exists()) await output.delete(); + await partFile.rename(output.path); + _log.i('Streaming backup written to ${output.path}'); + return output; + } finally { + if (encoder != null) { + try { + await encoder.close(); + } catch (_) {} + } + try { + if (await staging.exists()) await staging.delete(recursive: true); + } catch (_) {} + } + } + + static Future _newBackupFile([Directory? outputDirectory]) async { + final dir = outputDirectory ?? await getApplicationDocumentsDirectory(); + final backupsDir = + outputDirectory ?? Directory(p.join(dir.path, 'backups')); + await backupsDir.create(recursive: true); + final now = DateTime.now(); + String two(int value) => value.toString().padLeft(2, '0'); + final stamp = + '${now.year}${two(now.month)}${two(now.day)}_' + '${two(now.hour)}${two(now.minute)}${two(now.second)}'; + return File( + p.join(backupsDir.path, 'spotiflac_mobile_backup_$stamp.$fileExtension'), + ); + } + + /// Legacy JSON writer retained for compatibility with callers outside the + /// settings UI. It no longer defines the default backup format. static Future writeBackupFile(Map envelope) async { - final dir = await getApplicationDocumentsDirectory(); - final backupsDir = Directory(p.join(dir.path, 'backups')); - if (!await backupsDir.exists()) { - await backupsDir.create(recursive: true); + final output = await _newBackupFile(); + await output.writeAsString(encode(envelope), flush: true); + return output; + } + + static Future parseFile( + String path, { + Directory? temporaryDirectory, + }) async { + final file = File(path); + final header = await file + .openRead(0, 4) + .fold>([], (bytes, chunk) => bytes..addAll(chunk)); + final isZip = + header.length == 4 && + header[0] == 0x50 && + header[1] == 0x4b && + header[2] == 0x03 && + header[3] == 0x04; + return isZip + ? _parseArchive(file, temporaryDirectory: temporaryDirectory) + : parse(await file.readAsString()); + } + + static Future _parseArchive( + File file, { + Directory? temporaryDirectory, + }) async { + InputFileStream? input; + Archive? archive; + Directory? extractionDir; + try { + input = InputFileStream(file.path); + archive = ZipDecoder().decodeStream(input); + final metadataEntry = archive.find('metadata.json'); + final historyEntry = archive.find('history.ndjson'); + if (metadataEntry == null || + metadataEntry.size > _maxMetadataBytes || + historyEntry == null || + historyEntry.size > _maxHistoryBytes) { + return null; + } + final rootRaw = jsonDecode(utf8.decode(metadataEntry.content)); + if (rootRaw is! Map) return null; + final root = Map.from(rootRaw); + if (root['magic'] != magic || root['format_version'] != formatVersion) { + return null; + } + final dataRaw = root['data']; + if (dataRaw is! Map) return null; + final data = Map.from(dataRaw); + + final tempRoot = temporaryDirectory ?? await getTemporaryDirectory(); + extractionDir = await Directory( + p.join( + tempRoot.path, + 'spotiflac_restore_${DateTime.now().microsecondsSinceEpoch}', + ), + ).create(recursive: true); + final historyPath = p.join(extractionDir.path, 'history.ndjson'); + final historyOutput = OutputFileStream(historyPath); + historyEntry.writeContent(historyOutput); + historyOutput.closeSync(); + + final restoredCovers = {}; + final coverManifest = data['playlist_covers']; + if (coverManifest is Map) { + var index = 0; + var extractedCoverBytes = 0; + for (final manifestEntry in coverManifest.entries) { + if (manifestEntry.value is! Map) continue; + final cover = Map.from(manifestEntry.value as Map); + final archiveName = cover['file']?.toString() ?? ''; + if (!archiveName.startsWith('covers/') || + archiveName.contains('..')) { + continue; + } + final archiveEntry = archive.find(archiveName); + if (archiveEntry == null || archiveEntry.size > _maxCoverBytes) { + continue; + } + if (extractedCoverBytes + archiveEntry.size > _maxAllCoversBytes) { + break; + } + var ext = cover['ext']?.toString() ?? '.jpg'; + if (!RegExp(r'^\.[a-z0-9]{1,8}$').hasMatch(ext)) ext = '.jpg'; + final coverPath = p.join(extractionDir.path, 'cover_${index++}$ext'); + final output = OutputFileStream(coverPath); + archiveEntry.writeContent(output); + output.closeSync(); + extractedCoverBytes += archiveEntry.size; + restoredCovers[manifestEntry.key.toString()] = { + 'ext': ext, + 'path': coverPath, + }; + } + } + + return BackupBundle( + formatVersion: formatVersion, + appVersion: root['app_version'] as String? ?? '', + createdAt: DateTime.tryParse(root['created_at'] as String? ?? ''), + settings: _mapOrNull(data['settings']), + history: const [], + historyNdjsonPath: historyPath, + historyCount: (root['history_count'] as num?)?.toInt() ?? 0, + collections: _mapOrEmpty(data['collections']), + playlistCovers: restoredCovers, + extensions: _mapOrEmpty(data['extensions']), + temporaryDirectoryPath: extractionDir.path, + ); + } catch (e) { + _log.w('Backup archive parse failed: $e'); + if (extractionDir != null) { + try { + await extractionDir.delete(recursive: true); + } catch (_) {} + } + return null; + } finally { + if (input != null) await input.close(); } - - final now = DateTime.now(); - String two(int v) => v.toString().padLeft(2, '0'); - final stamp = - '${now.year}${two(now.month)}${two(now.day)}_${two(now.hour)}${two(now.minute)}${two(now.second)}'; - final fileName = 'spotiflac_backup_$stamp.$fileExtension'; - final file = File(p.join(backupsDir.path, fileName)); - - await file.writeAsString(encode(envelope), flush: true); - _log.i('Backup written to ${file.path}'); - return file; } - /// Parses and validates a backup file's contents. Returns null when the - /// content is not a recognizable SpotiFLAC backup. static BackupBundle? parse(String content) { dynamic decoded; try { @@ -137,65 +383,31 @@ class BackupService { _log.w('Backup parse failed: not valid JSON ($e)'); return null; } - - if (decoded is! Map) { - _log.w('Backup parse failed: root is not an object'); - return null; - } - + if (decoded is! Map) return null; final root = Map.from(decoded); - if (root['magic'] != magic) { - _log.w('Backup parse failed: magic marker missing'); - return null; - } - - final dataRaw = root['data']; - if (dataRaw is! Map) { - _log.w('Backup parse failed: missing data section'); - return null; - } - final data = Map.from(dataRaw); - - Map? settings; - final settingsRaw = data['settings']; - if (settingsRaw is Map) { - settings = Map.from(settingsRaw); - } - + if (root['magic'] != magic || root['data'] is! Map) return null; + final data = Map.from(root['data'] as Map); final history = >[]; - final historyRaw = data['history']; - if (historyRaw is List) { - for (final item in historyRaw) { - if (item is Map) { - history.add(Map.from(item)); - } + if (data['history'] is List) { + for (final item in data['history'] as List) { + if (item is Map) history.add(Map.from(item)); } } - - final collectionsRaw = data['collections']; - final collections = collectionsRaw is Map - ? Map.from(collectionsRaw) - : {}; - - final coversRaw = data['playlist_covers']; - final playlistCovers = coversRaw is Map - ? Map.from(coversRaw) - : {}; - - final extensionsRaw = data['extensions']; - final extensions = extensionsRaw is Map - ? Map.from(extensionsRaw) - : {}; - return BackupBundle( formatVersion: (root['format_version'] as num?)?.toInt() ?? 1, appVersion: root['app_version'] as String? ?? '', createdAt: DateTime.tryParse(root['created_at'] as String? ?? ''), - settings: settings, + settings: _mapOrNull(data['settings']), history: history, - collections: collections, - playlistCovers: playlistCovers, - extensions: extensions, + collections: _mapOrEmpty(data['collections']), + playlistCovers: _mapOrEmpty(data['playlist_covers']), + extensions: _mapOrEmpty(data['extensions']), ); } + + static Map? _mapOrNull(dynamic value) => + value is Map ? Map.from(value) : null; + + static Map _mapOrEmpty(dynamic value) => + value is Map ? Map.from(value) : {}; } diff --git a/test/backup_service_v2_test.dart b/test/backup_service_v2_test.dart new file mode 100644 index 00000000..0738d0c5 --- /dev/null +++ b/test/backup_service_v2_test.dart @@ -0,0 +1,74 @@ +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:spotiflac_android/services/backup_service.dart'; + +void main() { + test('ZIP v2 pages history and streams cover files during restore', () async { + final root = await Directory.systemTemp.createTemp( + 'spotiflac-backup-test-', + ); + addTearDown(() async { + if (await root.exists()) await root.delete(recursive: true); + }); + final output = await Directory(p.join(root.path, 'output')).create(); + final temporary = await Directory(p.join(root.path, 'temporary')).create(); + final cover = File(p.join(root.path, 'cover.jpg')); + await cover.writeAsBytes([0xff, 0xd8, 0xff, 0xd9]); + final history = List.generate( + 1203, + (index) => {'id': 'item-$index', 'value': index}, + ); + final offsets = []; + + final file = await BackupService.writeBackupArchive( + settings: const {'theme': 'dark'}, + loadHistoryPage: (limit, offset) async { + offsets.add(offset); + if (offset >= history.length) { + return const >[]; + } + final end = (offset + limit).clamp(0, history.length); + return history.sublist(offset, end); + }, + collections: const { + 'loved': [], + 'wishlist': [], + 'playlists': [], + }, + playlistCoverFiles: { + 'playlist-1': {'ext': '.jpg', 'path': cover.path}, + }, + extensions: const {'items': []}, + outputDirectory: output, + temporaryDirectory: temporary, + ); + + expect(file.path, endsWith('.${BackupService.fileExtension}')); + expect(offsets, [0, 500, 1000]); + final bundle = await BackupService.parseFile( + file.path, + temporaryDirectory: temporary, + ); + expect(bundle, isNotNull); + expect(bundle!.formatVersion, 2); + expect(bundle.historyCount, history.length); + final restoredHistory = await bundle.streamHistory().toList(); + expect(restoredHistory, hasLength(history.length)); + expect(restoredHistory.last['id'], 'item-1202'); + final restoredCover = bundle.playlistCovers['playlist-1']; + expect(restoredCover, isA>()); + final restoredCoverPath = + (restoredCover as Map)['path'] as String; + expect(await File(restoredCoverPath).readAsBytes(), [ + 0xff, + 0xd8, + 0xff, + 0xd9, + ]); + + await bundle.cleanup(); + expect(await File(restoredCoverPath).exists(), isFalse); + }); +}