diff --git a/lib/services/cover_cache_info_repository.dart b/lib/services/cover_cache_info_repository.dart new file mode 100644 index 000000000..032d8b296 --- /dev/null +++ b/lib/services/cover_cache_info_repository.dart @@ -0,0 +1,339 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; + +/// The cover cache index in the file format of flutter_cache_manager's +/// [JsonCacheInfoRepository], so older and newer builds read each other's +/// index. +/// +/// The file is read in a worker isolate. Lookups go by key, each row keeps +/// its encoded JSON until it changes, and the file is saved [saveDelay] after +/// an insert, delete or content update, or [touchSaveDelay] after a change of +/// only the last-used time. [flush] and [close] save pending changes at once. +class CoverCacheInfoRepository extends CacheInfoRepository { + /// [encodeRow] encodes one index row; tests count calls through it. + CoverCacheInfoRepository( + this._resolveFile, { + this.saveDelay = const Duration(seconds: 1), + this.touchSaveDelay = const Duration(seconds: 30), + String Function(Map row) encodeRow = jsonEncode, + }) : _encodeRow = encodeRow; + + final Future Function() _resolveFile; + final Duration saveDelay; + final Duration touchSaveDelay; + final String Function(Map row) _encodeRow; + + _Index _index = _Index(); + File? _file; + int _connections = 0; + Completer? _opened; + bool _dirty = false; + Timer? _saveTimer; + bool _saveTimerIsShort = false; + Future _writes = Future.value(); + + @override + Future open() async { + _connections++; + final opened = _opened ??= Completer(); + if (_connections != 1) return opened.future; + await _load(); + opened.complete(true); + return true; + } + + @override + Future get(String key) async => _index.objects[key]; + + @override + Future> getAllObjects() async => + _index.objects.values.toList(); + + @override + Future insert( + CacheObject cacheObject, { + bool setTouchedToNow = true, + }) async { + if (cacheObject.id != null) { + throw ArgumentError("Inserted objects shouldn't have an existing id."); + } + return _put(cacheObject.copyWith(id: _index.lastId + 1), setTouchedToNow); + } + + @override + Future update( + CacheObject cacheObject, { + bool setTouchedToNow = true, + }) async { + if (cacheObject.id == null) { + throw ArgumentError('Updated objects should have an existing id.'); + } + _put(cacheObject, setTouchedToNow); + return 1; + } + + @override + Future updateOrInsert(CacheObject cacheObject) { + return cacheObject.id == null ? insert(cacheObject) : update(cacheObject); + } + + @override + Future> getObjectsOverCapacity(int capacity) async { + if (_index.objects.length <= capacity) return []; + final allSorted = _index.objects.values.toList() + ..sort((c1, c2) => c1.touched!.compareTo(c2.touched!)); + return allSorted.getRange(0, allSorted.length - capacity).toList(); + } + + @override + Future> getOldObjects(Duration maxAge) async { + final oldestTimestamp = DateTime.now().subtract(maxAge); + return _index.objects.values + .where((element) => element.touched!.isBefore(oldestTimestamp)) + .toList(); + } + + @override + Future delete(int id) async { + if (!_index.remove(id)) return 0; + _scheduleSave(short: true); + return 1; + } + + @override + Future deleteAll(Iterable ids) async { + var deleted = 0; + for (final id in ids) { + if (_index.remove(id)) deleted++; + } + if (deleted > 0) _scheduleSave(short: true); + return deleted; + } + + @override + Future close() async { + _connections--; + if (_connections == 0) _opened = null; + await flush(); + return _connections == 0; + } + + /// Saves pending changes now; completes once they are on disk. + Future flush() { + _saveTimer?.cancel(); + _saveTimer = null; + if (_dirty) _writes = _writes.then((_) => _write()); + return _writes; + } + + @override + Future deleteDataFile() async { + final file = await _resolveFile(); + if (await file.exists()) { + await file.delete(); + } + final tempFile = File('${file.path}.tmp'); + if (await tempFile.exists()) { + await tempFile.delete(); + } + } + + @override + Future exists() async { + final file = await _resolveFile(); + return file.exists(); + } + + Future _load() async { + await _writes; + _index = _Index(); + _file = null; + final File file; + try { + file = await _resolveFile(); + await file.parent.create(recursive: true); + } on Object catch (error) { + // Without the file, changes stay in memory: a later save would replace + // an index this session never read. + debugPrint('CoverCacheInfoRepository: Index unavailable: $error'); + return; + } + _file = file; + final path = file.path; + _IndexFile read; + try { + read = await Isolate.run(() => _readIndexFile(path)); + } on Object catch (error, stack) { + read = _IndexFile(_Index(), error, stack); + } + _index = read.index; + final error = read.error; + if (error != null) { + FlutterError.reportError( + FlutterErrorDetails( + exception: error, + stack: read.stack, + library: 'flutter cache manager', + context: ErrorDescription( + 'Thrown when reading the file containing cache info. ' + 'The cached files cannot be used by the cache manager anymore.', + ), + ), + ); + } + } + + CacheObject _put(CacheObject cacheObject, bool setTouchedToNow) { + final id = cacheObject.id!; + final row = cacheObject.toMap(setTouchedToNow: setTouchedToNow); + final stored = CacheObject.fromMap(row); + final change = _changeOf(_index.rows[id], row); + _index.set( + stored, + row, + change == _RowChange.none ? _index.encodedRows[id] : null, + ); + if (change != _RowChange.none) { + _scheduleSave(short: change == _RowChange.content); + } + return stored; + } + + static _RowChange _changeOf( + Map? previous, + Map next, + ) { + if (previous == null || previous.length != next.length) { + return _RowChange.content; + } + var touched = false; + for (final MapEntry(:key, :value) in next.entries) { + if (!previous.containsKey(key)) return _RowChange.content; + if (previous[key] == value) continue; + if (key != CacheObject.columnTouched) return _RowChange.content; + touched = true; + } + return touched ? _RowChange.touch : _RowChange.none; + } + + void _scheduleSave({required bool short}) { + _dirty = true; + if (_saveTimer != null && (_saveTimerIsShort || !short)) return; + _saveTimer?.cancel(); + _saveTimerIsShort = short; + _saveTimer = Timer(short ? saveDelay : touchSaveDelay, () { + _saveTimer = null; + unawaited(flush()); + }); + } + + Future _write() async { + final file = _file; + if (!_dirty || file == null) return; + _dirty = false; + final encodedRows = _index.encodedRows; + final content = BytesBuilder(copy: false)..addByte(0x5b); + var first = true; + for (final MapEntry(:key, :value) in _index.rows.entries) { + if (!first) content.addByte(0x2c); + first = false; + content.add(encodedRows[key] ??= utf8.encode(_encodeRow(value))); + } + content.addByte(0x5d); + try { + final tempFile = File('${file.path}.tmp'); + await tempFile.writeAsBytes(content.takeBytes(), flush: true); + await tempFile.rename(file.path); + } on Object catch (e, stacktrace) { + // Keep the changes dirty so a later change or close retries the write. + _dirty = true; + FlutterError.reportError( + FlutterErrorDetails( + exception: e, + stack: stacktrace, + library: 'flutter cache manager', + context: ErrorDescription( + 'Thrown when writing the file containing cache info. ' + 'The cache info could not be persisted and may be lost when the ' + 'app is closed.', + ), + ), + ); + } + } +} + +enum _RowChange { none, touch, content } + +/// Rows by id in file order, with their encoded JSON, and objects by key. +class _Index { + final Map> rows = {}; + final Map encodedRows = {}; + final Map objects = {}; + final Map keys = {}; + int lastId = 0; + + void set(CacheObject object, Map row, Uint8List? json) { + final id = object.id!; + rows[id] = row; + if (json == null) { + encodedRows.remove(id); + } else { + encodedRows[id] = json; + } + final previous = objects[object.key]; + if (previous != null && previous.id != id) keys.remove(previous.id); + final previousKey = keys[id]; + if (previousKey != null && previousKey != object.key) { + objects.remove(previousKey); + } + objects[object.key] = object; + keys[id] = object.key; + if (id > lastId) lastId = id; + } + + bool remove(int id) { + final key = keys.remove(id); + if (key == null) return false; + objects.remove(key); + rows.remove(id); + encodedRows.remove(id); + return true; + } +} + +class _IndexFile { + const _IndexFile(this.index, [this.error, this.stack]); + + final _Index index; + final Object? error; + final StackTrace? stack; +} + +/// Runs in a worker isolate. Keeps the rows read before a bad one, as +/// [JsonCacheInfoRepository] does. +_IndexFile _readIndexFile(String path) { + final index = _Index(); + try { + final file = File(path); + if (!file.existsSync()) return _IndexFile(index); + final json = jsonDecode(file.readAsStringSync()) as List; + for (final element in json) { + if (element is! Map) continue; + index.set( + CacheObject.fromMap(element), + element, + utf8.encode(jsonEncode(element)), + ); + } + return _IndexFile(index); + } on Object catch (error, stack) { + return _IndexFile(index, error, StackTrace.fromString(stack.toString())); + } +} diff --git a/test/cover_cache_info_repository_test.dart b/test/cover_cache_info_repository_test.dart new file mode 100644 index 000000000..2f9bbfcba --- /dev/null +++ b/test/cover_cache_info_repository_test.dart @@ -0,0 +1,318 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:mornye/services/cover_cache_info_repository.dart'; + +/// Counts index rewrites: every save writes `.tmp` and renames it. +final class _IndexWrites extends IOOverrides { + int count = 0; + + @override + File createFile(String path) { + if (path.endsWith('.json.tmp')) count++; + return super.createFile(path); + } +} + +CacheObject _cover(int i, {DateTime? touched, String? eTag}) => CacheObject( + 'https://covers.example.invalid/album-$i/cover-1400.jpg', + relativePath: 'cover-$i.jpg', + validTill: DateTime(2030).add(Duration(minutes: i)), + eTag: eTag ?? '"etag-$i"', + length: 250000 + i, + touched: touched ?? DateTime(2026, 10, 1).add(Duration(seconds: i)), +); + +/// An index file exactly as [JsonCacheInfoRepository] writes it. +Future _writeIndex(File file, int count) => file.writeAsString( + jsonEncode([ + for (var i = 1; i <= count; i++) + _cover(i).copyWith(id: i).toMap(setTouchedToNow: false), + ]), +); + +List> _rows(File file) => + (jsonDecode(file.readAsStringSync()) as List) + .cast>(); + +Map> _maps(Iterable objects) => { + for (final object in objects) + object.key: object.toMap(setTouchedToNow: false), +}; + +Future _until(bool Function() condition, String what) async { + final deadline = DateTime.now().add(const Duration(seconds: 10)); + while (!condition()) { + if (DateTime.now().isAfter(deadline)) { + throw TimeoutException('Timed out waiting for $what'); + } + await Future.delayed(const Duration(milliseconds: 5)); + } +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late Directory directory; + late File file; + + setUp(() async { + directory = await Directory.systemTemp.createTemp('cover-index-'); + file = File('${directory.path}/null.json'); + }); + + tearDown(() => directory.delete(recursive: true)); + + test('reads and writes the package JSON index both ways', () async { + final package = JsonCacheInfoRepository.withFile(file); + await package.open(); + final inserted = []; + for (var i = 1; i <= 4; i++) { + inserted.add(await package.insert(_cover(i), setTouchedToNow: false)); + } + inserted.add( + await package.insert( + CacheObject( + 'https://covers.example.invalid/redirected.jpg', + key: 'cover-key', + relativePath: 'keyed.png', + validTill: DateTime(2031), + ), + setTouchedToNow: false, + ), + ); + await package.delete(inserted[1].id!); + await package.close(); + final written = file.readAsBytesSync(); + + final writes = _IndexWrites(); + await IOOverrides.runWithIOOverrides(() async { + final repository = CoverCacheInfoRepository(() async => file); + await repository.open(); + final reread = JsonCacheInfoRepository.withFile(file); + await reread.open(); + expect( + _maps(await repository.getAllObjects()), + _maps(await reread.getAllObjects()), + ); + await reread.close(); + expect((await repository.get('cover-key'))?.relativePath, 'keyed.png'); + expect(await repository.get(inserted[1].key), isNull); + await repository.close(); + expect(writes.count, 0); + expect(file.readAsBytesSync(), written); + + await repository.open(); + final added = await repository.insert(_cover(9)); + expect(added.id, 6); + expect(added.key, _cover(9).key); + await repository.update(inserted[0]); + await repository.update( + inserted[2].copyWith(eTag: '"changed"', length: 7), + ); + expect(await repository.delete(inserted[3].id!), 1); + await repository.close(); + expect(writes.count, 1); + }, writes); + final mine = file.readAsBytesSync(); + + final package2 = JsonCacheInfoRepository.withFile(file); + await package2.open(); + final objects = { + for (final object in await package2.getAllObjects()) object.key: object, + }; + expect(objects.keys.toSet(), { + inserted[0].key, + inserted[2].key, + 'cover-key', + _cover(9).key, + }); + expect(objects[_cover(9).key]!.id, 6); + expect(objects[inserted[2].key]!.eTag, '"changed"'); + expect(objects[inserted[2].key]!.length, 7); + expect( + objects[inserted[0].key]!.touched!.isAfter(inserted[0].touched!), + isTrue, + ); + // Rewritten by the package without changes: the same bytes. + await package2.update(objects[inserted[0].key]!, setTouchedToNow: false); + await package2.close(); + expect(file.readAsBytesSync(), mine); + }); + + test('saves a burst of changes once and waits longer for touches', () async { + await _writeIndex(file, 20); + final writes = _IndexWrites(); + var encoded = 0; + await IOOverrides.runWithIOOverrides(() async { + final repository = CoverCacheInfoRepository( + () async => file, + saveDelay: const Duration(milliseconds: 100), + touchSaveDelay: const Duration(hours: 1), + encodeRow: (row) { + encoded++; + return jsonEncode(row); + }, + ); + await repository.open(); + final objects = await repository.getAllObjects(); + + for (var i = 21; i <= 40; i++) { + await repository.insert(_cover(i)); + } + expect(writes.count, 0); + await _until(() => writes.count == 1, 'the insert save'); + await Future.delayed(const Duration(milliseconds: 300)); + expect(writes.count, 1); + expect(encoded, 20); + expect(_rows(file), hasLength(40)); + + for (final object in objects.take(10)) { + await repository.update(object); + } + await repository.update(objects[15], setTouchedToNow: false); + await Future.delayed(const Duration(milliseconds: 300)); + expect(writes.count, 1); + expect(encoded, 20); + + await repository.update(objects[12].copyWith(eTag: '"new"')); + await _until(() => writes.count == 2, 'the update save'); + expect(encoded, 31); + final touchedRows = { + for (final row in _rows(file)) row['_id'] as int: row['touched'], + }; + for (final object in objects.take(10)) { + expect( + touchedRows[object.id], + greaterThan(object.touched!.millisecondsSinceEpoch), + ); + } + + await repository.delete(objects[19].id!); + await repository.close(); + expect(writes.count, 3); + expect(encoded, 31); + }, writes); + expect(_rows(file), hasLength(39)); + }); + + test('a scroll over an old index writes once, at flush', () async { + const size = 600; + await _writeIndex(file, size); + final before = _rows(file); + final covers = Directory('${directory.path}/cover_cache'); + await covers.create(); + for (var i = 1; i <= size; i++) { + await File('${covers.path}/cover-$i.jpg').writeAsBytes([i % 256]); + } + const channel = MethodChannel('plugins.flutter.io/path_provider'); + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + messenger.setMockMethodCallHandler(channel, (_) async => directory.path); + addTearDown(() => messenger.setMockMethodCallHandler(channel, null)); + + final writes = _IndexWrites(); + var encoded = 0; + await IOOverrides.runWithIOOverrides(() async { + final repository = CoverCacheInfoRepository( + () async => file, + encodeRow: (row) { + encoded++; + return jsonEncode(row); + }, + ); + final manager = CacheManager( + Config( + 'coverScrollTest', + repo: repository, + fileSystem: IOFileSystem(covers.path), + maxNrOfCacheObjects: 1000, + stalePeriod: const Duration(days: 365), + ), + ); + for (var i = 1; i <= 200; i++) { + final cached = await manager.getFileFromCache(_cover(i).key); + expect(cached, isNotNull); + } + await Future.delayed(const Duration(milliseconds: 1500)); + expect(writes.count, 0); + expect(encoded, 0); + + await repository.flush(); + expect(writes.count, 1); + expect(encoded, 200); + await manager.dispose(); + expect(writes.count, 1); + }, writes); + + final after = _rows(file); + expect(after, hasLength(size)); + for (var i = 0; i < size; i++) { + final changed = Map.of(after[i])..remove('touched'); + final original = Map.of(before[i])..remove('touched'); + expect(changed, original); + if (i < 200) { + expect(after[i]['touched'], greaterThan(before[i]['touched'] as int)); + } else { + expect(after[i]['touched'], before[i]['touched']); + } + } + }); + + test('get does not slow down as the index grows', () async { + Future microsecondsPerThousandGets(int size) async { + await _writeIndex(file, size); + final repository = CoverCacheInfoRepository(() async => file); + await repository.open(); + final key = _cover(size).key; + for (var i = 0; i < 2000; i++) { + await repository.get(key); + } + final watch = Stopwatch()..start(); + for (var i = 0; i < 20000; i++) { + await repository.get(key); + } + final elapsed = watch.elapsedMicroseconds ~/ 20; + await repository.close(); + return elapsed; + } + + final small = await microsecondsPerThousandGets(20); + final large = await microsecondsPerThousandGets(20000); + // A scan of 20,000 entries per lookup is about 100 times slower. + expect(large, lessThan(small * 10 + 1000)); + }); + + test('an index it cannot locate stays in memory', () async { + await _writeIndex(file, 3); + final written = file.readAsBytesSync(); + final repository = CoverCacheInfoRepository( + () async => throw const FileSystemException('no support directory'), + ); + expect(await repository.open(), isTrue); + final added = await repository.insert(_cover(1)); + expect(await repository.get(added.key), isNotNull); + await repository.close(); + expect(file.readAsBytesSync(), written); + }); + + test('a corrupt index is reported and replaced', () async { + await file.writeAsString('[{"_id": 1, "url": '); + final errors = []; + final previous = FlutterError.onError; + FlutterError.onError = errors.add; + addTearDown(() => FlutterError.onError = previous); + final repository = CoverCacheInfoRepository(() async => file); + await repository.open(); + expect(errors, hasLength(1)); + expect(errors.single.exception, isA()); + expect(await repository.getAllObjects(), isEmpty); + await repository.insert(_cover(1)); + await repository.close(); + expect(_rows(file), hasLength(1)); + }); +}