CalvinBackup/deflock-app
1
0
Fork 0
mirror of https://github.com/FoggedLens/deflock-app.git synced 2026-03-21 02:13:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add centralized retry/fallback policy with hard-coded endpoints

Browse Source 
Extract duplicated retry logic from OverpassService and RoutingService
into a shared resilience framework in service_policy.dart:

- ResiliencePolicy: configurable retries, backoff, and HTTP timeout
- executeWithFallback: retry loop with primary→fallback endpoint chain
- ErrorDisposition enum: abort / fallback / retry classification
- ServicePolicy + ServicePolicyResolver: per-service compliance rules
  (rate limits, caching, concurrency) for OSMF and third-party services
- ServiceRateLimiter: async semaphore-based concurrency and rate control

OverpassService now hits overpass.deflock.org first, falls back to
overpass-api.de. RoutingService hits api.dontgetflocked.com first,
falls back to alprwatch.org. Both use per-service error classifiers
to determine retry vs fallback vs abort behavior.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Doug Borg
2026-03-11 23:13:52 -06:00
parent 4d1032e56d
commit 2833906c68
8 changed files with 968 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/dev_config.dart
View File

@@ -64,9 +64,6 @@ const Duration kChangesetCloseMaxRetryDelay = Duration(minutes: 5); // Cap at 5
const Duration kChangesetAutoCloseTimeout = Duration(minutes: 59); // Give up and trust OSM auto-close
const double kChangesetCloseBackoffMultiplier = 2.0;
// Navigation routing configuration
const Duration kNavigationRoutingTimeout = Duration(seconds: 90); // HTTP timeout for routing requests
// Overpass API configuration
const Duration kOverpassQueryTimeout = Duration(seconds: 45); // Timeout for Overpass API queries (was 25s hardcoded)

183
lib/services/overpass_service.dart
View File

@@ -8,97 +8,106 @@ import '../models/node_profile.dart';
import '../models/osm_node.dart';
import '../dev_config.dart';
import 'http_client.dart';
import 'service_policy.dart';
/// Simple Overpass API client with proper HTTP retry logic.
/// Simple Overpass API client with retry and fallback logic.
/// Single responsibility: Make requests, handle network errors, return data.
class OverpassService {
static const String _endpoint = 'https://overpass-api.de/api/interpreter';
static const String defaultEndpoint = 'https://overpass.deflock.org/api/interpreter';
static const String fallbackEndpoint = 'https://overpass-api.de/api/interpreter';
static const _policy = ResiliencePolicy(
maxRetries: 3,
httpTimeout: Duration(seconds: 45),
);
final http.Client _client;
/// Optional override endpoint. When null, uses [defaultEndpoint].
final String? _endpointOverride;
OverpassService({http.Client? client}) : _client = client ?? UserAgentClient();
OverpassService({http.Client? client, String? endpoint})
: _client = client ?? UserAgentClient(),
_endpointOverride = endpoint;
/// Resolve the primary endpoint: constructor override or default.
String get _primaryEndpoint => _endpointOverride ?? defaultEndpoint;
/// Fetch surveillance nodes from Overpass API with proper retry logic.
/// Fetch surveillance nodes from Overpass API with retry and fallback.
/// Throws NetworkError for retryable failures, NodeLimitError for area splitting.
Future<List<OsmNode>> fetchNodes({
required LatLngBounds bounds,
required List<NodeProfile> profiles,
int maxRetries = 3,
ResiliencePolicy? policy,
}) async {
if (profiles.isEmpty) return [];
final query = _buildQuery(bounds, profiles);
for (int attempt = 0; attempt <= maxRetries; attempt++) {
try {
debugPrint('[OverpassService] Attempt ${attempt + 1}/${maxRetries + 1} for ${profiles.length} profiles');
final response = await _client.post(
Uri.parse(_endpoint),
body: {'data': query},
).timeout(kOverpassQueryTimeout);
if (response.statusCode == 200) {
return _parseResponse(response.body);
}
// Check for specific error types
final errorBody = response.body;
// Node limit error - caller should split area
if (response.statusCode == 400 &&
(errorBody.contains('too many nodes') && errorBody.contains('50000'))) {
debugPrint('[OverpassService] Node limit exceeded, area should be split');
throw NodeLimitError('Query exceeded 50k node limit');
}
// Timeout error - also try splitting (complex query)
if (errorBody.contains('timeout') ||
errorBody.contains('runtime limit exceeded') ||
errorBody.contains('Query timed out')) {
debugPrint('[OverpassService] Query timeout, area should be split');
throw NodeLimitError('Query timed out - area too complex');
}
// Rate limit - throw immediately, don't retry
if (response.statusCode == 429 ||
errorBody.contains('rate limited') ||
errorBody.contains('too many requests')) {
debugPrint('[OverpassService] Rate limited by Overpass');
throw RateLimitError('Rate limited by Overpass API');
}
// Other HTTP errors - retry with backoff
if (attempt < maxRetries) {
final delay = Duration(milliseconds: (200 * (1 << attempt)).clamp(200, 5000));
debugPrint('[OverpassService] HTTP ${response.statusCode} error, retrying in ${delay.inMilliseconds}ms');
await Future.delayed(delay);
continue;
}
throw NetworkError('HTTP ${response.statusCode}: $errorBody');
} catch (e) {
// Handle specific error types without retry
if (e is NodeLimitError || e is RateLimitError) {
rethrow;
}
// Network/timeout errors - retry with backoff
if (attempt < maxRetries) {
final delay = Duration(milliseconds: (200 * (1 << attempt)).clamp(200, 5000));
debugPrint('[OverpassService] Network error ($e), retrying in ${delay.inMilliseconds}ms');
await Future.delayed(delay);
continue;
}
throw NetworkError('Network error after $maxRetries retries: $e');
}
}
throw NetworkError('Max retries exceeded');
final endpoint = _primaryEndpoint;
final canFallback = _endpointOverride == null;
final effectivePolicy = policy ?? _policy;
return executeWithFallback<List<OsmNode>>(
primaryUrl: endpoint,
fallbackUrl: canFallback ? fallbackEndpoint : null,
execute: (url) => _attemptFetch(url, query, effectivePolicy),
classifyError: _classifyError,
policy: effectivePolicy,
);
}
/// Single POST + parse attempt (no retry logic — handled by executeWithFallback).
Future<List<OsmNode>> _attemptFetch(String endpoint, String query, ResiliencePolicy policy) async {
debugPrint('[OverpassService] POST $endpoint');
try {
final response = await _client.post(
Uri.parse(endpoint),
body: {'data': query},
).timeout(policy.httpTimeout);
if (response.statusCode == 200) {
return _parseResponse(response.body);
}
final errorBody = response.body;
// Node limit error - caller should split area
if (response.statusCode == 400 &&
(errorBody.contains('too many nodes') && errorBody.contains('50000'))) {
debugPrint('[OverpassService] Node limit exceeded, area should be split');
throw NodeLimitError('Query exceeded 50k node limit');
}
// Timeout error - also try splitting (complex query)
if (errorBody.contains('timeout') ||
errorBody.contains('runtime limit exceeded') ||
errorBody.contains('Query timed out')) {
debugPrint('[OverpassService] Query timeout, area should be split');
throw NodeLimitError('Query timed out - area too complex');
}
// Rate limit
if (response.statusCode == 429 ||
errorBody.contains('rate limited') ||
errorBody.contains('too many requests')) {
debugPrint('[OverpassService] Rate limited by Overpass');
throw RateLimitError('Rate limited by Overpass API');
}
throw NetworkError('HTTP ${response.statusCode}: $errorBody');
} catch (e) {
if (e is NodeLimitError || e is RateLimitError || e is NetworkError) {
rethrow;
}
throw NetworkError('Network error: $e');
}
}
static ErrorDisposition _classifyError(Object error) {
if (error is NodeLimitError) return ErrorDisposition.abort;
if (error is RateLimitError) return ErrorDisposition.fallback;
return ErrorDisposition.retry;
}
/// Build Overpass QL query for given bounds and profiles
String _buildQuery(LatLngBounds bounds, List<NodeProfile> profiles) {
final nodeClauses = profiles.map((profile) {
@@ -107,7 +116,7 @@ class OverpassService {
.where((entry) => entry.value.trim().isNotEmpty)
.map((entry) => '["${entry.key}"="${entry.value}"]')
.join();
return 'node$tagFilters(${bounds.southWest.latitude},${bounds.southWest.longitude},${bounds.northEast.latitude},${bounds.northEast.longitude});';
}).join('\n ');
@@ -119,38 +128,38 @@ class OverpassService {
out body;
(
way(bn);
rel(bn);
rel(bn);
);
out skel;
''';
}
/// Parse Overpass JSON response into OsmNode objects
List<OsmNode> _parseResponse(String responseBody) {
final data = jsonDecode(responseBody) as Map<String, dynamic>;
final elements = data['elements'] as List<dynamic>;
final nodeElements = <Map<String, dynamic>>[];
final constrainedNodeIds = <int>{};
// First pass: collect surveillance nodes and identify constrained nodes
for (final element in elements.whereType<Map<String, dynamic>>()) {
final type = element['type'] as String?;
if (type == 'node') {
nodeElements.add(element);
} else if (type == 'way' || type == 'relation') {
// Mark referenced nodes as constrained
final refs = element['nodes'] as List<dynamic>? ??
final refs = element['nodes'] as List<dynamic>? ??
element['members']?.where((m) => m['type'] == 'node').map((m) => m['ref']) ?? [];
for (final ref in refs) {
final nodeId = ref is int ? ref : int.tryParse(ref.toString());
if (nodeId != null) constrainedNodeIds.add(nodeId);
}
}
}
// Second pass: create OsmNode objects
final nodes = nodeElements.map((element) {
final nodeId = element['id'] as int;
@@ -161,7 +170,7 @@ out skel;
isConstrained: constrainedNodeIds.contains(nodeId),
);
}).toList();
debugPrint('[OverpassService] Parsed ${nodes.length} nodes, ${constrainedNodeIds.length} constrained');
return nodes;
}
@@ -189,4 +198,4 @@ class NetworkError extends Error {
NetworkError(this.message);
@override
String toString() => 'NetworkError: $message';
}
}

100
lib/services/routing_service.dart
View File

@@ -5,20 +5,20 @@ import 'package:latlong2/latlong.dart';
import 'package:shared_preferences/shared_preferences.dart';
import '../app_state.dart';
import '../dev_config.dart';
import 'http_client.dart';
import 'service_policy.dart';
class RouteResult {
final List<LatLng> waypoints;
final double distanceMeters;
final double durationSeconds;
const RouteResult({
required this.waypoints,
required this.distanceMeters,
required this.durationSeconds,
});
@override
String toString() {
return 'RouteResult(waypoints: ${waypoints.length}, distance: ${(distanceMeters/1000).toStringAsFixed(1)}km, duration: ${(durationSeconds/60).toStringAsFixed(1)}min)';
@@ -26,14 +26,27 @@ class RouteResult {
}
class RoutingService {
static const String _baseUrl = 'https://alprwatch.org/api/v1/deflock/directions';
final http.Client _client;
static const String defaultUrl = 'https://api.dontgetflocked.com/api/v1/deflock/directions';
static const String fallbackUrl = 'https://alprwatch.org/api/v1/deflock/directions';
static const _policy = ResiliencePolicy(
maxRetries: 1,
httpTimeout: Duration(seconds: 30),
);
RoutingService({http.Client? client}) : _client = client ?? UserAgentClient();
final http.Client _client;
/// Optional override URL. When null, uses [defaultUrl].
final String? _baseUrlOverride;
RoutingService({http.Client? client, String? baseUrl})
: _client = client ?? UserAgentClient(),
_baseUrlOverride = baseUrl;
void close() => _client.close();
// Calculate route between two points using alprwatch
/// Resolve the primary URL to use: constructor override or default.
String get _primaryUrl => _baseUrlOverride ?? defaultUrl;
// Calculate route between two points
Future<RouteResult> calculateRoute({
required LatLng start,
required LatLng end,
@@ -53,8 +66,7 @@ class RoutingService {
'tags': tags,
};
}).toList();
final uri = Uri.parse(_baseUrl);
final params = {
'start': {
'longitude': start.longitude,
@@ -66,11 +78,25 @@ class RoutingService {
},
'avoidance_distance': avoidanceDistance,
'enabled_profiles': enabledProfiles,
'show_exclusion_zone': false, // for debugging: if true, returns a GeoJSON Feature MultiPolygon showing what areas are avoided in calculating the route
'show_exclusion_zone': false,
};
debugPrint('[RoutingService] alprwatch request: $uri $params');
final primaryUrl = _primaryUrl;
final canFallback = _baseUrlOverride == null;
return executeWithFallback<RouteResult>(
primaryUrl: primaryUrl,
fallbackUrl: canFallback ? fallbackUrl : null,
execute: (url) => _postRoute(url, params),
classifyError: _classifyError,
policy: _policy,
);
}
Future<RouteResult> _postRoute(String url, Map<String, dynamic> params) async {
final uri = Uri.parse(url);
debugPrint('[RoutingService] POST $uri');
try {
final response = await _client.post(
uri,
@@ -78,7 +104,7 @@ class RoutingService {
'Content-Type': 'application/json'
},
body: json.encode(params)
).timeout(kNavigationRoutingTimeout);
).timeout(_policy.httpTimeout);
if (response.statusCode != 200) {
if (kDebugMode) {
@@ -91,24 +117,25 @@ class RoutingService {
: body;
debugPrint('[RoutingService] Error response body ($maxLen char max): $truncated');
}
throw RoutingException('HTTP ${response.statusCode}: ${response.reasonPhrase}');
throw RoutingException('HTTP ${response.statusCode}: ${response.reasonPhrase}',
statusCode: response.statusCode);
}
final data = json.decode(response.body) as Map<String, dynamic>;
debugPrint('[RoutingService] alprwatch response data: $data');
// Check alprwatch response status
debugPrint('[RoutingService] response data: $data');
// Check response status
final ok = data['ok'] as bool? ?? false;
if ( ! ok ) {
final message = data['error'] as String? ?? 'Unknown routing error';
throw RoutingException('alprwatch error: $message');
throw RoutingException('API error: $message', isApiError: true);
}
final route = data['result']['route'] as Map<String, dynamic>?;
if (route == null) {
throw RoutingException('No route found between these points');
throw RoutingException('No route found between these points', isApiError: true);
}
final waypoints = (route['coordinates'] as List<dynamic>?)
?.map((inner) {
final pair = inner as List<dynamic>;
@@ -116,19 +143,19 @@ class RoutingService {
final lng = (pair[0] as num).toDouble();
final lat = (pair[1] as num).toDouble();
return LatLng(lat, lng);
}).whereType<LatLng>().toList() ?? [];
}).whereType<LatLng>().toList() ?? [];
final distance = (route['distance'] as num?)?.toDouble() ?? 0.0;
final duration = (route['duration'] as num?)?.toDouble() ?? 0.0;
final result = RouteResult(
waypoints: waypoints,
distanceMeters: distance,
durationSeconds: duration,
);
debugPrint('[RoutingService] Route calculated: $result');
return result;
} catch (e) {
debugPrint('[RoutingService] Route calculation failed: $e');
if (e is RoutingException) {
@@ -138,13 +165,26 @@ class RoutingService {
}
}
}
static ErrorDisposition _classifyError(Object error) {
if (error is! RoutingException) return ErrorDisposition.retry;
if (error.isApiError) return ErrorDisposition.abort;
final status = error.statusCode;
if (status != null && status >= 400 && status < 500) {
if (status == 429) return ErrorDisposition.fallback;
return ErrorDisposition.abort;
}
return ErrorDisposition.retry;
}
}
class RoutingException implements Exception {
final String message;
const RoutingException(this.message);
final int? statusCode;
final bool isApiError;
const RoutingException(this.message, {this.statusCode, this.isApiError = false});
@override
String toString() => 'RoutingException: $message';
}

141
lib/services/service_policy.dart
View File

@@ -152,11 +152,10 @@ class ServicePolicy {
'attributionUrl: $attributionUrl)';
}
/// Resolves URLs and tile providers to their applicable [ServicePolicy].
/// Resolves service URLs to their applicable [ServicePolicy].
///
/// Built-in patterns cover all OSMF official services and common third-party
/// tile providers. Custom overrides can be registered for self-hosted endpoints
/// via [registerCustomPolicy].
/// tile providers. Falls back to permissive defaults for unrecognized hosts.
class ServicePolicyResolver {
/// Host → ServiceType mapping for known services.
static final Map<String, ServiceType> _hostPatterns = {
@@ -166,6 +165,7 @@ class ServicePolicyResolver {
'tile.openstreetmap.org': ServiceType.osmTileServer,
'nominatim.openstreetmap.org': ServiceType.nominatim,
'overpass-api.de': ServiceType.overpass,
'overpass.deflock.org': ServiceType.overpass,
'taginfo.openstreetmap.org': ServiceType.tagInfo,
'tiles.virtualearth.net': ServiceType.bingTiles,
'api.mapbox.com': ServiceType.mapboxTiles,
@@ -183,25 +183,14 @@ class ServicePolicyResolver {
ServiceType.custom: const ServicePolicy(),
};
/// Custom host overrides registered at runtime (for self-hosted services).
static final Map<String, ServicePolicy> _customOverrides = {};
/// Resolve a URL to its applicable [ServicePolicy].
///
/// Checks custom overrides first, then built-in host patterns. Falls back
/// to [ServicePolicy.custom] for unrecognized hosts.
/// Checks built-in host patterns. Falls back to [ServicePolicy.custom]
/// for unrecognized hosts.
static ServicePolicy resolve(String url) {
final host = _extractHost(url);
if (host == null) return const ServicePolicy();
// Check custom overrides first (exact or subdomain matching)
for (final entry in _customOverrides.entries) {
if (host == entry.key || host.endsWith('.${entry.key}')) {
return entry.value;
}
}
// Check built-in patterns (support subdomain matching)
for (final entry in _hostPatterns.entries) {
if (host == entry.key || host.endsWith('.${entry.key}')) {
return _policies[entry.value] ?? const ServicePolicy();
@@ -218,14 +207,6 @@ class ServicePolicyResolver {
final host = _extractHost(url);
if (host == null) return ServiceType.custom;
// Check custom overrides first — a registered custom policy means
// the host is treated as ServiceType.custom with custom rules.
for (final entry in _customOverrides.entries) {
if (host == entry.key || host.endsWith('.${entry.key}')) {
return ServiceType.custom;
}
}
for (final entry in _hostPatterns.entries) {
if (host == entry.key || host.endsWith('.${entry.key}')) {
return entry.value;
@@ -239,29 +220,6 @@ class ServicePolicyResolver {
static ServicePolicy resolveByType(ServiceType type) =>
_policies[type] ?? const ServicePolicy();
/// Register a custom policy override for a host pattern.
///
/// Use this to configure self-hosted services:
/// ```dart
/// ServicePolicyResolver.registerCustomPolicy(
/// 'tiles.myserver.com',
/// ServicePolicy.custom(allowsOffline: true, maxConcurrent: 20),
/// );
/// ```
static void registerCustomPolicy(String hostPattern, ServicePolicy policy) {
_customOverrides[hostPattern] = policy;
}
/// Remove a custom policy override.
static void removeCustomPolicy(String hostPattern) {
_customOverrides.remove(hostPattern);
}
/// Clear all custom policy overrides (useful for testing).
static void clearCustomPolicies() {
_customOverrides.clear();
}
/// Extract the host from a URL or URL template.
static String? _extractHost(String url) {
// Handle URL templates like 'https://tile.openstreetmap.org/{z}/{x}/{y}.png'
@@ -283,6 +241,95 @@ class ServicePolicyResolver {
}
}
/// How the retry/fallback engine should handle an error.
enum ErrorDisposition {
/// Stop immediately. Don't retry, don't try fallback. (400, business logic)
abort,
/// Don't retry same server, but DO try fallback endpoint. (429 rate limit)
fallback,
/// Retry with backoff against same server, then fallback if exhausted. (5xx, network)
retry,
}
/// Retry and fallback configuration for resilient HTTP services.
class ResiliencePolicy {
final int maxRetries;
final Duration httpTimeout;
final Duration _retryBackoffBase;
final int _retryBackoffMaxMs;
const ResiliencePolicy({
this.maxRetries = 1,
this.httpTimeout = const Duration(seconds: 30),
Duration retryBackoffBase = const Duration(milliseconds: 200),
int retryBackoffMaxMs = 5000,
}) : _retryBackoffBase = retryBackoffBase,
_retryBackoffMaxMs = retryBackoffMaxMs;
Duration retryDelay(int attempt) {
final ms = (_retryBackoffBase.inMilliseconds * (1 << attempt))
.clamp(0, _retryBackoffMaxMs);
return Duration(milliseconds: ms);
}
}
/// Execute a request with retry and fallback logic.
///
/// 1. Tries [execute] against [primaryUrl] up to `policy.maxRetries + 1` times.
/// 2. On each failure, calls [classifyError] to determine disposition:
/// - [ErrorDisposition.abort]: rethrows immediately
/// - [ErrorDisposition.fallback]: skips retries, tries fallback (if available)
/// - [ErrorDisposition.retry]: retries with backoff, then fallback if exhausted
/// 3. If [fallbackUrl] is non-null and primary failed with a non-abort error,
/// repeats the retry loop against the fallback.
Future<T> executeWithFallback<T>({
required String primaryUrl,
required String? fallbackUrl,
required Future<T> Function(String url) execute,
required ErrorDisposition Function(Object error) classifyError,
ResiliencePolicy policy = const ResiliencePolicy(),
}) async {
try {
return await _executeWithRetries(primaryUrl, execute, classifyError, policy);
} catch (e) {
// _executeWithRetries rethrows abort/fallback/exhausted-retry errors.
// Re-classify only to distinguish abort (which must not fall back) from
// fallback/retry-exhausted (which should). This is the one intentional
// re-classification — _executeWithRetries cannot short-circuit past the
// outer try/catch.
if (classifyError(e) == ErrorDisposition.abort) rethrow;
if (fallbackUrl == null) rethrow;
debugPrint('[Resilience] Primary failed ($e), trying fallback');
return _executeWithRetries(fallbackUrl, execute, classifyError, policy);
}
}
Future<T> _executeWithRetries<T>(
String url,
Future<T> Function(String url) execute,
ErrorDisposition Function(Object error) classifyError,
ResiliencePolicy policy,
) async {
for (int attempt = 0; attempt <= policy.maxRetries; attempt++) {
try {
return await execute(url);
} catch (e) {
final disposition = classifyError(e);
if (disposition == ErrorDisposition.abort) rethrow;
if (disposition == ErrorDisposition.fallback) rethrow; // caller handles fallback
// disposition == retry
if (attempt < policy.maxRetries) {
final delay = policy.retryDelay(attempt);
debugPrint('[Resilience] Attempt ${attempt + 1} failed, retrying in ${delay.inMilliseconds}ms');
await Future.delayed(delay);
continue;
}
rethrow; // retries exhausted, let caller try fallback
}
}
throw StateError('Unreachable'); // loop always returns or throws
}
/// Reusable per-service rate limiter and concurrency controller.
///
/// Enforces the rate limits and concurrency constraints defined in each

6
lib/state/navigation_state.dart
View File

@@ -250,11 +250,11 @@ class NavigationState extends ChangeNotifier {
_calculateRoute();
}
/// Calculate route using alprwatch
/// Calculate route via RoutingService (primary + fallback endpoints).
void _calculateRoute() {
if (_routeStart == null || _routeEnd == null) return;
debugPrint('[NavigationState] Calculating route with alprwatch...');
debugPrint('[NavigationState] Calculating route...');
_isCalculating = true;
_routingError = null;
notifyListeners();
@@ -271,7 +271,7 @@ class NavigationState extends ChangeNotifier {
_showingOverview = true;
_provisionalPinLocation = null; // Hide provisional pin
debugPrint('[NavigationState] alprwatch route calculated: ${routeResult.toString()}');
debugPrint('[NavigationState] Route calculated: ${routeResult.toString()}');
notifyListeners();
}).catchError((error) {

200
test/services/overpass_service_test.dart
View File

@@ -7,6 +7,7 @@ import 'package:mocktail/mocktail.dart';
import 'package:deflockapp/models/node_profile.dart';
import 'package:deflockapp/services/overpass_service.dart';
import 'package:deflockapp/services/service_policy.dart';
class MockHttpClient extends Mock implements http.Client {}
@@ -36,6 +37,7 @@ void main() {
setUp(() {
mockClient = MockHttpClient();
// Initialize OverpassService with a mock HTTP client for testing
service = OverpassService(client: mockClient);
});
@@ -246,9 +248,9 @@ void main() {
stubErrorResponse(
400, 'Error: too many nodes (limit is 50000) in query');
expect(
await expectLater(
() => service.fetchNodes(
bounds: bounds, profiles: profiles, maxRetries: 0),
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0)),
throwsA(isA<NodeLimitError>()),
);
});
@@ -256,9 +258,9 @@ void main() {
test('response with "timeout" throws NodeLimitError', () async {
stubErrorResponse(400, 'runtime error: timeout in query execution');
expect(
await expectLater(
() => service.fetchNodes(
bounds: bounds, profiles: profiles, maxRetries: 0),
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0)),
throwsA(isA<NodeLimitError>()),
);
});
@@ -267,9 +269,9 @@ void main() {
() async {
stubErrorResponse(400, 'runtime limit exceeded');
expect(
await expectLater(
() => service.fetchNodes(
bounds: bounds, profiles: profiles, maxRetries: 0),
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0)),
throwsA(isA<NodeLimitError>()),
);
});
@@ -277,9 +279,9 @@ void main() {
test('HTTP 429 throws RateLimitError', () async {
stubErrorResponse(429, 'Too Many Requests');
expect(
await expectLater(
() => service.fetchNodes(
bounds: bounds, profiles: profiles, maxRetries: 0),
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0)),
throwsA(isA<RateLimitError>()),
);
});
@@ -287,9 +289,9 @@ void main() {
test('response with "rate limited" throws RateLimitError', () async {
stubErrorResponse(503, 'You are rate limited');
expect(
await expectLater(
() => service.fetchNodes(
bounds: bounds, profiles: profiles, maxRetries: 0),
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0)),
throwsA(isA<RateLimitError>()),
);
});
@@ -298,9 +300,9 @@ void main() {
() async {
stubErrorResponse(500, 'Internal Server Error');
expect(
await expectLater(
() => service.fetchNodes(
bounds: bounds, profiles: profiles, maxRetries: 0),
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0)),
throwsA(isA<NetworkError>()),
);
});
@@ -313,4 +315,178 @@ void main() {
verifyNever(() => mockClient.post(any(), body: any(named: 'body')));
});
});
group('fallback behavior', () {
test('falls back to overpass-api.de on NetworkError after retries', () async {
int callCount = 0;
when(() => mockClient.post(any(), body: any(named: 'body')))
.thenAnswer((invocation) async {
callCount++;
final uri = invocation.positionalArguments[0] as Uri;
if (uri.host == 'overpass.deflock.org') {
return http.Response('Internal Server Error', 500);
}
// Fallback succeeds
return http.Response(
jsonEncode({
'elements': [
{
'type': 'node',
'id': 1,
'lat': 38.9,
'lon': -77.0,
'tags': {'man_made': 'surveillance'},
},
]
}),
200,
);
});
final nodes = await service.fetchNodes(
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0));
expect(nodes, hasLength(1));
// primary (1 attempt, 0 retries) + fallback (1 attempt) = 2
expect(callCount, equals(2));
});
test('does NOT fallback on NodeLimitError', () async {
when(() => mockClient.post(any(), body: any(named: 'body')))
.thenAnswer((_) async => http.Response(
'Error: too many nodes (limit is 50000) in query',
400,
));
await expectLater(
() => service.fetchNodes(
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0)),
throwsA(isA<NodeLimitError>()),
);
// Only one call — no fallback (abort disposition)
verify(() => mockClient.post(any(), body: any(named: 'body')))
.called(1);
});
test('RateLimitError triggers fallback without retrying primary', () async {
int callCount = 0;
when(() => mockClient.post(any(), body: any(named: 'body')))
.thenAnswer((invocation) async {
callCount++;
final uri = invocation.positionalArguments[0] as Uri;
if (uri.host == 'overpass.deflock.org') {
return http.Response('Too Many Requests', 429);
}
// Fallback succeeds
return http.Response(
jsonEncode({
'elements': [
{
'type': 'node',
'id': 1,
'lat': 38.9,
'lon': -77.0,
'tags': {'man_made': 'surveillance'},
},
]
}),
200,
);
});
final nodes = await service.fetchNodes(
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 2));
expect(nodes, hasLength(1));
// 1 primary (no retry on fallback disposition) + 1 fallback = 2
expect(callCount, equals(2));
});
test('primary fails then fallback also fails -> error propagated', () async {
when(() => mockClient.post(any(), body: any(named: 'body')))
.thenAnswer((_) async =>
http.Response('Internal Server Error', 500));
await expectLater(
() => service.fetchNodes(
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0)),
throwsA(isA<NetworkError>()),
);
// primary + fallback
verify(() => mockClient.post(any(), body: any(named: 'body')))
.called(2);
});
test('does NOT fallback when using custom endpoint', () async {
final customService = OverpassService(
client: mockClient,
endpoint: 'https://custom.example.com/api/interpreter',
);
when(() => mockClient.post(any(), body: any(named: 'body')))
.thenAnswer((_) async =>
http.Response('Internal Server Error', 500));
await expectLater(
() => customService.fetchNodes(
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 0)),
throwsA(isA<NetworkError>()),
);
// Only one call - no fallback with custom endpoint
verify(() => mockClient.post(any(), body: any(named: 'body')))
.called(1);
});
test('retries exhaust before fallback kicks in', () async {
int callCount = 0;
when(() => mockClient.post(any(), body: any(named: 'body')))
.thenAnswer((invocation) async {
callCount++;
final uri = invocation.positionalArguments[0] as Uri;
if (uri.host == 'overpass.deflock.org') {
return http.Response('Server Error', 500);
}
// Fallback succeeds
return http.Response(
jsonEncode({
'elements': [
{
'type': 'node',
'id': 1,
'lat': 38.9,
'lon': -77.0,
'tags': {'man_made': 'surveillance'},
},
]
}),
200,
);
});
final nodes = await service.fetchNodes(
bounds: bounds, profiles: profiles, policy: const ResiliencePolicy(maxRetries: 2));
expect(nodes, hasLength(1));
// 3 primary attempts (1 + 2 retries) + 1 fallback = 4
expect(callCount, equals(4));
});
});
group('default endpoints', () {
test('default endpoint is overpass.deflock.org', () {
expect(OverpassService.defaultEndpoint,
equals('https://overpass.deflock.org/api/interpreter'));
});
test('fallback endpoint is overpass-api.de', () {
expect(OverpassService.fallbackEndpoint,
equals('https://overpass-api.de/api/interpreter'));
});
});
}

351
test/services/routing_service_test.dart
View File

@@ -41,6 +41,30 @@ void main() {
AppState.instance = MockAppState();
});
/// Helper: stub a successful routing response
void stubSuccessResponse() {
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((_) async => http.Response(
json.encode({
'ok': true,
'result': {
'route': {
'coordinates': [
[-77.0, 38.9],
[-77.1, 39.0],
],
'distance': 1000.0,
'duration': 600.0,
},
},
}),
200,
));
}
group('RoutingService', () {
test('empty tags are filtered from request body', () async {
// Profile with empty tag values (like builtin-flock has camera:mount: '')
@@ -57,29 +81,7 @@ void main() {
];
when(() => mockAppState.enabledProfiles).thenReturn(profiles);
// Capture the request body
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((invocation) async {
return http.Response(
json.encode({
'ok': true,
'result': {
'route': {
'coordinates': [
[-77.0, 38.9],
[-77.1, 39.0],
],
'distance': 1000.0,
'duration': 600.0,
},
},
}),
200,
);
});
stubSuccessResponse();
await service.calculateRoute(start: start, end: end);
@@ -147,7 +149,7 @@ void main() {
reasonPhrase: 'Bad Request',
));
expect(
await expectLater(
() => service.calculateRoute(start: start, end: end),
throwsA(isA<RoutingException>().having(
(e) => e.message,
@@ -166,7 +168,7 @@ void main() {
body: any(named: 'body'),
)).thenThrow(http.ClientException('Connection refused'));
expect(
await expectLater(
() => service.calculateRoute(start: start, end: end),
throwsA(isA<RoutingException>().having(
(e) => e.message,
@@ -176,7 +178,7 @@ void main() {
);
});
test('API-level error surfaces alprwatch message', () async {
test('API-level error surfaces message', () async {
when(() => mockAppState.enabledProfiles).thenReturn([]);
when(() => mockClient.post(
@@ -191,7 +193,7 @@ void main() {
200,
));
expect(
await expectLater(
() => service.calculateRoute(start: start, end: end),
throwsA(isA<RoutingException>().having(
(e) => e.message,
@@ -201,4 +203,299 @@ void main() {
);
});
});
group('fallback behavior', () {
test('falls back to secondary on server error (500) after retries', () async {
when(() => mockAppState.enabledProfiles).thenReturn([]);
int callCount = 0;
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((invocation) async {
callCount++;
final uri = invocation.positionalArguments[0] as Uri;
if (uri.host == 'api.dontgetflocked.com') {
return http.Response('Internal Server Error', 500,
reasonPhrase: 'Internal Server Error');
}
// Fallback succeeds
return http.Response(
json.encode({
'ok': true,
'result': {
'route': {
'coordinates': [
[-77.0, 38.9],
[-77.1, 39.0],
],
'distance': 5000.0,
'duration': 300.0,
},
},
}),
200,
);
});
final result = await service.calculateRoute(start: start, end: end);
expect(result.distanceMeters, equals(5000.0));
// 2 primary attempts (1 + 1 retry) + 1 fallback = 3
expect(callCount, equals(3));
});
test('falls back on 502 (GraphHopper unavailable) after retries', () async {
when(() => mockAppState.enabledProfiles).thenReturn([]);
int callCount = 0;
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((invocation) async {
callCount++;
final uri = invocation.positionalArguments[0] as Uri;
if (uri.host == 'api.dontgetflocked.com') {
return http.Response('Bad Gateway', 502, reasonPhrase: 'Bad Gateway');
}
return http.Response(
json.encode({
'ok': true,
'result': {
'route': {
'coordinates': [[-77.0, 38.9]],
'distance': 100.0,
'duration': 60.0,
},
},
}),
200,
);
});
final result = await service.calculateRoute(start: start, end: end);
expect(result.distanceMeters, equals(100.0));
// 2 primary attempts + 1 fallback = 3
expect(callCount, equals(3));
});
test('falls back on network error after retries', () async {
when(() => mockAppState.enabledProfiles).thenReturn([]);
int callCount = 0;
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((invocation) async {
callCount++;
final uri = invocation.positionalArguments[0] as Uri;
if (uri.host == 'api.dontgetflocked.com') {
throw http.ClientException('Connection refused');
}
return http.Response(
json.encode({
'ok': true,
'result': {
'route': {
'coordinates': [[-77.0, 38.9]],
'distance': 100.0,
'duration': 60.0,
},
},
}),
200,
);
});
final result = await service.calculateRoute(start: start, end: end);
expect(result.distanceMeters, equals(100.0));
// 2 primary attempts + 1 fallback = 3
expect(callCount, equals(3));
});
test('429 triggers fallback without retrying primary', () async {
when(() => mockAppState.enabledProfiles).thenReturn([]);
int callCount = 0;
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((invocation) async {
callCount++;
final uri = invocation.positionalArguments[0] as Uri;
if (uri.host == 'api.dontgetflocked.com') {
return http.Response('Too Many Requests', 429,
reasonPhrase: 'Too Many Requests');
}
return http.Response(
json.encode({
'ok': true,
'result': {
'route': {
'coordinates': [[-77.0, 38.9]],
'distance': 200.0,
'duration': 120.0,
},
},
}),
200,
);
});
final result = await service.calculateRoute(start: start, end: end);
expect(result.distanceMeters, equals(200.0));
// 1 primary (no retry on 429/fallback disposition) + 1 fallback = 2
expect(callCount, equals(2));
});
test('does NOT fallback on 400 (validation error)', () async {
when(() => mockAppState.enabledProfiles).thenReturn([]);
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((_) async => http.Response(
'Bad Request: missing start', 400,
reasonPhrase: 'Bad Request'));
await expectLater(
() => service.calculateRoute(start: start, end: end),
throwsA(isA<RoutingException>().having(
(e) => e.statusCode, 'statusCode', 400)),
);
// Only one call — no retry, no fallback (abort disposition)
verify(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).called(1);
});
test('does NOT fallback on 403 (all 4xx except 429 abort)', () async {
when(() => mockAppState.enabledProfiles).thenReturn([]);
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((_) async => http.Response(
'Forbidden', 403,
reasonPhrase: 'Forbidden'));
await expectLater(
() => service.calculateRoute(start: start, end: end),
throwsA(isA<RoutingException>().having(
(e) => e.statusCode, 'statusCode', 403)),
);
// Only one call — no retry, no fallback (abort disposition)
verify(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).called(1);
});
test('does NOT fallback on API-level business logic errors', () async {
when(() => mockAppState.enabledProfiles).thenReturn([]);
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((_) async => http.Response(
json.encode({
'ok': false,
'error': 'No route found',
}),
200,
));
await expectLater(
() => service.calculateRoute(start: start, end: end),
throwsA(isA<RoutingException>().having(
(e) => e.isApiError, 'isApiError', true)),
);
verify(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).called(1);
});
test('primary fails then fallback also fails -> error propagated', () async {
when(() => mockAppState.enabledProfiles).thenReturn([]);
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((_) async => http.Response(
'Internal Server Error', 500,
reasonPhrase: 'Internal Server Error'));
await expectLater(
() => service.calculateRoute(start: start, end: end),
throwsA(isA<RoutingException>().having(
(e) => e.statusCode, 'statusCode', 500)),
);
// 2 primary attempts + 2 fallback attempts = 4
verify(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).called(4);
});
test('does NOT fallback when using custom baseUrl', () async {
final customService = RoutingService(
client: mockClient,
baseUrl: 'https://custom.example.com/route',
);
when(() => mockAppState.enabledProfiles).thenReturn([]);
when(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).thenAnswer((_) async => http.Response(
'Service Unavailable', 503,
reasonPhrase: 'Service Unavailable'));
await expectLater(
() => customService.calculateRoute(start: start, end: end),
throwsA(isA<RoutingException>()),
);
// 2 attempts (1 + 1 retry), no fallback with custom URL
verify(() => mockClient.post(
any(),
headers: any(named: 'headers'),
body: any(named: 'body'),
)).called(2);
});
});
group('RoutingException', () {
test('statusCode is preserved', () {
const e = RoutingException('test', statusCode: 502);
expect(e.statusCode, 502);
expect(e.isApiError, false);
});
test('isApiError flag works', () {
const e = RoutingException('test', isApiError: true);
expect(e.isApiError, true);
expect(e.statusCode, isNull);
});
});
}

254
test/services/service_policy_test.dart
View File

@@ -5,10 +5,6 @@ import 'package:deflockapp/services/service_policy.dart';
void main() {
group('ServicePolicyResolver', () {
setUp(() {
ServicePolicyResolver.clearCustomPolicies();
});
group('resolveType', () {
test('resolves OSM editing API from production URL', () {
expect(
@@ -200,63 +196,6 @@ void main() {
});
});
group('custom policy overrides', () {
test('custom override takes precedence over built-in', () {
ServicePolicyResolver.registerCustomPolicy(
'overpass-api.de',
const ServicePolicy.custom(maxConcurrent: 20, allowsOffline: true),
);
final policy = ServicePolicyResolver.resolve(
'https://overpass-api.de/api/interpreter',
);
expect(policy.maxConcurrentRequests, 20);
});
test('custom policy for self-hosted tiles allows offline', () {
ServicePolicyResolver.registerCustomPolicy(
'tiles.myserver.com',
const ServicePolicy.custom(allowsOffline: true, maxConcurrent: 16),
);
final policy = ServicePolicyResolver.resolve(
'https://tiles.myserver.com/{z}/{x}/{y}.png',
);
expect(policy.allowsOfflineDownload, true);
expect(policy.maxConcurrentRequests, 16);
});
test('removing custom override restores built-in policy', () {
ServicePolicyResolver.registerCustomPolicy(
'overpass-api.de',
const ServicePolicy.custom(maxConcurrent: 20),
);
expect(
ServicePolicyResolver.resolve('https://overpass-api.de/api/interpreter').maxConcurrentRequests,
20,
);
ServicePolicyResolver.removeCustomPolicy('overpass-api.de');
// Should fall back to built-in Overpass policy (maxConcurrent: 0 = managed elsewhere)
expect(
ServicePolicyResolver.resolve('https://overpass-api.de/api/interpreter').maxConcurrentRequests,
0,
);
});
test('clearCustomPolicies removes all overrides', () {
ServicePolicyResolver.registerCustomPolicy('a.com', const ServicePolicy.custom(maxConcurrent: 1));
ServicePolicyResolver.registerCustomPolicy('b.com', const ServicePolicy.custom(maxConcurrent: 2));
ServicePolicyResolver.clearCustomPolicies();
// Both should now return custom (default) policy
expect(
ServicePolicyResolver.resolve('https://a.com/test').maxConcurrentRequests,
8, // default custom maxConcurrent
);
});
});
});
group('ServiceRateLimiter', () {
@@ -423,4 +362,197 @@ void main() {
expect(policy.attributionUrl, 'https://example.com/license');
});
});
group('ResiliencePolicy', () {
test('retryDelay uses exponential backoff', () {
const policy = ResiliencePolicy(
retryBackoffBase: Duration(milliseconds: 100),
retryBackoffMaxMs: 2000,
);
expect(policy.retryDelay(0), const Duration(milliseconds: 100));
expect(policy.retryDelay(1), const Duration(milliseconds: 200));
expect(policy.retryDelay(2), const Duration(milliseconds: 400));
});
test('retryDelay clamps to max', () {
const policy = ResiliencePolicy(
retryBackoffBase: Duration(milliseconds: 1000),
retryBackoffMaxMs: 3000,
);
expect(policy.retryDelay(0), const Duration(milliseconds: 1000));
expect(policy.retryDelay(1), const Duration(milliseconds: 2000));
expect(policy.retryDelay(2), const Duration(milliseconds: 3000)); // clamped
expect(policy.retryDelay(10), const Duration(milliseconds: 3000)); // clamped
});
});
group('executeWithFallback', () {
const policy = ResiliencePolicy(
maxRetries: 2,
retryBackoffBase: Duration.zero, // no delay in tests
);
test('abort error stops immediately, no fallback', () async {
int callCount = 0;
await expectLater(
() => executeWithFallback<String>(
primaryUrl: 'https://primary.example.com',
fallbackUrl: 'https://fallback.example.com',
execute: (url) {
callCount++;
throw Exception('bad request');
},
classifyError: (_) => ErrorDisposition.abort,
policy: policy,
),
throwsA(isA<Exception>()),
);
expect(callCount, 1); // no retries, no fallback
});
test('fallback error skips retries, goes to fallback', () async {
final urlsSeen = <String>[];
final result = await executeWithFallback<String>(
primaryUrl: 'https://primary.example.com',
fallbackUrl: 'https://fallback.example.com',
execute: (url) {
urlsSeen.add(url);
if (url.contains('primary')) {
throw Exception('rate limited');
}
return Future.value('ok from fallback');
},
classifyError: (_) => ErrorDisposition.fallback,
policy: policy,
);
expect(result, 'ok from fallback');
// 1 primary (no retries) + 1 fallback = 2
expect(urlsSeen, ['https://primary.example.com', 'https://fallback.example.com']);
});
test('retry error retries N times then falls back', () async {
final urlsSeen = <String>[];
final result = await executeWithFallback<String>(
primaryUrl: 'https://primary.example.com',
fallbackUrl: 'https://fallback.example.com',
execute: (url) {
urlsSeen.add(url);
if (url.contains('primary')) {
throw Exception('server error');
}
return Future.value('ok from fallback');
},
classifyError: (_) => ErrorDisposition.retry,
policy: policy,
);
expect(result, 'ok from fallback');
// 3 primary attempts (1 + 2 retries) + 1 fallback = 4
expect(urlsSeen.where((u) => u.contains('primary')).length, 3);
expect(urlsSeen.where((u) => u.contains('fallback')).length, 1);
});
test('no fallback URL rethrows after retries', () async {
int callCount = 0;
await expectLater(
() => executeWithFallback<String>(
primaryUrl: 'https://primary.example.com',
fallbackUrl: null,
execute: (url) {
callCount++;
throw Exception('server error');
},
classifyError: (_) => ErrorDisposition.retry,
policy: policy,
),
throwsA(isA<Exception>()),
);
// 3 attempts (1 + 2 retries), then rethrow
expect(callCount, 3);
});
test('fallback disposition with no fallback URL rethrows immediately', () async {
int callCount = 0;
await expectLater(
() => executeWithFallback<String>(
primaryUrl: 'https://primary.example.com',
fallbackUrl: null,
execute: (url) {
callCount++;
throw Exception('rate limited');
},
classifyError: (_) => ErrorDisposition.fallback,
policy: policy,
),
throwsA(isA<Exception>()),
);
// Only 1 attempt — fallback disposition skips retries, and no fallback URL
expect(callCount, 1);
});
test('both fail propagates last error', () async {
await expectLater(
() => executeWithFallback<String>(
primaryUrl: 'https://primary.example.com',
fallbackUrl: 'https://fallback.example.com',
execute: (url) {
if (url.contains('fallback')) {
throw Exception('fallback also failed');
}
throw Exception('primary failed');
},
classifyError: (_) => ErrorDisposition.retry,
policy: policy,
),
throwsA(isA<Exception>().having(
(e) => e.toString(), 'message', contains('fallback also failed'))),
);
});
test('success on first try returns immediately', () async {
int callCount = 0;
final result = await executeWithFallback<String>(
primaryUrl: 'https://primary.example.com',
fallbackUrl: 'https://fallback.example.com',
execute: (url) {
callCount++;
return Future.value('success');
},
classifyError: (_) => ErrorDisposition.retry,
policy: policy,
);
expect(result, 'success');
expect(callCount, 1);
});
test('success after retry does not try fallback', () async {
int callCount = 0;
final result = await executeWithFallback<String>(
primaryUrl: 'https://primary.example.com',
fallbackUrl: 'https://fallback.example.com',
execute: (url) {
callCount++;
if (callCount == 1) throw Exception('transient');
return Future.value('recovered');
},
classifyError: (_) => ErrorDisposition.retry,
policy: policy,
);
expect(result, 'recovered');
expect(callCount, 2); // 1 fail + 1 success, no fallback
});
});
}