too much, sorry

This commit is contained in:
stopflock
2025-10-21 15:11:50 -05:00
parent 2ccd01c691
commit de0bd7f275
15 changed files with 259 additions and 88 deletions
@@ -47,6 +47,13 @@ Future<List<OsmNode>> _fetchOverpassNodesWithSplitting({
profiles: profiles,
maxResults: maxResults,
);
} on OverpassRateLimitException catch (e) {
// Rate limits should NOT be split - just fail with extended backoff
debugPrint('[fetchOverpassNodes] Rate limited - using extended backoff, not splitting');
// Wait longer for rate limits before giving up entirely
await Future.delayed(const Duration(seconds: 30));
rethrow; // Let caller handle as a regular failure
} on OverpassNodeLimitException {
// If we've hit max split depth, give up to avoid infinite recursion
if (splitDepth >= maxSplitDepth) {
@@ -99,16 +106,31 @@ Future<List<OsmNode>> _fetchSingleOverpassQuery({
final errorBody = response.body;
debugPrint('[fetchOverpassNodes] Overpass API error: $errorBody');
// Check if it's a node limit exceeded error
if (errorBody.contains('too many') ||
errorBody.contains('50000') ||
errorBody.contains('50,000') ||
errorBody.contains('limit') ||
errorBody.contains('runtime error')) {
debugPrint('[fetchOverpassNodes] Detected node limit error, will attempt splitting');
// Check if it's specifically the 50k node limit error (HTTP 400)
// Exact message: "You requested too many nodes (limit is 50000)"
if (errorBody.contains('too many nodes') &&
errorBody.contains('50000')) {
debugPrint('[fetchOverpassNodes] Detected 50k node limit error, will attempt splitting');
throw OverpassNodeLimitException('Query exceeded node limit', serverResponse: errorBody);
}
// Check for timeout errors that indicate query complexity (should split)
// Common timeout messages from Overpass
if (errorBody.contains('timeout') ||
errorBody.contains('runtime limit exceeded') ||
errorBody.contains('Query timed out')) {
debugPrint('[fetchOverpassNodes] Detected timeout error, will attempt splitting to reduce complexity');
throw OverpassNodeLimitException('Query timed out', serverResponse: errorBody);
}
// Check for rate limiting (should NOT split - needs longer backoff)
if (errorBody.contains('rate limited') ||
errorBody.contains('too many requests') ||
response.statusCode == 429) {
debugPrint('[fetchOverpassNodes] Rate limited by Overpass API - needs extended backoff');
throw OverpassRateLimitException('Rate limited by server', serverResponse: errorBody);
}
NetworkStatus.instance.reportOverpassIssue();
return [];
}
@@ -3,6 +3,8 @@ import 'dart:io';
import 'dart:async';
import 'package:http/http.dart' as http;
import 'package:flutter/foundation.dart';
import 'package:latlong2/latlong.dart';
import 'package:flutter_map/flutter_map.dart';
import 'package:deflockapp/dev_config.dart';
import '../network_status.dart';
@@ -18,6 +20,77 @@ void clearRemoteTileQueue() {
}
}
/// Clear only tile requests that are no longer visible in the given bounds
void clearRemoteTileQueueSelective(LatLngBounds currentBounds) {
final clearedCount = _tileFetchSemaphore.clearStaleRequests((z, x, y) {
// Return true if tile should be cleared (i.e., is NOT visible)
return !_isTileVisible(z, x, y, currentBounds);
});
if (clearedCount > 0) {
debugPrint('[RemoteTiles] Selectively cleared $clearedCount non-visible tile requests');
}
}
/// Calculate retry delay using configurable backoff strategy.
/// Uses: initialDelay * (multiplier ^ (attempt - 1)) + randomJitter, capped at maxDelay
int _calculateRetryDelay(int attempt, Random random) {
// Calculate exponential backoff: initialDelay * (multiplier ^ (attempt - 1))
final baseDelay = (kTileFetchInitialDelayMs *
pow(kTileFetchBackoffMultiplier, attempt - 1)).round();
// Add random jitter to avoid thundering herd
final jitter = random.nextInt(kTileFetchRandomJitterMs + 1);
// Apply max delay cap
return (baseDelay + jitter).clamp(0, kTileFetchMaxDelayMs);
}
/// Convert tile coordinates to lat/lng bounds for spatial filtering
class _TileBounds {
final double north, south, east, west;
_TileBounds({required this.north, required this.south, required this.east, required this.west});
}
/// Calculate the lat/lng bounds for a given tile
_TileBounds _tileToBounds(int z, int x, int y) {
final n = pow(2, z);
final lon1 = (x / n) * 360.0 - 180.0;
final lon2 = ((x + 1) / n) * 360.0 - 180.0;
final lat1 = _yToLatitude(y, z);
final lat2 = _yToLatitude(y + 1, z);
return _TileBounds(
north: max(lat1, lat2),
south: min(lat1, lat2),
east: max(lon1, lon2),
west: min(lon1, lon2),
);
}
/// Convert tile Y coordinate to latitude
double _yToLatitude(int y, int z) {
final n = pow(2, z);
final latRad = atan(_sinh(pi * (1 - 2 * y / n)));
return latRad * 180.0 / pi;
}
/// Hyperbolic sine function: sinh(x) = (e^x - e^(-x)) / 2
double _sinh(double x) {
return (exp(x) - exp(-x)) / 2;
}
/// Check if a tile intersects with the current view bounds
bool _isTileVisible(int z, int x, int y, LatLngBounds viewBounds) {
final tileBounds = _tileToBounds(z, x, y);
// Check if tile bounds intersect with view bounds
return !(tileBounds.east < viewBounds.west ||
tileBounds.west > viewBounds.east ||
tileBounds.north < viewBounds.south ||
tileBounds.south > viewBounds.north);
}
/// Fetches a tile from any remote provider, with in-memory retries/backoff, and global concurrency limit.
@@ -31,16 +104,10 @@ Future<List<int>> fetchRemoteTile({
const int maxAttempts = kTileFetchMaxAttempts;
int attempt = 0;
final random = Random();
final delays = [
kTileFetchInitialDelayMs + random.nextInt(kTileFetchJitter1Ms),
kTileFetchSecondDelayMs + random.nextInt(kTileFetchJitter2Ms),
kTileFetchThirdDelayMs + random.nextInt(kTileFetchJitter3Ms),
];
final hostInfo = Uri.parse(url).host; // For logging
while (true) {
await _tileFetchSemaphore.acquire();
await _tileFetchSemaphore.acquire(z: z, x: x, y: y);
try {
// Only log on first attempt or errors
if (attempt == 1) {
@@ -71,7 +138,7 @@ Future<List<int>> fetchRemoteTile({
rethrow;
}
final delay = delays[attempt - 1].clamp(0, 60000);
final delay = _calculateRetryDelay(attempt, random);
if (attempt == 1) {
debugPrint("[fetchRemoteTile] Attempt $attempt for $z/$x/$y from $hostInfo failed: $e. Retrying in ${delay}ms.");
}
@@ -97,28 +164,42 @@ Future<List<int>> fetchOSMTile({
);
}
/// Simple counting semaphore, suitable for single-thread Flutter concurrency
/// Enhanced tile request entry that tracks coordinates for spatial filtering
class _TileRequest {
final int z, x, y;
final VoidCallback callback;
_TileRequest({required this.z, required this.x, required this.y, required this.callback});
}
/// Spatially-aware counting semaphore for tile requests
class _SimpleSemaphore {
final int _max;
int _current = 0;
final List<VoidCallback> _queue = [];
final List<_TileRequest> _queue = [];
_SimpleSemaphore(this._max);
Future<void> acquire() async {
Future<void> acquire({int? z, int? x, int? y}) async {
if (_current < _max) {
_current++;
return;
} else {
final c = Completer<void>();
_queue.add(() => c.complete());
final request = _TileRequest(
z: z ?? -1,
x: x ?? -1,
y: y ?? -1,
callback: () => c.complete(),
);
_queue.add(request);
await c.future;
}
}
void release() {
if (_queue.isNotEmpty) {
final callback = _queue.removeAt(0);
callback();
final request = _queue.removeAt(0);
request.callback();
} else {
_current--;
}
@@ -130,4 +211,17 @@ class _SimpleSemaphore {
_queue.clear();
return clearedCount;
}
/// Clear only tiles that don't pass the visibility filter
int clearStaleRequests(bool Function(int z, int x, int y) isStale) {
final initialCount = _queue.length;
_queue.removeWhere((request) => isStale(request.z, request.x, request.y));
final clearedCount = initialCount - _queue.length;
if (clearedCount > 0) {
debugPrint('[SimpleSemaphore] Cleared $clearedCount stale tile requests, kept ${_queue.length}');
}
return clearedCount;
}
}