CalvinBackup/deflock-app
1
0
Fork 0
mirror of https://github.com/FoggedLens/deflock-app.git synced 2026-05-20 23:44:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

too much, sorry

Browse Source
This commit is contained in:
stopflock
2025-10-21 15:11:50 -05:00
parent 2ccd01c691
commit de0bd7f275
15 changed files with 259 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1
View File
@@ -97,6 +97,7 @@ cp lib/keys.dart.example lib/keys.dart
- Clean cache when nodes have disappeared / been deleted by others / queue item was deleted
- Improve offline area node refresh live display
- Add default operator profiles (Lowe’s etc)
- Add Rekor, generic PTZ profiles
### Future Features & Wishlist
- Update offline area nodes while browsing?
lib/dev_config.dart
+6 -8
View File
@@ -79,14 +79,12 @@ const int kProximityAlertMinDistance = 50; // meters
const int kProximityAlertMaxDistance = 1000; // meters
const Duration kProximityAlertCooldown = Duration(minutes: 10); // Cooldown between alerts for same node
// Tile/OSM fetch retry parameters (for tunable backoff)
const int kTileFetchMaxAttempts = 3;
const int kTileFetchInitialDelayMs = 4000;
const int kTileFetchJitter1Ms = 1000;
const int kTileFetchSecondDelayMs = 15000;
const int kTileFetchJitter2Ms = 4000;
const int kTileFetchThirdDelayMs = 60000;
const int kTileFetchJitter3Ms = 5000;
// Tile fetch retry parameters (configurable backoff system)
const int kTileFetchMaxAttempts = 6; // Number of retry attempts before giving up
const int kTileFetchInitialDelayMs = 1000; // Base delay for first retry (1 second)
const double kTileFetchBackoffMultiplier = 1.5; // Multiply delay by this each attempt
const int kTileFetchMaxDelayMs = 8000; // Cap delays at this value (8 seconds max)
const int kTileFetchRandomJitterMs = 500; // Random fuzz to add (0 to 500ms)
// User download max zoom span (user can download up to kMaxUserDownloadZoomSpan zooms above min)
const int kMaxUserDownloadZoomSpan = 7;
lib/screens/about_screen.dart
-2
View File
@@ -102,8 +102,6 @@ class AboutScreen extends StatelessWidget {
_buildLinkText(context, 'Source Code', 'https://github.com/FoggedLens/deflock-app'),
const SizedBox(height: 8),
_buildLinkText(context, 'Contact', 'https://deflock.me/contact'),
const SizedBox(height: 8),
_buildLinkText(context, 'Donate', 'https://deflock.me/donate'),
const SizedBox(height: 24),
// Divider for account management section
lib/services/map_data_provider.dart
+22 -46
View File
@@ -93,62 +93,33 @@ class MapDataProvider {
// Production mode: use pre-fetch service for efficient area loading
final preFetchService = PrefetchAreaService();
final List<Future<List<OsmNode>>> futures = [];
// Always try to get local nodes (fast, cached)
futures.add(fetchLocalNodes(
// Always get local nodes first (fast, from cache)
final localNodes = await fetchLocalNodes(
bounds: bounds,
profiles: profiles,
maxNodes: AppState.instance.maxCameras,
));
);
// Check if we need to fetch remote data or if pre-fetch covers this area
if (preFetchService.isWithinPreFetchedArea(bounds, profiles, uploadMode)) {
// Current view is within pre-fetched area, just use local cache
debugPrint('[MapDataProvider] Using pre-fetched data from cache');
final localNodes = await futures[0];
return localNodes.take(AppState.instance.maxCameras).toList();
} else {
// Not within pre-fetched area, request pre-fetch and also get immediate data
// Check if we need to trigger a new pre-fetch
if (!preFetchService.isWithinPreFetchedArea(bounds, profiles, uploadMode)) {
// Outside pre-fetched area - trigger new pre-fetch but don't wait for it
debugPrint('[MapDataProvider] Outside pre-fetched area, triggering new pre-fetch');
preFetchService.requestPreFetchIfNeeded(
viewBounds: bounds,
profiles: profiles,
uploadMode: uploadMode,
);
// For immediate response, still try to get some remote data for current view
futures.add(_fetchRemoteNodes(
bounds: bounds,
profiles: profiles,
uploadMode: uploadMode,
maxResults: AppState.instance.maxCameras,
).catchError((e) {
debugPrint('[MapDataProvider] Remote node fetch failed, error: $e. Continuing with local only.');
return <OsmNode>[]; // Return empty list on remote failure
}));
// Wait for both, then merge with deduplication by node ID
final results = await Future.wait(futures);
final localNodes = results[0];
final remoteNodes = results[1];
// Merge with deduplication - prefer remote data over local for same node ID
final Map<int, OsmNode> mergedNodes = {};
// Add local nodes first
for (final node in localNodes) {
mergedNodes[node.id] = node;
}
// Add remote nodes, overwriting any local duplicates
for (final node in remoteNodes) {
mergedNodes[node.id] = node;
}
// Apply maxCameras limit to the merged result
final finalNodes = mergedNodes.values.take(AppState.instance.maxCameras).toList();
return finalNodes;
} else {
debugPrint('[MapDataProvider] Using existing pre-fetched area cache');
}
// Apply rendering limit and warn if nodes are being excluded
final maxNodes = AppState.instance.maxCameras;
if (localNodes.length > maxNodes) {
NetworkStatus.instance.reportNodeLimitReached(localNodes.length, maxNodes);
}
return localNodes.take(maxNodes).toList();
}
} finally {
// Always report node completion, regardless of success or failure
@@ -230,6 +201,11 @@ class MapDataProvider {
void clearTileQueue() {
clearRemoteTileQueue();
}
/// Clear only tile requests that are no longer visible in the current bounds
void clearTileQueueSelective(LatLngBounds currentBounds) {
clearRemoteTileQueueSelective(currentBounds);
}
/// Fetch remote nodes with Overpass first, OSM API fallback
Future<List<OsmNode>> _fetchRemoteNodes({
lib/services/map_data_submodules/nodes_from_overpass.dart
+29 -7
View File
@@ -47,6 +47,13 @@ Future<List<OsmNode>> _fetchOverpassNodesWithSplitting({
profiles: profiles,
maxResults: maxResults,
);
} on OverpassRateLimitException catch (e) {
// Rate limits should NOT be split - just fail with extended backoff
debugPrint('[fetchOverpassNodes] Rate limited - using extended backoff, not splitting');
// Wait longer for rate limits before giving up entirely
await Future.delayed(const Duration(seconds: 30));
rethrow; // Let caller handle as a regular failure
} on OverpassNodeLimitException {
// If we've hit max split depth, give up to avoid infinite recursion
if (splitDepth >= maxSplitDepth) {
@@ -99,16 +106,31 @@ Future<List<OsmNode>> _fetchSingleOverpassQuery({
final errorBody = response.body;
debugPrint('[fetchOverpassNodes] Overpass API error: $errorBody');
// Check if it's a node limit exceeded error
if (errorBody.contains('too many') ||
errorBody.contains('50000') ||
errorBody.contains('50,000') ||
errorBody.contains('limit') ||
errorBody.contains('runtime error')) {
debugPrint('[fetchOverpassNodes] Detected node limit error, will attempt splitting');
// Check if it's specifically the 50k node limit error (HTTP 400)
// Exact message: "You requested too many nodes (limit is 50000)"
if (errorBody.contains('too many nodes') &&
errorBody.contains('50000')) {
debugPrint('[fetchOverpassNodes] Detected 50k node limit error, will attempt splitting');
throw OverpassNodeLimitException('Query exceeded node limit', serverResponse: errorBody);
}
// Check for timeout errors that indicate query complexity (should split)
// Common timeout messages from Overpass
if (errorBody.contains('timeout') ||
errorBody.contains('runtime limit exceeded') ||
errorBody.contains('Query timed out')) {
debugPrint('[fetchOverpassNodes] Detected timeout error, will attempt splitting to reduce complexity');
throw OverpassNodeLimitException('Query timed out', serverResponse: errorBody);
}
// Check for rate limiting (should NOT split - needs longer backoff)
if (errorBody.contains('rate limited') ||
errorBody.contains('too many requests') ||
response.statusCode == 429) {
debugPrint('[fetchOverpassNodes] Rate limited by Overpass API - needs extended backoff');
throw OverpassRateLimitException('Rate limited by server', serverResponse: errorBody);
}
NetworkStatus.instance.reportOverpassIssue();
return [];
}
lib/services/map_data_submodules/tiles_from_remote.dart
+108 -14
View File
@@ -3,6 +3,8 @@ import 'dart:io';
import 'dart:async';
import 'package:http/http.dart' as http;
import 'package:flutter/foundation.dart';
import 'package:latlong2/latlong.dart';
import 'package:flutter_map/flutter_map.dart';
import 'package:deflockapp/dev_config.dart';
import '../network_status.dart';
@@ -18,6 +20,77 @@ void clearRemoteTileQueue() {
}
}
/// Clear only tile requests that are no longer visible in the given bounds
void clearRemoteTileQueueSelective(LatLngBounds currentBounds) {
final clearedCount = _tileFetchSemaphore.clearStaleRequests((z, x, y) {
// Return true if tile should be cleared (i.e., is NOT visible)
return !_isTileVisible(z, x, y, currentBounds);
});
if (clearedCount > 0) {
debugPrint('[RemoteTiles] Selectively cleared $clearedCount non-visible tile requests');
}
}
/// Calculate retry delay using configurable backoff strategy.
/// Uses: initialDelay * (multiplier ^ (attempt - 1)) + randomJitter, capped at maxDelay
int _calculateRetryDelay(int attempt, Random random) {
// Calculate exponential backoff: initialDelay * (multiplier ^ (attempt - 1))
final baseDelay = (kTileFetchInitialDelayMs *
pow(kTileFetchBackoffMultiplier, attempt - 1)).round();
// Add random jitter to avoid thundering herd
final jitter = random.nextInt(kTileFetchRandomJitterMs + 1);
// Apply max delay cap
return (baseDelay + jitter).clamp(0, kTileFetchMaxDelayMs);
}
/// Convert tile coordinates to lat/lng bounds for spatial filtering
class _TileBounds {
final double north, south, east, west;
_TileBounds({required this.north, required this.south, required this.east, required this.west});
}
/// Calculate the lat/lng bounds for a given tile
_TileBounds _tileToBounds(int z, int x, int y) {
final n = pow(2, z);
final lon1 = (x / n) * 360.0 - 180.0;
final lon2 = ((x + 1) / n) * 360.0 - 180.0;
final lat1 = _yToLatitude(y, z);
final lat2 = _yToLatitude(y + 1, z);
return _TileBounds(
north: max(lat1, lat2),
south: min(lat1, lat2),
east: max(lon1, lon2),
west: min(lon1, lon2),
);
}
/// Convert tile Y coordinate to latitude
double _yToLatitude(int y, int z) {
final n = pow(2, z);
final latRad = atan(_sinh(pi * (1 - 2 * y / n)));
return latRad * 180.0 / pi;
}
/// Hyperbolic sine function: sinh(x) = (e^x - e^(-x)) / 2
double _sinh(double x) {
return (exp(x) - exp(-x)) / 2;
}
/// Check if a tile intersects with the current view bounds
bool _isTileVisible(int z, int x, int y, LatLngBounds viewBounds) {
final tileBounds = _tileToBounds(z, x, y);
// Check if tile bounds intersect with view bounds
return !(tileBounds.east < viewBounds.west ||
tileBounds.west > viewBounds.east ||
tileBounds.north < viewBounds.south ||
tileBounds.south > viewBounds.north);
}
/// Fetches a tile from any remote provider, with in-memory retries/backoff, and global concurrency limit.
@@ -31,16 +104,10 @@ Future<List<int>> fetchRemoteTile({
const int maxAttempts = kTileFetchMaxAttempts;
int attempt = 0;
final random = Random();
final delays = [
kTileFetchInitialDelayMs + random.nextInt(kTileFetchJitter1Ms),
kTileFetchSecondDelayMs + random.nextInt(kTileFetchJitter2Ms),
kTileFetchThirdDelayMs + random.nextInt(kTileFetchJitter3Ms),
];
final hostInfo = Uri.parse(url).host; // For logging
while (true) {
await _tileFetchSemaphore.acquire();
await _tileFetchSemaphore.acquire(z: z, x: x, y: y);
try {
// Only log on first attempt or errors
if (attempt == 1) {
@@ -71,7 +138,7 @@ Future<List<int>> fetchRemoteTile({
rethrow;
}
final delay = delays[attempt - 1].clamp(0, 60000);
final delay = _calculateRetryDelay(attempt, random);
if (attempt == 1) {
debugPrint("[fetchRemoteTile] Attempt $attempt for $z/$x/$y from $hostInfo failed: $e. Retrying in ${delay}ms.");
}
@@ -97,28 +164,42 @@ Future<List<int>> fetchOSMTile({
);
}
/// Simple counting semaphore, suitable for single-thread Flutter concurrency
/// Enhanced tile request entry that tracks coordinates for spatial filtering
class _TileRequest {
final int z, x, y;
final VoidCallback callback;
_TileRequest({required this.z, required this.x, required this.y, required this.callback});
}
/// Spatially-aware counting semaphore for tile requests
class _SimpleSemaphore {
final int _max;
int _current = 0;
final List<VoidCallback> _queue = [];
final List<_TileRequest> _queue = [];
_SimpleSemaphore(this._max);
Future<void> acquire() async {
Future<void> acquire({int? z, int? x, int? y}) async {
if (_current < _max) {
_current++;
return;
} else {
final c = Completer<void>();
_queue.add(() => c.complete());
final request = _TileRequest(
z: z ?? -1,
x: x ?? -1,
y: y ?? -1,
callback: () => c.complete(),
);
_queue.add(request);
await c.future;
}
}
void release() {
if (_queue.isNotEmpty) {
final callback = _queue.removeAt(0);
callback();
final request = _queue.removeAt(0);
request.callback();
} else {
_current--;
}
@@ -130,4 +211,17 @@ class _SimpleSemaphore {
_queue.clear();
return clearedCount;
}
/// Clear only tiles that don't pass the visibility filter
int clearStaleRequests(bool Function(int z, int x, int y) isStale) {
final initialCount = _queue.length;
_queue.removeWhere((request) => isStale(request.z, request.x, request.y));
final clearedCount = initialCount - _queue.length;
if (clearedCount > 0) {
debugPrint('[SimpleSemaphore] Cleared $clearedCount stale tile requests, kept ${_queue.length}');
}
return clearedCount;
}
}
lib/services/network_status.dart
+22 -1
View File
@@ -4,7 +4,7 @@ import 'dart:async';
import '../app_state.dart';
enum NetworkIssueType { osmTiles, overpassApi, both }
enum NetworkStatusType { waiting, issues, timedOut, noData, ready, success }
enum NetworkStatusType { waiting, issues, timedOut, noData, ready, success, nodeLimitReached }
/// Simple loading state for dual-source async operations (brutalist approach)
enum LoadingState { ready, waiting, success, timeout }
@@ -25,6 +25,8 @@ class NetworkStatus extends ChangeNotifier {
Timer? _waitingTimer;
Timer? _noDataResetTimer;
Timer? _successResetTimer;
bool _nodeLimitReached = false;
Timer? _nodeLimitResetTimer;
// New dual-source loading state (brutalist approach)
LoadingState _tileLoadingState = LoadingState.ready;
@@ -41,6 +43,7 @@ class NetworkStatus extends ChangeNotifier {
bool get isTimedOut => _isTimedOut;
bool get hasNoData => _hasNoData;
bool get hasSuccess => _hasSuccess;
bool get nodeLimitReached => _nodeLimitReached;
// New dual-source getters (brutalist approach)
LoadingState get tileLoadingState => _tileLoadingState;
@@ -63,6 +66,7 @@ class NetworkStatus extends ChangeNotifier {
if (_isTimedOut) return NetworkStatusType.timedOut;
if (_hasNoData) return NetworkStatusType.noData;
if (_hasSuccess) return NetworkStatusType.success;
if (_nodeLimitReached) return NetworkStatusType.nodeLimitReached;
return NetworkStatusType.ready;
}
@@ -206,6 +210,22 @@ class NetworkStatus extends ChangeNotifier {
notifyListeners();
}
}
/// Show notification that node display limit was reached
void reportNodeLimitReached(int totalNodes, int maxNodes) {
_nodeLimitReached = true;
notifyListeners();
debugPrint('[NetworkStatus] Node display limit reached: $totalNodes found, showing $maxNodes');
// Auto-clear after 8 seconds
_nodeLimitResetTimer?.cancel();
_nodeLimitResetTimer = Timer(const Duration(seconds: 8), () {
if (_nodeLimitReached) {
_nodeLimitReached = false;
notifyListeners();
}
});
}
@@ -316,6 +336,7 @@ class NetworkStatus extends ChangeNotifier {
_tileTimeoutTimer?.cancel();
_nodeTimeoutTimer?.cancel();
_successDisplayTimer?.cancel();
_nodeLimitResetTimer?.cancel();
super.dispose();
}
}
lib/services/overpass_node_limit_exception.dart
+12
View File
@@ -8,4 +8,16 @@ class OverpassNodeLimitException implements Exception {
@override
String toString() => 'OverpassNodeLimitException: $message';
}
/// Exception thrown when Overpass API rate limits the request.
/// Should trigger longer backoff delays, not area splitting.
class OverpassRateLimitException implements Exception {
final String message;
final String? serverResponse;
OverpassRateLimitException(this.message, {this.serverResponse});
@override
String toString() => 'OverpassRateLimitException: $message';
}
lib/services/prefetch_area_service.dart
+3 -3
View File
@@ -100,17 +100,17 @@ class PrefetchAreaService {
debugPrint('[PrefetchAreaService] Starting pre-fetch for area: ${preFetchArea.south},${preFetchArea.west} to ${preFetchArea.north},${preFetchArea.east}');
// Fetch nodes for the expanded area (no maxResults limit for pre-fetch)
// Fetch nodes for the expanded area (unlimited - let splitting handle 50k limit)
final nodes = await fetchOverpassNodes(
bounds: preFetchArea,
profiles: profiles,
uploadMode: uploadMode,
maxResults: 0, // Unlimited - let Overpass splitting handle large areas
maxResults: 0, // Unlimited - our splitting system handles the 50k limit gracefully
);
debugPrint('[PrefetchAreaService] Pre-fetch completed: ${nodes.length} nodes retrieved');
// Update cache with new nodes
// Update cache with new nodes (fresh data overwrites stale, but preserves underscore tags)
if (nodes.isNotEmpty) {
NodeCache.instance.addOrUpdate(nodes);
}
lib/services/simple_tile_service.dart
+7
View File
@@ -1,5 +1,7 @@
import 'package:http/http.dart' as http;
import 'package:flutter/foundation.dart';
import 'package:latlong2/latlong.dart';
import 'package:flutter_map/flutter_map.dart';
import '../app_state.dart';
import 'map_data_provider.dart';
@@ -93,6 +95,11 @@ class SimpleTileHttpClient extends http.BaseClient {
void clearTileQueue() {
_mapDataProvider.clearTileQueue();
}
/// Clear only tile requests that are no longer visible in the current bounds
void clearStaleRequests(LatLngBounds currentBounds) {
_mapDataProvider.clearTileQueueSelective(currentBounds);
}
/// Format date for HTTP headers (RFC 7231)
String _httpDateFormat(DateTime date) {
lib/widgets/map/tile_layer_manager.dart
+6
View File
@@ -1,5 +1,6 @@
import 'package:flutter/material.dart';
import 'package:flutter_map/flutter_map.dart';
import 'package:latlong2/latlong.dart';
import '../../models/tile_provider.dart' as models;
import '../../services/simple_tile_service.dart';
@@ -64,6 +65,11 @@ class TileLayerManager {
void clearTileQueueImmediate() {
_tileHttpClient.clearTileQueue();
}
/// Clear only tiles that are no longer visible in the current bounds
void clearStaleRequests({required LatLngBounds currentBounds}) {
_tileHttpClient.clearStaleRequests(currentBounds);
}
/// Build tile layer widget with current provider and type.
/// Uses fake domain that SimpleTileHttpClient can parse for cache separation.
lib/widgets/map_view.dart
+36 -5
View File
@@ -71,6 +71,9 @@ class MapViewState extends State<MapView> {
// Track zoom to clear queue on zoom changes
double? _lastZoom;
// Track map center to clear queue on significant panning
LatLng? _lastCenter;
// State for proximity alert banner
bool _showProximityBanner = false;
@@ -232,6 +235,22 @@ class MapViewState extends State<MapView> {
}
}
/// Check if the map has moved significantly enough to cancel stale tile requests.
/// Uses a simple distance threshold - roughly equivalent to 1/4 screen width at zoom 15.
bool _mapMovedSignificantly(LatLng? newCenter, LatLng? oldCenter) {
if (newCenter == null || oldCenter == null) return false;
// Calculate approximate distance in meters (rough calculation for performance)
final latDiff = (newCenter.latitude - oldCenter.latitude).abs();
final lngDiff = (newCenter.longitude - oldCenter.longitude).abs();
// Threshold: ~500 meters (roughly 1/4 screen at zoom 15)
// This prevents excessive cancellations on small movements while catching real pans
const double significantMovementThreshold = 0.005; // degrees (~500m at equator)
return latDiff > significantMovementThreshold || lngDiff > significantMovementThreshold;
}
/// Show zoom warning if user is below minimum zoom level
void _showZoomWarningIfNeeded(BuildContext context, double currentZoom, int minZoom) {
// Only show warning once per zoom level to avoid spam
@@ -542,17 +561,29 @@ class MapViewState extends State<MapView> {
// Start dual-source waiting when map moves (user is expecting new tiles AND nodes)
NetworkStatus.instance.setDualSourceWaiting();
// Only clear tile queue on significant ZOOM changes (not panning)
// Clear tile queue on tile level changes OR significant panning
final currentZoom = pos.zoom;
final zoomChanged = _lastZoom != null && (currentZoom - _lastZoom!).abs() > 0.5;
final currentCenter = pos.center;
final currentTileLevel = currentZoom.round();
final lastTileLevel = _lastZoom?.round();
final tileLevelChanged = lastTileLevel != null && currentTileLevel != lastTileLevel;
final centerMoved = _mapMovedSignificantly(currentCenter, _lastCenter);
if (zoomChanged) {
if (tileLevelChanged || centerMoved) {
_tileDebounce(() {
// Clear stale tile requests on zoom change (quietly)
_tileManager.clearTileQueueImmediate();
// Use selective clearing to only cancel tiles that are no longer visible
try {
final currentBounds = _controller.mapController.camera.visibleBounds;
_tileManager.clearStaleRequests(currentBounds: currentBounds);
} catch (e) {
// Fallback to clearing all if bounds calculation fails
debugPrint('[MapView] Could not get current bounds for selective clearing: $e');
_tileManager.clearTileQueueImmediate();
}
});
}
_lastZoom = currentZoom;
_lastCenter = currentCenter;
// Save map position (debounced to avoid excessive writes)
_mapPositionDebounce(() {
lib/widgets/network_status_indicator.dart
+6
View File
@@ -40,6 +40,12 @@ class NetworkStatusIndicator extends StatelessWidget {
color = Colors.green;
break;
case NetworkStatusType.nodeLimitReached:
message = 'Showing limit - increase in settings';
icon = Icons.visibility_off;
color = Colors.amber;
break;
case NetworkStatusType.issues:
switch (networkStatus.currentIssueType) {
case NetworkIssueType.osmTiles:
lib/widgets/welcome_dialog.dart
-1
View File
@@ -78,7 +78,6 @@ class _WelcomeDialogState extends State<WelcomeDialog> {
_buildLinkButton('Website', 'https://deflock.me'),
_buildLinkButton('GitHub', 'https://github.com/FoggedLens/deflock-app'),
_buildLinkButton('Discord', 'https://discord.gg/aV7v4R3sKT'),
_buildLinkButton('Donate', 'https://deflock.me/donate'),
],
),
const SizedBox(height: 16),
pubspec.yaml
+1 -1
View File
@@ -1,7 +1,7 @@
name: deflockapp
description: Map public surveillance infrastructure with OpenStreetMap
publish_to: "none"
version: 1.2.4+5 # The thing after the + is the version code, incremented with each release
version: 1.2.5+5 # The thing after the + is the version code, incremented with each release
environment:
sdk: ">=3.5.0 <4.0.0" # oauth2_client 4.x needs Dart 3.5+