diff --git a/lib/dev_config.dart b/lib/dev_config.dart index 588ddb2..91686ae 100644 --- a/lib/dev_config.dart +++ b/lib/dev_config.dart @@ -64,6 +64,11 @@ const int kOsmApiMinZoomLevel = 13; // Minimum zoom for OSM API bbox queries (sa const Duration kMarkerTapTimeout = Duration(milliseconds: 250); const Duration kDebounceCameraRefresh = Duration(milliseconds: 500); +// Pre-fetch area configuration +const double kPreFetchAreaExpansionMultiplier = 3.0; // Expand visible bounds by this factor for pre-fetching +const int kPreFetchZoomLevel = 10; // Always pre-fetch at this zoom level for consistent area sizes +const int kMaxPreFetchSplitDepth = 3; // Maximum recursive splits when hitting Overpass node limit + // Follow-me mode smooth transitions const Duration kFollowMeAnimationDuration = Duration(milliseconds: 600); const double kMinSpeedForRotationMps = 1.0; // Minimum speed (m/s) to apply rotation diff --git a/lib/services/map_data_provider.dart b/lib/services/map_data_provider.dart index 9b9bb83..9979b67 100644 --- a/lib/services/map_data_provider.dart +++ b/lib/services/map_data_provider.dart @@ -11,6 +11,7 @@ import 'map_data_submodules/tiles_from_remote.dart'; import 'map_data_submodules/nodes_from_local.dart'; import 'map_data_submodules/tiles_from_local.dart'; import 'network_status.dart'; +import 'prefetch_area_service.dart'; enum MapSource { local, remote, auto } // For future use @@ -89,7 +90,9 @@ class MapDataProvider { maxResults: AppState.instance.maxCameras, ); } else { - // Production mode: fetch both remote and local, then merge with deduplication + // Production mode: use pre-fetch service for efficient area loading + final preFetchService = PrefetchAreaService(); + final List>> futures = []; // Always try to get local nodes (fast, cached) @@ -99,38 +102,53 @@ class MapDataProvider { maxNodes: AppState.instance.maxCameras, )); - // Always try to get remote nodes (slower, fresh data) - futures.add(_fetchRemoteNodes( - bounds: bounds, - profiles: profiles, - uploadMode: uploadMode, - maxResults: AppState.instance.maxCameras, - ).catchError((e) { - debugPrint('[MapDataProvider] Remote node fetch failed, error: $e. Continuing with local only.'); - return []; // Return empty list on remote failure - })); - - // Wait for both, then merge with deduplication by node ID - final results = await Future.wait(futures); - final localNodes = results[0]; - final remoteNodes = results[1]; - - // Merge with deduplication - prefer remote data over local for same node ID - final Map mergedNodes = {}; - - // Add local nodes first - for (final node in localNodes) { - mergedNodes[node.id] = node; + // Check if we need to fetch remote data or if pre-fetch covers this area + if (preFetchService.isWithinPreFetchedArea(bounds, profiles, uploadMode)) { + // Current view is within pre-fetched area, just use local cache + debugPrint('[MapDataProvider] Using pre-fetched data from cache'); + final localNodes = await futures[0]; + return localNodes.take(AppState.instance.maxCameras).toList(); + } else { + // Not within pre-fetched area, request pre-fetch and also get immediate data + preFetchService.requestPreFetchIfNeeded( + viewBounds: bounds, + profiles: profiles, + uploadMode: uploadMode, + ); + + // For immediate response, still try to get some remote data for current view + futures.add(_fetchRemoteNodes( + bounds: bounds, + profiles: profiles, + uploadMode: uploadMode, + maxResults: AppState.instance.maxCameras, + ).catchError((e) { + debugPrint('[MapDataProvider] Remote node fetch failed, error: $e. Continuing with local only.'); + return []; // Return empty list on remote failure + })); + + // Wait for both, then merge with deduplication by node ID + final results = await Future.wait(futures); + final localNodes = results[0]; + final remoteNodes = results[1]; + + // Merge with deduplication - prefer remote data over local for same node ID + final Map mergedNodes = {}; + + // Add local nodes first + for (final node in localNodes) { + mergedNodes[node.id] = node; + } + + // Add remote nodes, overwriting any local duplicates + for (final node in remoteNodes) { + mergedNodes[node.id] = node; + } + + // Apply maxCameras limit to the merged result + final finalNodes = mergedNodes.values.take(AppState.instance.maxCameras).toList(); + return finalNodes; } - - // Add remote nodes, overwriting any local duplicates - for (final node in remoteNodes) { - mergedNodes[node.id] = node; - } - - // Apply maxCameras limit to the merged result - final finalNodes = mergedNodes.values.take(AppState.instance.maxCameras).toList(); - return finalNodes; } } finally { // Always report node completion, regardless of success or failure diff --git a/lib/services/map_data_submodules/nodes_from_overpass.dart b/lib/services/map_data_submodules/nodes_from_overpass.dart index 6eb4cc2..a644a27 100644 --- a/lib/services/map_data_submodules/nodes_from_overpass.dart +++ b/lib/services/map_data_submodules/nodes_from_overpass.dart @@ -8,17 +8,79 @@ import '../../models/node_profile.dart'; import '../../models/osm_node.dart'; import '../../models/pending_upload.dart'; import '../../app_state.dart'; +import '../../dev_config.dart'; import '../network_status.dart'; +import '../overpass_node_limit_exception.dart'; /// Fetches surveillance nodes from the Overpass OSM API for the given bounds and profiles. +/// If the query fails due to too many nodes, automatically splits the area and retries. Future> fetchOverpassNodes({ required LatLngBounds bounds, required List profiles, UploadMode uploadMode = UploadMode.production, required int maxResults, +}) async { + return _fetchOverpassNodesWithSplitting( + bounds: bounds, + profiles: profiles, + uploadMode: uploadMode, + maxResults: maxResults, + splitDepth: 0, + ); +} + +/// Internal method that handles splitting when node limit is exceeded. +Future> _fetchOverpassNodesWithSplitting({ + required LatLngBounds bounds, + required List profiles, + UploadMode uploadMode = UploadMode.production, + required int maxResults, + required int splitDepth, }) async { if (profiles.isEmpty) return []; + const int maxSplitDepth = kMaxPreFetchSplitDepth; // Maximum times we'll split (4^3 = 64 max sub-areas) + + try { + return await _fetchSingleOverpassQuery( + bounds: bounds, + profiles: profiles, + maxResults: maxResults, + ); + } on OverpassNodeLimitException { + // If we've hit max split depth, give up to avoid infinite recursion + if (splitDepth >= maxSplitDepth) { + debugPrint('[fetchOverpassNodes] Max split depth reached, giving up on area: $bounds'); + return []; + } + + // Split the bounds into 4 quadrants and try each separately + debugPrint('[fetchOverpassNodes] Splitting area into quadrants (depth: $splitDepth)'); + final quadrants = _splitBounds(bounds); + final List allNodes = []; + + for (final quadrant in quadrants) { + final nodes = await _fetchOverpassNodesWithSplitting( + bounds: quadrant, + profiles: profiles, + uploadMode: uploadMode, + maxResults: 0, // No limit on individual quadrants to avoid double-limiting + splitDepth: splitDepth + 1, + ); + allNodes.addAll(nodes); + } + + debugPrint('[fetchOverpassNodes] Collected ${allNodes.length} nodes from ${quadrants.length} quadrants'); + return allNodes; + } +} + +/// Perform a single Overpass query without splitting logic. +Future> _fetchSingleOverpassQuery({ + required LatLngBounds bounds, + required List profiles, + required int maxResults, +}) async { const String overpassEndpoint = 'https://overpass-api.de/api/interpreter'; // Build the Overpass query @@ -34,7 +96,19 @@ Future> fetchOverpassNodes({ ); if (response.statusCode != 200) { - debugPrint('[fetchOverpassNodes] Overpass API error: ${response.body}'); + final errorBody = response.body; + debugPrint('[fetchOverpassNodes] Overpass API error: $errorBody'); + + // Check if it's a node limit exceeded error + if (errorBody.contains('too many') || + errorBody.contains('50000') || + errorBody.contains('50,000') || + errorBody.contains('limit') || + errorBody.contains('runtime error')) { + debugPrint('[fetchOverpassNodes] Detected node limit error, will attempt splitting'); + throw OverpassNodeLimitException('Query exceeded node limit', serverResponse: errorBody); + } + NetworkStatus.instance.reportOverpassIssue(); return []; } @@ -62,6 +136,9 @@ Future> fetchOverpassNodes({ return nodes; } catch (e) { + // Re-throw OverpassNodeLimitException so splitting logic can catch it + if (e is OverpassNodeLimitException) rethrow; + debugPrint('[fetchOverpassNodes] Exception: $e'); // Report network issues for connection errors @@ -100,6 +177,35 @@ $outputClause '''; } +/// Split a LatLngBounds into 4 quadrants (NW, NE, SW, SE). +List _splitBounds(LatLngBounds bounds) { + final centerLat = (bounds.north + bounds.south) / 2; + final centerLng = (bounds.east + bounds.west) / 2; + + return [ + // Southwest quadrant (bottom-left) + LatLngBounds( + LatLng(bounds.south, bounds.west), + LatLng(centerLat, centerLng), + ), + // Southeast quadrant (bottom-right) + LatLngBounds( + LatLng(bounds.south, centerLng), + LatLng(centerLat, bounds.east), + ), + // Northwest quadrant (top-left) + LatLngBounds( + LatLng(centerLat, bounds.west), + LatLng(bounds.north, centerLng), + ), + // Northeast quadrant (top-right) + LatLngBounds( + LatLng(centerLat, centerLng), + LatLng(bounds.north, bounds.east), + ), + ]; +} + /// Clean up pending uploads that now appear in Overpass results void _cleanupCompletedUploads(List overpassNodes) { try { diff --git a/lib/services/overpass_node_limit_exception.dart b/lib/services/overpass_node_limit_exception.dart new file mode 100644 index 0000000..77e24db --- /dev/null +++ b/lib/services/overpass_node_limit_exception.dart @@ -0,0 +1,11 @@ +/// Exception thrown when Overpass API returns an error indicating too many nodes were requested. +/// This typically happens when querying large areas that would return more than 50k nodes. +class OverpassNodeLimitException implements Exception { + final String message; + final String? serverResponse; + + OverpassNodeLimitException(this.message, {this.serverResponse}); + + @override + String toString() => 'OverpassNodeLimitException: $message'; +} \ No newline at end of file diff --git a/lib/services/prefetch_area_service.dart b/lib/services/prefetch_area_service.dart new file mode 100644 index 0000000..f8d8ff6 --- /dev/null +++ b/lib/services/prefetch_area_service.dart @@ -0,0 +1,165 @@ +import 'dart:async'; +import 'package:flutter/foundation.dart'; +import 'package:latlong2/latlong.dart'; +import 'package:flutter_map/flutter_map.dart'; + +import '../models/node_profile.dart'; +import '../models/osm_node.dart'; +import '../app_state.dart'; +import '../dev_config.dart'; +import 'map_data_submodules/nodes_from_overpass.dart'; +import 'node_cache.dart'; + +/// Manages pre-fetching larger areas to reduce Overpass API calls. +/// Uses zoom level 10 areas and automatically splits if hitting node limits. +class PrefetchAreaService { + static final PrefetchAreaService _instance = PrefetchAreaService._(); + factory PrefetchAreaService() => _instance; + PrefetchAreaService._(); + + // Current pre-fetched area and associated data + LatLngBounds? _preFetchedArea; + List? _preFetchedProfiles; + UploadMode? _preFetchedUploadMode; + bool _preFetchInProgress = false; + + // Debounce timer to avoid rapid requests while user is panning + Timer? _debounceTimer; + + // Configuration from dev_config + static const double _areaExpansionMultiplier = kPreFetchAreaExpansionMultiplier; + static const int _preFetchZoomLevel = kPreFetchZoomLevel; + + /// Check if the given bounds are fully within the current pre-fetched area. + bool isWithinPreFetchedArea(LatLngBounds bounds, List profiles, UploadMode uploadMode) { + if (_preFetchedArea == null || _preFetchedProfiles == null || _preFetchedUploadMode == null) { + return false; + } + + // Check if profiles and upload mode match + if (_preFetchedUploadMode != uploadMode) { + return false; + } + + if (!_profileListsEqual(_preFetchedProfiles!, profiles)) { + return false; + } + + // Check if bounds are fully contained within pre-fetched area + return bounds.north <= _preFetchedArea!.north && + bounds.south >= _preFetchedArea!.south && + bounds.east <= _preFetchedArea!.east && + bounds.west >= _preFetchedArea!.west; + } + + /// Request pre-fetch for the given view bounds if not already covered. + /// Uses debouncing to avoid rapid requests while user is panning. + void requestPreFetchIfNeeded({ + required LatLngBounds viewBounds, + required List profiles, + required UploadMode uploadMode, + }) { + // Skip if already in progress + if (_preFetchInProgress) { + debugPrint('[PrefetchAreaService] Pre-fetch already in progress, skipping'); + return; + } + + // Skip if current view is within pre-fetched area + if (isWithinPreFetchedArea(viewBounds, profiles, uploadMode)) { + debugPrint('[PrefetchAreaService] Current view within pre-fetched area, no fetch needed'); + return; + } + + // Cancel any pending debounced request + _debounceTimer?.cancel(); + + // Debounce to avoid rapid requests while user is still moving + _debounceTimer = Timer(const Duration(milliseconds: 800), () { + _startPreFetch( + viewBounds: viewBounds, + profiles: profiles, + uploadMode: uploadMode, + ); + }); + } + + /// Start the actual pre-fetch operation. + Future _startPreFetch({ + required LatLngBounds viewBounds, + required List profiles, + required UploadMode uploadMode, + }) async { + if (_preFetchInProgress) return; + + _preFetchInProgress = true; + + try { + // Calculate expanded area for pre-fetching + final preFetchArea = _expandBounds(viewBounds, _areaExpansionMultiplier); + + debugPrint('[PrefetchAreaService] Starting pre-fetch for area: ${preFetchArea.south},${preFetchArea.west} to ${preFetchArea.north},${preFetchArea.east}'); + + // Fetch nodes for the expanded area (no maxResults limit for pre-fetch) + final nodes = await fetchOverpassNodes( + bounds: preFetchArea, + profiles: profiles, + uploadMode: uploadMode, + maxResults: 0, // Unlimited - let Overpass splitting handle large areas + ); + + debugPrint('[PrefetchAreaService] Pre-fetch completed: ${nodes.length} nodes retrieved'); + + // Update cache with new nodes + if (nodes.isNotEmpty) { + NodeCache.instance.addOrUpdate(nodes); + } + + // Store the pre-fetched area info + _preFetchedArea = preFetchArea; + _preFetchedProfiles = List.from(profiles); + _preFetchedUploadMode = uploadMode; + + } catch (e) { + debugPrint('[PrefetchAreaService] Pre-fetch failed: $e'); + // Don't update pre-fetched area info on failure + } finally { + _preFetchInProgress = false; + } + } + + /// Expand bounds by the given multiplier, maintaining center point. + LatLngBounds _expandBounds(LatLngBounds bounds, double multiplier) { + final centerLat = (bounds.north + bounds.south) / 2; + final centerLng = (bounds.east + bounds.west) / 2; + + final latSpan = (bounds.north - bounds.south) * multiplier / 2; + final lngSpan = (bounds.east - bounds.west) * multiplier / 2; + + return LatLngBounds( + LatLng(centerLat - latSpan, centerLng - lngSpan), // Southwest + LatLng(centerLat + latSpan, centerLng + lngSpan), // Northeast + ); + } + + /// Check if two profile lists are equal by comparing IDs. + bool _profileListsEqual(List list1, List list2) { + if (list1.length != list2.length) return false; + final ids1 = list1.map((p) => p.id).toSet(); + final ids2 = list2.map((p) => p.id).toSet(); + return ids1.length == ids2.length && ids1.containsAll(ids2); + } + + /// Clear the pre-fetched area (e.g., when profiles change significantly). + void clearPreFetchedArea() { + _preFetchedArea = null; + _preFetchedProfiles = null; + _preFetchedUploadMode = null; + debugPrint('[PrefetchAreaService] Pre-fetched area cleared'); + } + + /// Dispose of resources. + void dispose() { + _debounceTimer?.cancel(); + } +} \ No newline at end of file diff --git a/lib/widgets/map/camera_refresh_controller.dart b/lib/widgets/map/camera_refresh_controller.dart index 3df21dd..381eefa 100644 --- a/lib/widgets/map/camera_refresh_controller.dart +++ b/lib/widgets/map/camera_refresh_controller.dart @@ -5,6 +5,7 @@ import 'package:latlong2/latlong.dart'; import '../../models/node_profile.dart'; import '../../app_state.dart' show UploadMode; +import '../../services/prefetch_area_service.dart'; import '../camera_provider_with_cache.dart'; import '../../dev_config.dart'; @@ -43,6 +44,8 @@ class CameraRefreshController { WidgetsBinding.instance.addPostFrameCallback((_) { // Clear camera cache to ensure fresh data for new profile combination _cameraProvider.clearCache(); + // Clear pre-fetch area since profiles changed + PrefetchAreaService().clearPreFetchedArea(); // Force display refresh first (for immediate UI update) _cameraProvider.refreshDisplay(); // Notify that profiles changed (triggers camera refresh) diff --git a/lib/widgets/map_view.dart b/lib/widgets/map_view.dart index 3aad009..9749a3e 100644 --- a/lib/widgets/map_view.dart +++ b/lib/widgets/map_view.dart @@ -7,6 +7,7 @@ import 'package:provider/provider.dart'; import '../app_state.dart'; import '../services/offline_area_service.dart'; import '../services/network_status.dart'; +import '../services/prefetch_area_service.dart'; import '../models/osm_node.dart'; import '../models/node_profile.dart'; import '../models/suspected_location.dart'; @@ -194,6 +195,7 @@ class MapViewState extends State { _cameraController.dispose(); _tileManager.dispose(); _gpsController.dispose(); + PrefetchAreaService().dispose(); super.dispose(); }