Pre-fetch larger areas to reduce calls to overpass; split large requests up

lib/dev_config.dart

@@ -64,6 +64,11 @@ const int kOsmApiMinZoomLevel = 13; // Minimum zoom for OSM API bbox queries (sa
 const Duration kMarkerTapTimeout = Duration(milliseconds: 250);
 const Duration kDebounceCameraRefresh = Duration(milliseconds: 500);
 
+// Pre-fetch area configuration
+const double kPreFetchAreaExpansionMultiplier = 3.0; // Expand visible bounds by this factor for pre-fetching
+const int kPreFetchZoomLevel = 10; // Always pre-fetch at this zoom level for consistent area sizes
+const int kMaxPreFetchSplitDepth = 3; // Maximum recursive splits when hitting Overpass node limit
+
 // Follow-me mode smooth transitions
 const Duration kFollowMeAnimationDuration = Duration(milliseconds: 600);
 const double kMinSpeedForRotationMps = 1.0; // Minimum speed (m/s) to apply rotation

lib/services/map_data_provider.dart

@@ -11,6 +11,7 @@ import 'map_data_submodules/tiles_from_remote.dart';
 import 'map_data_submodules/nodes_from_local.dart';
 import 'map_data_submodules/tiles_from_local.dart';
 import 'network_status.dart';
+import 'prefetch_area_service.dart';
 
 enum MapSource { local, remote, auto } // For future use
 
@@ -89,7 +90,9 @@ class MapDataProvider {
         maxResults: AppState.instance.maxCameras,
       );
     } else {
-      // Production mode: fetch both remote and local, then merge with deduplication
+      // Production mode: use pre-fetch service for efficient area loading
+      final preFetchService = PrefetchAreaService();
+      
       final List<Future<List<OsmNode>>> futures = [];
       
       // Always try to get local nodes (fast, cached)
@@ -99,38 +102,53 @@ class MapDataProvider {
         maxNodes: AppState.instance.maxCameras,
       ));
       
-      // Always try to get remote nodes (slower, fresh data)
-      futures.add(_fetchRemoteNodes(
-        bounds: bounds,
-        profiles: profiles,
-        uploadMode: uploadMode,
-        maxResults: AppState.instance.maxCameras,
-      ).catchError((e) {
-        debugPrint('[MapDataProvider] Remote node fetch failed, error: $e. Continuing with local only.');
-        return <OsmNode>[]; // Return empty list on remote failure
-      }));
-      
-      // Wait for both, then merge with deduplication by node ID
-      final results = await Future.wait(futures);
-      final localNodes = results[0];
-      final remoteNodes = results[1];
-      
-      // Merge with deduplication - prefer remote data over local for same node ID
-      final Map<int, OsmNode> mergedNodes = {};
-      
-      // Add local nodes first
-      for (final node in localNodes) {
-        mergedNodes[node.id] = node;
+      // Check if we need to fetch remote data or if pre-fetch covers this area
+      if (preFetchService.isWithinPreFetchedArea(bounds, profiles, uploadMode)) {
+        // Current view is within pre-fetched area, just use local cache
+        debugPrint('[MapDataProvider] Using pre-fetched data from cache');
+        final localNodes = await futures[0];
+        return localNodes.take(AppState.instance.maxCameras).toList();
+      } else {
+        // Not within pre-fetched area, request pre-fetch and also get immediate data
+        preFetchService.requestPreFetchIfNeeded(
+          viewBounds: bounds,
+          profiles: profiles,
+          uploadMode: uploadMode,
+        );
+        
+        // For immediate response, still try to get some remote data for current view
+        futures.add(_fetchRemoteNodes(
+          bounds: bounds,
+          profiles: profiles,
+          uploadMode: uploadMode,
+          maxResults: AppState.instance.maxCameras,
+        ).catchError((e) {
+          debugPrint('[MapDataProvider] Remote node fetch failed, error: $e. Continuing with local only.');
+          return <OsmNode>[]; // Return empty list on remote failure
+        }));
+        
+        // Wait for both, then merge with deduplication by node ID
+        final results = await Future.wait(futures);
+        final localNodes = results[0];
+        final remoteNodes = results[1];
+        
+        // Merge with deduplication - prefer remote data over local for same node ID
+        final Map<int, OsmNode> mergedNodes = {};
+        
+        // Add local nodes first
+        for (final node in localNodes) {
+          mergedNodes[node.id] = node;
+        }
+        
+        // Add remote nodes, overwriting any local duplicates
+        for (final node in remoteNodes) {
+          mergedNodes[node.id] = node;
+        }
+        
+        // Apply maxCameras limit to the merged result
+        final finalNodes = mergedNodes.values.take(AppState.instance.maxCameras).toList();
+        return finalNodes;
       }
-      
-      // Add remote nodes, overwriting any local duplicates
-      for (final node in remoteNodes) {
-        mergedNodes[node.id] = node;
-      }
-      
-      // Apply maxCameras limit to the merged result
-      final finalNodes = mergedNodes.values.take(AppState.instance.maxCameras).toList();
-      return finalNodes;
     }
     } finally {
       // Always report node completion, regardless of success or failure

lib/services/map_data_submodules/nodes_from_overpass.dart

@@ -8,17 +8,79 @@ import '../../models/node_profile.dart';
 import '../../models/osm_node.dart';
 import '../../models/pending_upload.dart';
 import '../../app_state.dart';
+import '../../dev_config.dart';
 import '../network_status.dart';
+import '../overpass_node_limit_exception.dart';
 
 /// Fetches surveillance nodes from the Overpass OSM API for the given bounds and profiles.
+/// If the query fails due to too many nodes, automatically splits the area and retries.
 Future<List<OsmNode>> fetchOverpassNodes({
   required LatLngBounds bounds,
   required List<NodeProfile> profiles,
   UploadMode uploadMode = UploadMode.production,
   required int maxResults,
+}) async {
+  return _fetchOverpassNodesWithSplitting(
+    bounds: bounds,
+    profiles: profiles,
+    uploadMode: uploadMode,
+    maxResults: maxResults,
+    splitDepth: 0,
+  );
+}
+
+/// Internal method that handles splitting when node limit is exceeded.
+Future<List<OsmNode>> _fetchOverpassNodesWithSplitting({
+  required LatLngBounds bounds,
+  required List<NodeProfile> profiles,
+  UploadMode uploadMode = UploadMode.production,
+  required int maxResults,
+  required int splitDepth,
 }) async {
   if (profiles.isEmpty) return [];
   
+  const int maxSplitDepth = kMaxPreFetchSplitDepth; // Maximum times we'll split (4^3 = 64 max sub-areas)
+  
+  try {
+    return await _fetchSingleOverpassQuery(
+      bounds: bounds,
+      profiles: profiles,
+      maxResults: maxResults,
+    );
+  } on OverpassNodeLimitException {
+    // If we've hit max split depth, give up to avoid infinite recursion
+    if (splitDepth >= maxSplitDepth) {
+      debugPrint('[fetchOverpassNodes] Max split depth reached, giving up on area: $bounds');
+      return [];
+    }
+    
+    // Split the bounds into 4 quadrants and try each separately
+    debugPrint('[fetchOverpassNodes] Splitting area into quadrants (depth: $splitDepth)');
+    final quadrants = _splitBounds(bounds);
+    final List<OsmNode> allNodes = [];
+    
+    for (final quadrant in quadrants) {
+      final nodes = await _fetchOverpassNodesWithSplitting(
+        bounds: quadrant,
+        profiles: profiles,
+        uploadMode: uploadMode,
+        maxResults: 0, // No limit on individual quadrants to avoid double-limiting
+        splitDepth: splitDepth + 1,
+      );
+      allNodes.addAll(nodes);
+    }
+    
+    debugPrint('[fetchOverpassNodes] Collected ${allNodes.length} nodes from ${quadrants.length} quadrants');
+    return allNodes;
+  }
+}
+
+/// Perform a single Overpass query without splitting logic.
+Future<List<OsmNode>> _fetchSingleOverpassQuery({
+  required LatLngBounds bounds,
+  required List<NodeProfile> profiles,
+  required int maxResults,
+}) async {
   const String overpassEndpoint = 'https://overpass-api.de/api/interpreter';
   
   // Build the Overpass query
@@ -34,7 +96,19 @@ Future<List<OsmNode>> fetchOverpassNodes({
     );
     
     if (response.statusCode != 200) {
-      debugPrint('[fetchOverpassNodes] Overpass API error: ${response.body}');
+      final errorBody = response.body;
+      debugPrint('[fetchOverpassNodes] Overpass API error: $errorBody');
+      
+      // Check if it's a node limit exceeded error
+      if (errorBody.contains('too many') || 
+          errorBody.contains('50000') || 
+          errorBody.contains('50,000') ||
+          errorBody.contains('limit') ||
+          errorBody.contains('runtime error')) {
+        debugPrint('[fetchOverpassNodes] Detected node limit error, will attempt splitting');
+        throw OverpassNodeLimitException('Query exceeded node limit', serverResponse: errorBody);
+      }
+      
       NetworkStatus.instance.reportOverpassIssue();
       return [];
     }
@@ -62,6 +136,9 @@ Future<List<OsmNode>> fetchOverpassNodes({
     return nodes;
     
   } catch (e) {
+    // Re-throw OverpassNodeLimitException so splitting logic can catch it
+    if (e is OverpassNodeLimitException) rethrow;
+    
     debugPrint('[fetchOverpassNodes] Exception: $e');
     
     // Report network issues for connection errors
@@ -100,6 +177,35 @@ $outputClause
 ''';
 }
 
+/// Split a LatLngBounds into 4 quadrants (NW, NE, SW, SE).
+List<LatLngBounds> _splitBounds(LatLngBounds bounds) {
+  final centerLat = (bounds.north + bounds.south) / 2;
+  final centerLng = (bounds.east + bounds.west) / 2;
+  
+  return [
+    // Southwest quadrant (bottom-left)
+    LatLngBounds(
+      LatLng(bounds.south, bounds.west),
+      LatLng(centerLat, centerLng),
+    ),
+    // Southeast quadrant (bottom-right)
+    LatLngBounds(
+      LatLng(bounds.south, centerLng),
+      LatLng(centerLat, bounds.east),
+    ),
+    // Northwest quadrant (top-left)
+    LatLngBounds(
+      LatLng(centerLat, bounds.west),
+      LatLng(bounds.north, centerLng),
+    ),
+    // Northeast quadrant (top-right)
+    LatLngBounds(
+      LatLng(centerLat, centerLng),
+      LatLng(bounds.north, bounds.east),
+    ),
+  ];
+}
+
 /// Clean up pending uploads that now appear in Overpass results
 void _cleanupCompletedUploads(List<OsmNode> overpassNodes) {
   try {

lib/services/overpass_node_limit_exception.dart

@@ -0,0 +1,11 @@
+/// Exception thrown when Overpass API returns an error indicating too many nodes were requested.
+/// This typically happens when querying large areas that would return more than 50k nodes.
+class OverpassNodeLimitException implements Exception {
+  final String message;
+  final String? serverResponse;
+
+  OverpassNodeLimitException(this.message, {this.serverResponse});
+
+  @override
+  String toString() => 'OverpassNodeLimitException: $message';
+}

lib/services/prefetch_area_service.dart

@@ -0,0 +1,165 @@
+import 'dart:async';
+import 'package:flutter/foundation.dart';
+import 'package:latlong2/latlong.dart';
+import 'package:flutter_map/flutter_map.dart';
+
+import '../models/node_profile.dart';
+import '../models/osm_node.dart';
+import '../app_state.dart';
+import '../dev_config.dart';
+import 'map_data_submodules/nodes_from_overpass.dart';
+import 'node_cache.dart';
+
+/// Manages pre-fetching larger areas to reduce Overpass API calls.
+/// Uses zoom level 10 areas and automatically splits if hitting node limits.
+class PrefetchAreaService {
+  static final PrefetchAreaService _instance = PrefetchAreaService._();
+  factory PrefetchAreaService() => _instance;
+  PrefetchAreaService._();
+
+  // Current pre-fetched area and associated data
+  LatLngBounds? _preFetchedArea;
+  List<NodeProfile>? _preFetchedProfiles;
+  UploadMode? _preFetchedUploadMode;
+  bool _preFetchInProgress = false;
+  
+  // Debounce timer to avoid rapid requests while user is panning
+  Timer? _debounceTimer;
+  
+  // Configuration from dev_config
+  static const double _areaExpansionMultiplier = kPreFetchAreaExpansionMultiplier;
+  static const int _preFetchZoomLevel = kPreFetchZoomLevel;
+  
+  /// Check if the given bounds are fully within the current pre-fetched area.
+  bool isWithinPreFetchedArea(LatLngBounds bounds, List<NodeProfile> profiles, UploadMode uploadMode) {
+    if (_preFetchedArea == null || _preFetchedProfiles == null || _preFetchedUploadMode == null) {
+      return false;
+    }
+    
+    // Check if profiles and upload mode match
+    if (_preFetchedUploadMode != uploadMode) {
+      return false;
+    }
+    
+    if (!_profileListsEqual(_preFetchedProfiles!, profiles)) {
+      return false;
+    }
+    
+    // Check if bounds are fully contained within pre-fetched area
+    return bounds.north <= _preFetchedArea!.north &&
+           bounds.south >= _preFetchedArea!.south &&
+           bounds.east <= _preFetchedArea!.east &&
+           bounds.west >= _preFetchedArea!.west;
+  }
+  
+  /// Request pre-fetch for the given view bounds if not already covered.
+  /// Uses debouncing to avoid rapid requests while user is panning.
+  void requestPreFetchIfNeeded({
+    required LatLngBounds viewBounds,
+    required List<NodeProfile> profiles,
+    required UploadMode uploadMode,
+  }) {
+    // Skip if already in progress
+    if (_preFetchInProgress) {
+      debugPrint('[PrefetchAreaService] Pre-fetch already in progress, skipping');
+      return;
+    }
+    
+    // Skip if current view is within pre-fetched area
+    if (isWithinPreFetchedArea(viewBounds, profiles, uploadMode)) {
+      debugPrint('[PrefetchAreaService] Current view within pre-fetched area, no fetch needed');
+      return;
+    }
+    
+    // Cancel any pending debounced request
+    _debounceTimer?.cancel();
+    
+    // Debounce to avoid rapid requests while user is still moving
+    _debounceTimer = Timer(const Duration(milliseconds: 800), () {
+      _startPreFetch(
+        viewBounds: viewBounds,
+        profiles: profiles,
+        uploadMode: uploadMode,
+      );
+    });
+  }
+  
+  /// Start the actual pre-fetch operation.
+  Future<void> _startPreFetch({
+    required LatLngBounds viewBounds,
+    required List<NodeProfile> profiles,
+    required UploadMode uploadMode,
+  }) async {
+    if (_preFetchInProgress) return;
+    
+    _preFetchInProgress = true;
+    
+    try {
+      // Calculate expanded area for pre-fetching
+      final preFetchArea = _expandBounds(viewBounds, _areaExpansionMultiplier);
+      
+      debugPrint('[PrefetchAreaService] Starting pre-fetch for area: ${preFetchArea.south},${preFetchArea.west} to ${preFetchArea.north},${preFetchArea.east}');
+      
+      // Fetch nodes for the expanded area (no maxResults limit for pre-fetch)
+      final nodes = await fetchOverpassNodes(
+        bounds: preFetchArea,
+        profiles: profiles,
+        uploadMode: uploadMode,
+        maxResults: 0, // Unlimited - let Overpass splitting handle large areas
+      );
+      
+      debugPrint('[PrefetchAreaService] Pre-fetch completed: ${nodes.length} nodes retrieved');
+      
+      // Update cache with new nodes
+      if (nodes.isNotEmpty) {
+        NodeCache.instance.addOrUpdate(nodes);
+      }
+      
+      // Store the pre-fetched area info
+      _preFetchedArea = preFetchArea;
+      _preFetchedProfiles = List.from(profiles);
+      _preFetchedUploadMode = uploadMode;
+      
+    } catch (e) {
+      debugPrint('[PrefetchAreaService] Pre-fetch failed: $e');
+      // Don't update pre-fetched area info on failure
+    } finally {
+      _preFetchInProgress = false;
+    }
+  }
+  
+  /// Expand bounds by the given multiplier, maintaining center point.
+  LatLngBounds _expandBounds(LatLngBounds bounds, double multiplier) {
+    final centerLat = (bounds.north + bounds.south) / 2;
+    final centerLng = (bounds.east + bounds.west) / 2;
+    
+    final latSpan = (bounds.north - bounds.south) * multiplier / 2;
+    final lngSpan = (bounds.east - bounds.west) * multiplier / 2;
+    
+    return LatLngBounds(
+      LatLng(centerLat - latSpan, centerLng - lngSpan), // Southwest
+      LatLng(centerLat + latSpan, centerLng + lngSpan), // Northeast
+    );
+  }
+  
+  /// Check if two profile lists are equal by comparing IDs.
+  bool _profileListsEqual(List<NodeProfile> list1, List<NodeProfile> list2) {
+    if (list1.length != list2.length) return false;
+    final ids1 = list1.map((p) => p.id).toSet();
+    final ids2 = list2.map((p) => p.id).toSet();
+    return ids1.length == ids2.length && ids1.containsAll(ids2);
+  }
+  
+  /// Clear the pre-fetched area (e.g., when profiles change significantly).
+  void clearPreFetchedArea() {
+    _preFetchedArea = null;
+    _preFetchedProfiles = null;
+    _preFetchedUploadMode = null;
+    debugPrint('[PrefetchAreaService] Pre-fetched area cleared');
+  }
+  
+  /// Dispose of resources.
+  void dispose() {
+    _debounceTimer?.cancel();
+  }
+}

lib/widgets/map/camera_refresh_controller.dart

@@ -5,6 +5,7 @@ import 'package:latlong2/latlong.dart';
 
 import '../../models/node_profile.dart';
 import '../../app_state.dart' show UploadMode;
+import '../../services/prefetch_area_service.dart';
 import '../camera_provider_with_cache.dart';
 import '../../dev_config.dart';
 
@@ -43,6 +44,8 @@ class CameraRefreshController {
       WidgetsBinding.instance.addPostFrameCallback((_) {
         // Clear camera cache to ensure fresh data for new profile combination
         _cameraProvider.clearCache();
+        // Clear pre-fetch area since profiles changed
+        PrefetchAreaService().clearPreFetchedArea();
         // Force display refresh first (for immediate UI update)
         _cameraProvider.refreshDisplay();
         // Notify that profiles changed (triggers camera refresh)

lib/widgets/map_view.dart

@@ -7,6 +7,7 @@ import 'package:provider/provider.dart';
 import '../app_state.dart';
 import '../services/offline_area_service.dart';
 import '../services/network_status.dart';
+import '../services/prefetch_area_service.dart';
 import '../models/osm_node.dart';
 import '../models/node_profile.dart';
 import '../models/suspected_location.dart';
@@ -194,6 +195,7 @@ class MapViewState extends State<MapView> {
     _cameraController.dispose();
     _tileManager.dispose();
     _gpsController.dispose();
+    PrefetchAreaService().dispose();
     super.dispose();
   }