diff --git a/lib/dev_config.dart b/lib/dev_config.dart new file mode 100644 index 0000000..e69de29 diff --git a/lib/services/map_data_provider.dart b/lib/services/map_data_provider.dart index 10eba77..0ab5028 100644 --- a/lib/services/map_data_provider.dart +++ b/lib/services/map_data_provider.dart @@ -44,14 +44,15 @@ class MapDataProvider { // Explicit remote request: error if offline, else always remote if (source == MapSource.remote) { if (offline) { - print('[MapDataProvider] BLOCKED by offlineMode for remote camera fetch'); + print('[MapDataProvider] Overpass request BLOCKED because we are in offlineMode'); throw OfflineModeException("Cannot fetch remote cameras in offline mode."); } return camerasFromOverpass( bounds: bounds, profiles: profiles, uploadMode: uploadMode, - maxCameras: AppState.instance.maxCameras, + pageSize: AppState.instance.maxCameras, + fetchAllPages: false, ); } @@ -76,17 +77,42 @@ class MapDataProvider { bounds: bounds, profiles: profiles, uploadMode: uploadMode, - maxCameras: AppState.instance.maxCameras, + pageSize: AppState.instance.maxCameras, ); } catch (e) { print('[MapDataProvider] Remote camera fetch failed, error: $e. Falling back to local.'); return fetchLocalCameras( bounds: bounds, profiles: profiles, + maxCameras: AppState.instance.maxCameras, ); } } } + + /// Bulk/paged camera fetch for offline downloads (handling paging, dedup, and Overpass retries) + /// Only use for offline area download, not for map browsing! Ignores maxCameras config. + Future> getAllCamerasForDownload({ + required LatLngBounds bounds, + required List profiles, + UploadMode uploadMode = UploadMode.production, + int pageSize = 500, + int maxTries = 3, + }) async { + final offline = AppState.instance.offlineMode; + if (offline) { + throw OfflineModeException("Cannot fetch remote cameras for offline area download in offline mode."); + } + return camerasFromOverpass( + bounds: bounds, + profiles: profiles, + uploadMode: uploadMode, + fetchAllPages: true, + pageSize: pageSize, + maxTries: maxTries, + ); + } + /// Fetch tile image bytes. Default is to try local first, then remote if not offline. Honors explicit source. Future> getTile({ required int z, diff --git a/lib/services/map_data_submodules/cameras_from_local.dart b/lib/services/map_data_submodules/cameras_from_local.dart index b4a4434..86f4d54 100644 --- a/lib/services/map_data_submodules/cameras_from_local.dart +++ b/lib/services/map_data_submodules/cameras_from_local.dart @@ -11,10 +11,10 @@ import '../offline_areas/offline_area_models.dart'; Future> fetchLocalCameras({ required LatLngBounds bounds, required List profiles, + int? maxCameras, }) async { final areas = OfflineAreaService().offlineAreas; - final List result = []; - final seenIds = {}; + final Map deduped = {}; for (final area in areas) { if (area.status != OfflineAreaStatus.complete) continue; @@ -22,16 +22,18 @@ Future> fetchLocalCameras({ final nodes = await _loadAreaCameras(area); for (final cam in nodes) { - if (seenIds.contains(cam.id)) continue; - // Check geo bounds + // Deduplicate by camera ID, preferring the first occurrence + if (deduped.containsKey(cam.id)) continue; + // Within view bounds? if (!_pointInBounds(cam.coord, bounds)) continue; - // Check profiles + // Profile filter if used if (profiles.isNotEmpty && !_matchesAnyProfile(cam, profiles)) continue; - result.add(cam); - seenIds.add(cam.id); + deduped[cam.id] = cam; } } - return result; + + final out = deduped.values.take(maxCameras ?? deduped.length).toList(); + return out; } // Try in-memory first, else load from disk diff --git a/lib/services/map_data_submodules/cameras_from_overpass.dart b/lib/services/map_data_submodules/cameras_from_overpass.dart index d8339ea..d283398 100644 --- a/lib/services/map_data_submodules/cameras_from_overpass.dart +++ b/lib/services/map_data_submodules/cameras_from_overpass.dart @@ -8,13 +8,18 @@ import '../../models/osm_camera_node.dart'; import '../../app_state.dart'; /// Fetches cameras from the Overpass OSM API for the given bounds and profiles. +/// If fetchAllPages is true, returns all possible cameras using multiple API calls (paging with pageSize). +/// If false (the default), returns only the first page of up to pageSize results. Future> camerasFromOverpass({ required LatLngBounds bounds, required List profiles, UploadMode uploadMode = UploadMode.production, - int? maxCameras, + int pageSize = 500, // Used for both default limit and paging chunk + bool fetchAllPages = false, // True for offline area download, else just grabs first chunk + int maxTries = 3, }) async { if (profiles.isEmpty) return []; + const String prodEndpoint = 'https://overpass-api.de/api/interpreter'; final nodeClauses = profiles.map((profile) { final tagFilters = profile.tags.entries @@ -23,38 +28,71 @@ Future> camerasFromOverpass({ return '''node\n $tagFilters\n (${bounds.southWest.latitude},${bounds.southWest.longitude},\n ${bounds.northEast.latitude},${bounds.northEast.longitude});'''; }).join('\n '); - const String prodEndpoint = 'https://overpass-api.de/api/interpreter'; - - final limit = maxCameras ?? AppState.instance.maxCameras; - final query = ''' - [out:json][timeout:25]; - ( - $nodeClauses - ); - out body $limit; - '''; - - try { - print('[camerasFromOverpass] Querying Overpass...'); - print('[camerasFromOverpass] Query:\n$query'); - final resp = await http.post(Uri.parse(prodEndpoint), body: {'data': query.trim()}); - print('[camerasFromOverpass] Status: ${resp.statusCode}, Length: ${resp.body.length}'); - if (resp.statusCode != 200) { - print('[camerasFromOverpass] Overpass failed: ${resp.body}'); + // Helper for one Overpass chunk fetch + Future> fetchChunk() async { + final query = ''' + [out:json][timeout:25]; + ( + $nodeClauses + ); + out body $pageSize; + '''; + try { + print('[camerasFromOverpass] Querying Overpass...'); + print('[camerasFromOverpass] Query:\n$query'); + final resp = await http.post(Uri.parse(prodEndpoint), body: {'data': query.trim()}); + print('[camerasFromOverpass] Status: ${resp.statusCode}, Length: ${resp.body.length}'); + if (resp.statusCode != 200) { + print('[camerasFromOverpass] Overpass failed: ${resp.body}'); + return []; + } + final data = jsonDecode(resp.body) as Map; + final elements = data['elements'] as List; + print('[camerasFromOverpass] Retrieved elements: ${elements.length}'); + return elements.whereType>().map((e) { + return OsmCameraNode( + id: e['id'], + coord: LatLng(e['lat'], e['lon']), + tags: Map.from(e['tags'] ?? {}), + ); + }).toList(); + } catch (e) { + print('[camerasFromOverpass] Overpass exception: $e'); return []; } - final data = jsonDecode(resp.body) as Map; - final elements = data['elements'] as List; - print('[camerasFromOverpass] Retrieved elements: ${elements.length}'); - return elements.whereType>().map((e) { - return OsmCameraNode( - id: e['id'], - coord: LatLng(e['lat'], e['lon']), - tags: Map.from(e['tags'] ?? {}), - ); - }).toList(); - } catch (e) { - print('[camerasFromOverpass] Overpass exception: $e'); - return []; + } + + if (!fetchAllPages) { + // Just one page + return await fetchChunk(); + } else { + // Fetch all possible data, paging with deduplication and backoff + final seenIds = {}; + final allCameras = []; + int page = 0; + while (true) { + page++; + List pageCameras = []; + int tries = 0; + while (tries < maxTries) { + try { + final cams = await fetchChunk(); + pageCameras = cams.where((c) => !seenIds.contains(c.id)).toList(); + break; + } catch (e) { + tries++; + final delayMs = 400 * (1 << tries); + print('[camerasFromOverpass][paged] Error on page $page try $tries: $e. Retrying in ${delayMs}ms.'); + await Future.delayed(Duration(milliseconds: delayMs)); + } + } + if (pageCameras.isEmpty) break; + print('[camerasFromOverpass][paged] Page $page: got ${pageCameras.length} new cameras.'); + allCameras.addAll(pageCameras); + seenIds.addAll(pageCameras.map((c) => c.id)); + if (pageCameras.length < pageSize) break; + } + print('[camerasFromOverpass][paged] DONE. Found ${allCameras.length} cameras for download.'); + return allCameras; } } diff --git a/lib/services/map_data_submodules/tiles_from_local.dart b/lib/services/map_data_submodules/tiles_from_local.dart index 1d24dbe..16d78f9 100644 --- a/lib/services/map_data_submodules/tiles_from_local.dart +++ b/lib/services/map_data_submodules/tiles_from_local.dart @@ -15,7 +15,8 @@ Future> fetchLocalTile({required int z, required int x, required int y // Get tile coverage for area at this zoom only final coveredTiles = computeTileList(area.bounds, z, z); - if (coveredTiles.contains([z, x, y])) { + final hasTile = coveredTiles.any((tile) => tile[0] == z && tile[1] == x && tile[2] == y); + if (hasTile) { final tilePath = _tilePath(area.directory, z, x, y); final file = File(tilePath); if (await file.exists()) { diff --git a/lib/services/offline_area_service.dart b/lib/services/offline_area_service.dart index dc92c70..72bad86 100644 --- a/lib/services/offline_area_service.dart +++ b/lib/services/offline_area_service.dart @@ -111,7 +111,7 @@ class OfflineAreaService { } } if (filesFound != expectedTiles.length) { - debugPrint('World area: missing \\${expectedTiles.length - filesFound} tiles. First few: \\$missingTiles'); + debugPrint('World area: missing ${expectedTiles.length - filesFound} tiles. First few: $missingTiles'); } else { debugPrint('World area: all tiles accounted for.'); } @@ -212,7 +212,7 @@ class OfflineAreaService { while (pass < maxPasses && tilesToFetch.isNotEmpty) { pass++; int doneThisPass = 0; - debugPrint('DownloadArea: pass #$pass for area $id. Need \\${tilesToFetch.length} tiles.'); + debugPrint('DownloadArea: pass #$pass for area $id. Need ${tilesToFetch.length} tiles.'); for (final tile in tilesToFetch) { if (area.status == OfflineAreaStatus.cancelled) break; try { @@ -245,7 +245,7 @@ class OfflineAreaService { } if (!area.isPermanent) { - final cameras = await camerasFromOverpass( + final cameras = await MapDataProvider().getAllCamerasForDownload( bounds: bounds, profiles: AppState.instance.enabledProfiles, ); @@ -262,7 +262,7 @@ class OfflineAreaService { debugPrint('Area $id: all tiles accounted for and area marked complete.'); } else { area.status = OfflineAreaStatus.error; - debugPrint('Area $id: MISSING tiles after $maxPasses passes. First 10: \\${tilesToFetch.toList().take(10)}'); + debugPrint('Area $id: MISSING tiles after $maxPasses passes. First 10: ${tilesToFetch.toList().take(10)}'); if (!area.isPermanent) { final dirObj = Directory(area.directory); if (await dirObj.exists()) {