From 4a44ab96d61a036cdbf1abf0c21182c063f5ee6b Mon Sep 17 00:00:00 2001 From: stopflock Date: Mon, 6 Oct 2025 21:07:08 -0500 Subject: [PATCH] Sorta working suspected locations --- lib/app_state.dart | 1 - lib/models/suspected_location.dart | 40 +++- lib/services/suspected_location_cache.dart | 224 +++++++++++++++++++ lib/services/suspected_location_service.dart | 114 +++------- lib/state/suspected_location_state.dart | 31 ++- lib/widgets/map_view.dart | 8 + pubspec.yaml | 2 +- 7 files changed, 316 insertions(+), 104 deletions(-) create mode 100644 lib/services/suspected_location_cache.dart diff --git a/lib/app_state.dart b/lib/app_state.dart index 0e0a5de..8565f22 100644 --- a/lib/app_state.dart +++ b/lib/app_state.dart @@ -145,7 +145,6 @@ class AppState extends ChangeNotifier { List get pendingUploads => _uploadQueueState.pendingUploads; // Suspected location state - List get suspectedLocations => _suspectedLocationState.locations; SuspectedLocation? get selectedSuspectedLocation => _suspectedLocationState.selectedLocation; bool get suspectedLocationsEnabled => _suspectedLocationState.isEnabled; bool get suspectedLocationsLoading => _suspectedLocationState.isLoading; diff --git a/lib/models/suspected_location.dart b/lib/models/suspected_location.dart index 48b17d7..dc9a1d0 100644 --- a/lib/models/suspected_location.dart +++ b/lib/models/suspected_location.dart @@ -1,4 +1,5 @@ import 'dart:convert'; +import 'dart:math' as math; import 'package:latlong2/latlong.dart'; /// A suspected surveillance location from the CSV data @@ -41,16 +42,39 @@ class SuspectedLocation { // Parse GeoJSON if available if (locationString != null && locationString.isNotEmpty) { try { - print('[SuspectedLocation] Parsing GeoJSON for ticket ${row['ticket_no']}, length: ${locationString.length}'); + // Only log first few entries to avoid spam + final ticketNo = row['ticket_no']?.toString() ?? 'unknown'; + if (ticketNo.endsWith('0') || ticketNo.endsWith('1') || ticketNo.endsWith('2')) { + print('[SuspectedLocation] Raw location string for ticket $ticketNo: ${locationString.substring(0, math.min(100, locationString.length))}...'); + } + if (ticketNo.endsWith('0') || ticketNo.endsWith('1') || ticketNo.endsWith('2')) { + if (centroid.latitude != 0 || centroid.longitude != 0) { + print('[SuspectedLocation] Successfully parsed centroid: $centroid'); + } else { + print('[SuspectedLocation] Parsed but got zero coordinates'); + } + } geoJson = jsonDecode(locationString) as Map; final coordinates = _extractCoordinatesFromGeoJson(geoJson); centroid = coordinates.centroid; bounds = coordinates.bounds; - print('[SuspectedLocation] Successfully parsed, centroid: $centroid, bounds count: ${bounds.length}'); - } catch (e, stackTrace) { + if (ticketNo.endsWith('0') || ticketNo.endsWith('1') || ticketNo.endsWith('2')) { + if (centroid.latitude != 0 || centroid.longitude != 0) { + print('[SuspectedLocation] Successfully parsed centroid: $centroid'); + } else { + print('[SuspectedLocation] Parsed but got zero coordinates'); + } + } + if (ticketNo.endsWith('0') || ticketNo.endsWith('1') || ticketNo.endsWith('2')) { + if (centroid.latitude != 0 || centroid.longitude != 0) { + print('[SuspectedLocation] Successfully parsed centroid: $centroid'); + } else { + print('[SuspectedLocation] Parsed but got zero coordinates'); + } + } + } catch (e) { // If GeoJSON parsing fails, use default coordinates print('[SuspectedLocation] Failed to parse GeoJSON for ticket ${row['ticket_no']}: $e'); - print('[SuspectedLocation] Stack trace: $stackTrace'); print('[SuspectedLocation] Location string: $locationString'); } } @@ -74,17 +98,17 @@ class SuspectedLocation { /// Extract coordinates from GeoJSON static ({LatLng centroid, List bounds}) _extractCoordinatesFromGeoJson(Map geoJson) { try { - final geometry = geoJson['geometry'] as Map?; - final coordinates = geometry?['coordinates'] as List?; - + // The geoJson IS the geometry object (not wrapped in a 'geometry' property) + final coordinates = geoJson['coordinates'] as List?; if (coordinates == null || coordinates.isEmpty) { + print('[SuspectedLocation] No coordinates found in GeoJSON'); return (centroid: const LatLng(0, 0), bounds: []); } final List points = []; // Handle different geometry types - final type = geometry?['type'] as String?; + final type = geoJson['type'] as String?; switch (type) { case 'Point': if (coordinates.length >= 2) { diff --git a/lib/services/suspected_location_cache.dart b/lib/services/suspected_location_cache.dart new file mode 100644 index 0000000..ba28f2c --- /dev/null +++ b/lib/services/suspected_location_cache.dart @@ -0,0 +1,224 @@ +import 'dart:convert'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_map/flutter_map.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:latlong2/latlong.dart'; + +import '../models/suspected_location.dart'; +import 'suspected_location_service.dart'; + +/// Lightweight entry with pre-calculated centroid for efficient bounds checking +class SuspectedLocationEntry { + final Map rawData; + final LatLng centroid; + + SuspectedLocationEntry({required this.rawData, required this.centroid}); + + Map toJson() => { + 'rawData': rawData, + 'centroid': [centroid.latitude, centroid.longitude], + }; + + factory SuspectedLocationEntry.fromJson(Map json) { + final centroidList = json['centroid'] as List; + return SuspectedLocationEntry( + rawData: Map.from(json['rawData']), + centroid: LatLng( + (centroidList[0] as num).toDouble(), + (centroidList[1] as num).toDouble(), + ), + ); + } +} + +class SuspectedLocationCache extends ChangeNotifier { + static final SuspectedLocationCache _instance = SuspectedLocationCache._(); + factory SuspectedLocationCache() => _instance; + SuspectedLocationCache._(); + + static const String _prefsKeyProcessedData = 'suspected_locations_processed_data'; + static const String _prefsKeyLastFetch = 'suspected_locations_last_fetch'; + + List _processedEntries = []; + DateTime? _lastFetchTime; + final Map> _boundsCache = {}; + + /// Get suspected locations within specific bounds (cached) + List getLocationsForBounds(LatLngBounds bounds) { + if (!SuspectedLocationService().isEnabled) { + debugPrint('[SuspectedLocationCache] Service not enabled'); + return []; + } + + final boundsKey = '${bounds.north.toStringAsFixed(4)},${bounds.south.toStringAsFixed(4)},${bounds.east.toStringAsFixed(4)},${bounds.west.toStringAsFixed(4)}'; + + debugPrint('[SuspectedLocationCache] Getting locations for bounds: $boundsKey, processed entries count: ${_processedEntries.length}'); + + // Check cache first + if (_boundsCache.containsKey(boundsKey)) { + debugPrint('[SuspectedLocationCache] Using cached result: ${_boundsCache[boundsKey]!.length} locations'); + return _boundsCache[boundsKey]!; + } + + // Filter processed entries for this bounds (very fast since centroids are pre-calculated) + final locations = []; + int inBoundsCount = 0; + + for (final entry in _processedEntries) { + // Quick bounds check using pre-calculated centroid + final lat = entry.centroid.latitude; + final lng = entry.centroid.longitude; + + if (lat <= bounds.north && lat >= bounds.south && + lng <= bounds.east && lng >= bounds.west) { + try { + // Only create SuspectedLocation object if it's in bounds + final location = SuspectedLocation.fromCsvRow(entry.rawData); + locations.add(location); + inBoundsCount++; + } catch (e) { + // Skip invalid entries + continue; + } + } + } + + debugPrint('[SuspectedLocationCache] Checked ${_processedEntries.length} entries, $inBoundsCount in bounds, result: ${locations.length} locations'); + + // Cache the result + _boundsCache[boundsKey] = locations; + + // Limit cache size to prevent memory issues + if (_boundsCache.length > 100) { + final oldestKey = _boundsCache.keys.first; + _boundsCache.remove(oldestKey); + } + + return locations; + } + + /// Load processed data from storage + Future loadFromStorage() async { + try { + final prefs = await SharedPreferences.getInstance(); + + // Load last fetch time + final lastFetchMs = prefs.getInt(_prefsKeyLastFetch); + if (lastFetchMs != null) { + _lastFetchTime = DateTime.fromMillisecondsSinceEpoch(lastFetchMs); + } + + // Load processed data + final processedDataString = prefs.getString(_prefsKeyProcessedData); + if (processedDataString != null) { + final List processedDataList = jsonDecode(processedDataString); + _processedEntries = processedDataList + .map((json) => SuspectedLocationEntry.fromJson(json as Map)) + .toList(); + debugPrint('[SuspectedLocationCache] Loaded ${_processedEntries.length} processed entries from storage'); + } + } catch (e) { + debugPrint('[SuspectedLocationCache] Error loading from storage: $e'); + _processedEntries.clear(); + _lastFetchTime = null; + } + } + + /// Process raw CSV data and save to storage (calculates centroids once) + Future processAndSave(List> rawData, DateTime fetchTime) async { + try { + debugPrint('[SuspectedLocationCache] Processing ${rawData.length} raw entries...'); + + final processedEntries = []; + int validCount = 0; + int errorCount = 0; + int zeroCoordCount = 0; + + for (int i = 0; i < rawData.length; i++) { + final rowData = rawData[i]; + try { + // Create a temporary SuspectedLocation to extract the centroid + final tempLocation = SuspectedLocation.fromCsvRow(rowData); + + // Only save if we have a valid centroid (not at 0,0) + if (tempLocation.centroid.latitude != 0 || tempLocation.centroid.longitude != 0) { + processedEntries.add(SuspectedLocationEntry( + rawData: rowData, + centroid: tempLocation.centroid, + )); + validCount++; + } else { + zeroCoordCount++; + if (i < 3) { // Log first few zero coord cases + debugPrint('[SuspectedLocationCache] Row $i has zero coordinates: ticket=${rowData['ticket_no']}, location=${rowData['location']?.toString().length} chars'); + } + } + } catch (e) { + errorCount++; + if (errorCount <= 5) { // Log first few errors + debugPrint('[SuspectedLocationCache] Row $i error: $e, ticket=${rowData['ticket_no']}'); + } + continue; + } + } + + debugPrint('[SuspectedLocationCache] Processing complete - Valid: $validCount, Zero coords: $zeroCoordCount, Errors: $errorCount'); + + _processedEntries = processedEntries; + _lastFetchTime = fetchTime; + + // Clear bounds cache since data changed + _boundsCache.clear(); + + final prefs = await SharedPreferences.getInstance(); + + // Save processed data + final processedDataString = jsonEncode(processedEntries.map((e) => e.toJson()).toList()); + await prefs.setString(_prefsKeyProcessedData, processedDataString); + + // Save last fetch time + await prefs.setInt(_prefsKeyLastFetch, fetchTime.millisecondsSinceEpoch); + + // Log coordinate ranges for debugging + if (processedEntries.isNotEmpty) { + double minLat = processedEntries.first.centroid.latitude; + double maxLat = minLat; + double minLng = processedEntries.first.centroid.longitude; + double maxLng = minLng; + + for (final entry in processedEntries) { + final lat = entry.centroid.latitude; + final lng = entry.centroid.longitude; + if (lat < minLat) minLat = lat; + if (lat > maxLat) maxLat = lat; + if (lng < minLng) minLng = lng; + if (lng > maxLng) maxLng = lng; + } + + debugPrint('[SuspectedLocationCache] Coordinate ranges - Lat: $minLat to $maxLat, Lng: $minLng to $maxLng'); + } + + debugPrint('[SuspectedLocationCache] Processed and saved $validCount valid entries (${processedEntries.length} total)'); + notifyListeners(); + } catch (e) { + debugPrint('[SuspectedLocationCache] Error processing and saving: $e'); + } + } + + /// Clear all cached data + void clear() { + _processedEntries.clear(); + _boundsCache.clear(); + _lastFetchTime = null; + notifyListeners(); + } + + /// Get last fetch time + DateTime? get lastFetchTime => _lastFetchTime; + + /// Get total count of processed entries + int get totalCount => _processedEntries.length; + + /// Check if we have data + bool get hasData => _processedEntries.isNotEmpty; +} \ No newline at end of file diff --git a/lib/services/suspected_location_service.dart b/lib/services/suspected_location_service.dart index 003c3c2..75ecb74 100644 --- a/lib/services/suspected_location_service.dart +++ b/lib/services/suspected_location_service.dart @@ -1,11 +1,14 @@ import 'dart:convert'; import 'dart:io'; import 'package:flutter/foundation.dart'; +import 'package:flutter_map/flutter_map.dart'; import 'package:http/http.dart' as http; +import 'package:latlong2/latlong.dart'; import 'package:shared_preferences/shared_preferences.dart'; import 'package:csv/csv.dart'; import '../models/suspected_location.dart'; +import 'suspected_location_cache.dart'; class SuspectedLocationService { static final SuspectedLocationService _instance = SuspectedLocationService._(); @@ -13,22 +16,16 @@ class SuspectedLocationService { SuspectedLocationService._(); static const String _csvUrl = 'https://alprwatch.org/pub/flock_utilities_mini_2025-10-06.csv'; - static const String _prefsKeyData = 'suspected_locations_data'; - static const String _prefsKeyLastFetch = 'suspected_locations_last_fetch'; static const String _prefsKeyEnabled = 'suspected_locations_enabled'; static const Duration _maxAge = Duration(days: 7); static const Duration _timeout = Duration(seconds: 30); - List _locations = []; - DateTime? _lastFetchTime; + final SuspectedLocationCache _cache = SuspectedLocationCache(); bool _isEnabled = false; bool _isLoading = false; - /// Get all suspected locations - List get locations => List.unmodifiable(_locations); - /// Get last fetch time - DateTime? get lastFetchTime => _lastFetchTime; + DateTime? get lastFetchTime => _cache.lastFetchTime; /// Check if suspected locations are enabled bool get isEnabled => _isEnabled; @@ -40,6 +37,9 @@ class SuspectedLocationService { Future init() async { await _loadFromStorage(); + // Load cache data + await _cache.loadFromStorage(); + // Only auto-fetch if enabled and data is stale or missing if (_isEnabled && _shouldRefresh()) { await _fetchData(); @@ -53,13 +53,13 @@ class SuspectedLocationService { await prefs.setBool(_prefsKeyEnabled, enabled); // If enabling for the first time and no data, fetch it - if (enabled && _locations.isEmpty) { + if (enabled && !_cache.hasData) { await _fetchData(); } - // If disabling, clear the data from memory (but keep in storage) + // If disabling, clear the cache if (!enabled) { - _locations.clear(); + _cache.clear(); } } @@ -70,12 +70,12 @@ class SuspectedLocationService { /// Check if data should be refreshed bool _shouldRefresh() { - if (_locations.isEmpty) return true; - if (_lastFetchTime == null) return true; - return DateTime.now().difference(_lastFetchTime!) > _maxAge; + if (!_cache.hasData) return true; + if (_cache.lastFetchTime == null) return true; + return DateTime.now().difference(_cache.lastFetchTime!) > _maxAge; } - /// Load data from shared preferences + /// Load settings from shared preferences Future _loadFromStorage() async { try { final prefs = await SharedPreferences.getInstance(); @@ -83,50 +83,9 @@ class SuspectedLocationService { // Load enabled state _isEnabled = prefs.getBool(_prefsKeyEnabled) ?? false; - // Load last fetch time - final lastFetchMs = prefs.getInt(_prefsKeyLastFetch); - if (lastFetchMs != null) { - _lastFetchTime = DateTime.fromMillisecondsSinceEpoch(lastFetchMs); - } - - // Only load data if enabled - if (!_isEnabled) { - return; - } - - // Load data - final jsonString = prefs.getString(_prefsKeyData); - if (jsonString != null) { - final List jsonList = jsonDecode(jsonString); - _locations = jsonList - .map((json) => SuspectedLocation.fromJson(json as Map)) - .toList(); - debugPrint('[SuspectedLocationService] Loaded ${_locations.length} suspected locations from storage'); - } + debugPrint('[SuspectedLocationService] Loaded settings - enabled: $_isEnabled'); } catch (e) { debugPrint('[SuspectedLocationService] Error loading from storage: $e'); - _locations.clear(); - _lastFetchTime = null; - } - } - - /// Save data to shared preferences - Future _saveToStorage() async { - try { - final prefs = await SharedPreferences.getInstance(); - - // Save data - final jsonString = jsonEncode(_locations.map((loc) => loc.toJson()).toList()); - await prefs.setString(_prefsKeyData, jsonString); - - // Save last fetch time - if (_lastFetchTime != null) { - await prefs.setInt(_prefsKeyLastFetch, _lastFetchTime!.millisecondsSinceEpoch); - } - - debugPrint('[SuspectedLocationService] Saved ${_locations.length} suspected locations to storage'); - } catch (e) { - debugPrint('[SuspectedLocationService] Error saving to storage: $e'); } } @@ -188,9 +147,10 @@ class SuspectedLocationService { return false; } - // Parse rows - final List newLocations = []; + // Parse rows and store as raw data (don't process GeoJSON yet) + final List> rawDataList = []; int rowIndex = 0; + int validRows = 0; for (final row in dataRows) { rowIndex++; try { @@ -213,28 +173,30 @@ class SuspectedLocationService { } if (locationIndex < row.length) rowData['location'] = row[locationIndex]; - debugPrint('[SuspectedLocationService] Row $rowIndex data keys: ${rowData.keys.toList()}'); - if (rowIndex <= 3) { // Log first few rows - debugPrint('[SuspectedLocationService] Row $rowIndex ticket_no: ${rowData['ticket_no']}, location length: ${rowData['location']?.toString().length}'); + // Basic validation - must have ticket_no and location + if (rowData['ticket_no']?.toString().isNotEmpty == true && + rowData['location']?.toString().isNotEmpty == true) { + rawDataList.add(rowData); + validRows++; } - final location = SuspectedLocation.fromCsvRow(rowData); - newLocations.add(location); + // Log progress every 1000 rows + if (rowIndex % 1000 == 0) { + debugPrint('[SuspectedLocationService] Processing row $rowIndex...'); + } } catch (e, stackTrace) { // Skip rows that can't be parsed debugPrint('[SuspectedLocationService] Error parsing row $rowIndex: $e'); - debugPrint('[SuspectedLocationService] Stack trace: $stackTrace'); continue; } } - _locations = newLocations; - _lastFetchTime = DateTime.now(); + final fetchTime = DateTime.now(); - // Save to storage - await _saveToStorage(); + // Process raw data and save (calculates centroids once) + await _cache.processAndSave(rawDataList, fetchTime); - debugPrint('[SuspectedLocationService] Successfully fetched and parsed ${_locations.length} suspected locations'); + debugPrint('[SuspectedLocationService] Successfully fetched and stored $validRows valid raw entries (${rawDataList.length} total)'); return true; } catch (e, stackTrace) { @@ -253,13 +215,9 @@ class SuspectedLocationService { required double east, required double west, }) { - if (!_isEnabled || _locations.isEmpty) return []; - - return _locations.where((location) { - final lat = location.centroid.latitude; - final lng = location.centroid.longitude; - - return lat <= north && lat >= south && lng <= east && lng >= west; - }).toList(); + return _cache.getLocationsForBounds(LatLngBounds( + LatLng(north, west), + LatLng(south, east), + )); } } \ No newline at end of file diff --git a/lib/state/suspected_location_state.dart b/lib/state/suspected_location_state.dart index 96c2ba8..b929480 100644 --- a/lib/state/suspected_location_state.dart +++ b/lib/state/suspected_location_state.dart @@ -12,8 +12,20 @@ class SuspectedLocationState extends ChangeNotifier { /// Currently selected suspected location (for detail view) SuspectedLocation? get selectedLocation => _selectedLocation; - /// All suspected locations - List get locations => _service.locations; + /// Get suspected locations in bounds (this should be called by the map view) + List getLocationsInBounds({ + required double north, + required double south, + required double east, + required double west, + }) { + return _service.getLocationsInBounds( + north: north, + south: south, + east: east, + west: west, + ); + } /// Whether suspected locations are enabled bool get isEnabled => _service.isEnabled; @@ -62,18 +74,5 @@ class SuspectedLocationState extends ChangeNotifier { notifyListeners(); } - /// Get suspected locations within a bounding box - List getLocationsInBounds({ - required double north, - required double south, - required double east, - required double west, - }) { - return _service.getLocationsInBounds( - north: north, - south: south, - east: east, - west: west, - ); - } + } \ No newline at end of file diff --git a/lib/widgets/map_view.dart b/lib/widgets/map_view.dart index 6522cc1..fa69e93 100644 --- a/lib/widgets/map_view.dart +++ b/lib/widgets/map_view.dart @@ -355,6 +355,8 @@ class MapViewState extends State { // Build suspected location markers final suspectedLocationMarkers = []; if (appState.suspectedLocationsEnabled && mapBounds != null) { + debugPrint('[MapView] Suspected locations enabled, getting bounds: N${mapBounds.north.toStringAsFixed(4)}, S${mapBounds.south.toStringAsFixed(4)}, E${mapBounds.east.toStringAsFixed(4)}, W${mapBounds.west.toStringAsFixed(4)}'); + final suspectedLocations = appState.getSuspectedLocationsInBounds( north: mapBounds.north, south: mapBounds.south, @@ -362,6 +364,8 @@ class MapViewState extends State { west: mapBounds.west, ); + debugPrint('[MapView] Found ${suspectedLocations.length} suspected locations in bounds'); + suspectedLocationMarkers.addAll( SuspectedLocationMarkersBuilder.buildSuspectedLocationMarkers( locations: suspectedLocations, @@ -370,6 +374,10 @@ class MapViewState extends State { onLocationTap: widget.onSuspectedLocationTap, ), ); + + debugPrint('[MapView] Created ${suspectedLocationMarkers.length} suspected location markers'); + } else { + debugPrint('[MapView] Suspected locations not enabled (${appState.suspectedLocationsEnabled}) or no mapBounds ($mapBounds)'); } // Get current zoom level for direction cones diff --git a/pubspec.yaml b/pubspec.yaml index ee32dbc..561cb53 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,7 +1,7 @@ name: deflockapp description: Map public surveillance infrastructure with OpenStreetMap publish_to: "none" -version: 1.1.0+2 # The thing after the + is the google versionCode +version: 1.2.0+3 # The thing after the + is the google versionCode environment: sdk: ">=3.5.0 <4.0.0" # oauth2_client 4.x needs Dart 3.5+