mirror of
https://github.com/FoggedLens/deflock-app.git
synced 2026-09-28 12:02:02 +02:00
Suspected locations database
This commit is contained in:
stopflock
@@ -1,130 +1,124 @@
|
||||
import 'dart:convert';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter_map/flutter_map.dart';
|
||||
import 'package:shared_preferences/shared_preferences.dart';
|
||||
import 'package:latlong2/latlong.dart';
|
||||
|
||||
import '../models/suspected_location.dart';
|
||||
import 'suspected_location_service.dart';
|
||||
|
||||
/// Lightweight entry with pre-calculated centroid for efficient bounds checking
|
||||
class SuspectedLocationEntry {
|
||||
final Map<String, dynamic> rawData;
|
||||
final LatLng centroid;
|
||||
|
||||
SuspectedLocationEntry({required this.rawData, required this.centroid});
|
||||
|
||||
Map<String, dynamic> toJson() => {
|
||||
'rawData': rawData,
|
||||
'centroid': [centroid.latitude, centroid.longitude],
|
||||
};
|
||||
|
||||
factory SuspectedLocationEntry.fromJson(Map<String, dynamic> json) {
|
||||
final centroidList = json['centroid'] as List;
|
||||
return SuspectedLocationEntry(
|
||||
rawData: Map<String, dynamic>.from(json['rawData']),
|
||||
centroid: LatLng(
|
||||
(centroidList[0] as num).toDouble(),
|
||||
(centroidList[1] as num).toDouble(),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
import 'suspected_location_database.dart';
|
||||
|
||||
class SuspectedLocationCache extends ChangeNotifier {
|
||||
static final SuspectedLocationCache _instance = SuspectedLocationCache._();
|
||||
factory SuspectedLocationCache() => _instance;
|
||||
SuspectedLocationCache._();
|
||||
|
||||
static const String _prefsKeyProcessedData = 'suspected_locations_processed_data';
|
||||
static const String _prefsKeyLastFetch = 'suspected_locations_last_fetch';
|
||||
|
||||
List<SuspectedLocationEntry> _processedEntries = [];
|
||||
DateTime? _lastFetchTime;
|
||||
final SuspectedLocationDatabase _database = SuspectedLocationDatabase();
|
||||
final Map<String, List<SuspectedLocation>> _boundsCache = {};
|
||||
|
||||
/// Get suspected locations within specific bounds (cached)
|
||||
List<SuspectedLocation> getLocationsForBounds(LatLngBounds bounds) {
|
||||
// Add a synchronous cache for UI responsiveness
|
||||
// This holds recently fetched bounds data to support synchronous API calls
|
||||
final Map<String, List<SuspectedLocation>> _syncCache = {};
|
||||
final Set<String> _pendingQueries = {};
|
||||
|
||||
/// Get suspected locations within specific bounds (async version)
|
||||
Future<List<SuspectedLocation>> getLocationsForBounds(LatLngBounds bounds) async {
|
||||
if (!SuspectedLocationService().isEnabled) {
|
||||
debugPrint('[SuspectedLocationCache] Service not enabled');
|
||||
return [];
|
||||
}
|
||||
|
||||
final boundsKey = '${bounds.north.toStringAsFixed(4)},${bounds.south.toStringAsFixed(4)},${bounds.east.toStringAsFixed(4)},${bounds.west.toStringAsFixed(4)}';
|
||||
|
||||
// debugPrint('[SuspectedLocationCache] Getting locations for bounds: $boundsKey, processed entries count: ${_processedEntries.length}');
|
||||
|
||||
// Check cache first
|
||||
if (_boundsCache.containsKey(boundsKey)) {
|
||||
debugPrint('[SuspectedLocationCache] Using cached result: ${_boundsCache[boundsKey]!.length} locations');
|
||||
return _boundsCache[boundsKey]!;
|
||||
}
|
||||
|
||||
// Filter processed entries for this bounds (very fast since centroids are pre-calculated)
|
||||
final locations = <SuspectedLocation>[];
|
||||
int inBoundsCount = 0;
|
||||
|
||||
for (final entry in _processedEntries) {
|
||||
// Quick bounds check using pre-calculated centroid
|
||||
final lat = entry.centroid.latitude;
|
||||
final lng = entry.centroid.longitude;
|
||||
try {
|
||||
// Query database for locations in bounds
|
||||
final locations = await _database.getLocationsInBounds(bounds);
|
||||
|
||||
if (lat <= bounds.north && lat >= bounds.south &&
|
||||
lng <= bounds.east && lng >= bounds.west) {
|
||||
try {
|
||||
// Only create SuspectedLocation object if it's in bounds
|
||||
final location = SuspectedLocation.fromCsvRow(entry.rawData);
|
||||
locations.add(location);
|
||||
inBoundsCount++;
|
||||
} catch (e) {
|
||||
// Skip invalid entries
|
||||
continue;
|
||||
}
|
||||
// Cache the result in both caches
|
||||
_boundsCache[boundsKey] = locations;
|
||||
_syncCache[boundsKey] = locations;
|
||||
|
||||
// Limit cache sizes to prevent memory issues
|
||||
if (_boundsCache.length > 100) {
|
||||
final oldestKey = _boundsCache.keys.first;
|
||||
_boundsCache.remove(oldestKey);
|
||||
}
|
||||
if (_syncCache.length > 50) {
|
||||
final oldestKey = _syncCache.keys.first;
|
||||
_syncCache.remove(oldestKey);
|
||||
}
|
||||
|
||||
return locations;
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationCache] Error querying database: $e');
|
||||
return [];
|
||||
}
|
||||
|
||||
// debugPrint('[SuspectedLocationCache] Checked ${_processedEntries.length} entries, $inBoundsCount in bounds, result: ${locations.length} locations');
|
||||
|
||||
// Cache the result
|
||||
_boundsCache[boundsKey] = locations;
|
||||
|
||||
// Limit cache size to prevent memory issues
|
||||
if (_boundsCache.length > 100) {
|
||||
final oldestKey = _boundsCache.keys.first;
|
||||
_boundsCache.remove(oldestKey);
|
||||
}
|
||||
|
||||
return locations;
|
||||
}
|
||||
|
||||
/// Load processed data from storage
|
||||
/// Get suspected locations within specific bounds (synchronous version for UI)
|
||||
/// This returns cached data immediately and triggers async fetch if needed
|
||||
List<SuspectedLocation> getLocationsForBoundsSync(LatLngBounds bounds) {
|
||||
if (!SuspectedLocationService().isEnabled) {
|
||||
return [];
|
||||
}
|
||||
|
||||
final boundsKey = '${bounds.north.toStringAsFixed(4)},${bounds.south.toStringAsFixed(4)},${bounds.east.toStringAsFixed(4)},${bounds.west.toStringAsFixed(4)}';
|
||||
|
||||
// Return sync cache immediately if available
|
||||
if (_syncCache.containsKey(boundsKey)) {
|
||||
return _syncCache[boundsKey]!;
|
||||
}
|
||||
|
||||
// If not cached and not already being fetched, trigger async fetch
|
||||
if (!_pendingQueries.contains(boundsKey)) {
|
||||
_pendingQueries.add(boundsKey);
|
||||
_fetchAndCacheAsync(bounds, boundsKey);
|
||||
}
|
||||
|
||||
// Return empty list immediately (will be updated when async fetch completes)
|
||||
return [];
|
||||
}
|
||||
|
||||
/// Async fetch and cache helper
|
||||
void _fetchAndCacheAsync(LatLngBounds bounds, String boundsKey) async {
|
||||
try {
|
||||
final locations = await _database.getLocationsInBounds(bounds);
|
||||
|
||||
_syncCache[boundsKey] = locations;
|
||||
_boundsCache[boundsKey] = locations;
|
||||
|
||||
// Limit cache sizes
|
||||
if (_syncCache.length > 50) {
|
||||
final oldestKey = _syncCache.keys.first;
|
||||
_syncCache.remove(oldestKey);
|
||||
}
|
||||
if (_boundsCache.length > 100) {
|
||||
final oldestKey = _boundsCache.keys.first;
|
||||
_boundsCache.remove(oldestKey);
|
||||
}
|
||||
|
||||
// Notify listeners to trigger UI rebuild
|
||||
notifyListeners();
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationCache] Error in async fetch: $e');
|
||||
} finally {
|
||||
_pendingQueries.remove(boundsKey);
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize the cache (ensures database is ready)
|
||||
Future<void> loadFromStorage() async {
|
||||
try {
|
||||
final prefs = await SharedPreferences.getInstance();
|
||||
|
||||
// Load last fetch time
|
||||
final lastFetchMs = prefs.getInt(_prefsKeyLastFetch);
|
||||
if (lastFetchMs != null) {
|
||||
_lastFetchTime = DateTime.fromMillisecondsSinceEpoch(lastFetchMs);
|
||||
}
|
||||
|
||||
// Load processed data
|
||||
final processedDataString = prefs.getString(_prefsKeyProcessedData);
|
||||
if (processedDataString != null) {
|
||||
final List<dynamic> processedDataList = jsonDecode(processedDataString);
|
||||
_processedEntries = processedDataList
|
||||
.map((json) => SuspectedLocationEntry.fromJson(json as Map<String, dynamic>))
|
||||
.toList();
|
||||
debugPrint('[SuspectedLocationCache] Loaded ${_processedEntries.length} processed entries from storage');
|
||||
}
|
||||
await _database.init();
|
||||
debugPrint('[SuspectedLocationCache] Database initialized successfully');
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationCache] Error loading from storage: $e');
|
||||
_processedEntries.clear();
|
||||
_lastFetchTime = null;
|
||||
debugPrint('[SuspectedLocationCache] Error initializing database: $e');
|
||||
}
|
||||
}
|
||||
|
||||
/// Process raw CSV data and save to storage (calculates centroids once)
|
||||
/// Process raw CSV data and save to database
|
||||
Future<void> processAndSave(
|
||||
List<Map<String, dynamic>> rawData,
|
||||
DateTime fetchTime,
|
||||
@@ -132,96 +126,39 @@ class SuspectedLocationCache extends ChangeNotifier {
|
||||
try {
|
||||
debugPrint('[SuspectedLocationCache] Processing ${rawData.length} raw entries...');
|
||||
|
||||
final processedEntries = <SuspectedLocationEntry>[];
|
||||
int validCount = 0;
|
||||
int errorCount = 0;
|
||||
int zeroCoordCount = 0;
|
||||
|
||||
for (int i = 0; i < rawData.length; i++) {
|
||||
final rowData = rawData[i];
|
||||
|
||||
// Log progress every 1000 entries for debugging
|
||||
if (i % 1000 == 0) {
|
||||
debugPrint('[SuspectedLocationCache] Processed ${i + 1}/${rawData.length} entries...');
|
||||
}
|
||||
|
||||
try {
|
||||
// Create a temporary SuspectedLocation to extract the centroid
|
||||
final tempLocation = SuspectedLocation.fromCsvRow(rowData);
|
||||
|
||||
// Only save if we have a valid centroid (not at 0,0)
|
||||
if (tempLocation.centroid.latitude != 0 || tempLocation.centroid.longitude != 0) {
|
||||
processedEntries.add(SuspectedLocationEntry(
|
||||
rawData: rowData,
|
||||
centroid: tempLocation.centroid,
|
||||
));
|
||||
validCount++;
|
||||
} else {
|
||||
zeroCoordCount++;
|
||||
}
|
||||
} catch (e) {
|
||||
errorCount++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
debugPrint('[SuspectedLocationCache] Processing complete - Valid: $validCount, Zero coords: $zeroCoordCount, Errors: $errorCount');
|
||||
|
||||
_processedEntries = processedEntries;
|
||||
_lastFetchTime = fetchTime;
|
||||
|
||||
// Clear bounds cache since data changed
|
||||
// Clear all caches since data will change
|
||||
_boundsCache.clear();
|
||||
_syncCache.clear();
|
||||
_pendingQueries.clear();
|
||||
|
||||
final prefs = await SharedPreferences.getInstance();
|
||||
// Insert data into database in batch
|
||||
await _database.insertBatch(rawData, fetchTime);
|
||||
|
||||
// Save processed data
|
||||
final processedDataString = jsonEncode(processedEntries.map((e) => e.toJson()).toList());
|
||||
await prefs.setString(_prefsKeyProcessedData, processedDataString);
|
||||
final totalCount = await _database.getTotalCount();
|
||||
debugPrint('[SuspectedLocationCache] Processed and saved $totalCount entries to database');
|
||||
|
||||
// Save last fetch time
|
||||
await prefs.setInt(_prefsKeyLastFetch, fetchTime.millisecondsSinceEpoch);
|
||||
|
||||
// Log coordinate ranges for debugging
|
||||
if (processedEntries.isNotEmpty) {
|
||||
double minLat = processedEntries.first.centroid.latitude;
|
||||
double maxLat = minLat;
|
||||
double minLng = processedEntries.first.centroid.longitude;
|
||||
double maxLng = minLng;
|
||||
|
||||
for (final entry in processedEntries) {
|
||||
final lat = entry.centroid.latitude;
|
||||
final lng = entry.centroid.longitude;
|
||||
if (lat < minLat) minLat = lat;
|
||||
if (lat > maxLat) maxLat = lat;
|
||||
if (lng < minLng) minLng = lng;
|
||||
if (lng > maxLng) maxLng = lng;
|
||||
}
|
||||
|
||||
debugPrint('[SuspectedLocationCache] Coordinate ranges - Lat: $minLat to $maxLat, Lng: $minLng to $maxLng');
|
||||
}
|
||||
|
||||
debugPrint('[SuspectedLocationCache] Processed and saved $validCount valid entries (${processedEntries.length} total)');
|
||||
notifyListeners();
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationCache] Error processing and saving: $e');
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear all cached data
|
||||
void clear() {
|
||||
_processedEntries.clear();
|
||||
Future<void> clear() async {
|
||||
_boundsCache.clear();
|
||||
_lastFetchTime = null;
|
||||
_syncCache.clear();
|
||||
_pendingQueries.clear();
|
||||
await _database.clearAllData();
|
||||
notifyListeners();
|
||||
}
|
||||
|
||||
/// Get last fetch time
|
||||
DateTime? get lastFetchTime => _lastFetchTime;
|
||||
Future<DateTime?> get lastFetchTime => _database.getLastFetchTime();
|
||||
|
||||
/// Get total count of processed entries
|
||||
int get totalCount => _processedEntries.length;
|
||||
Future<int> get totalCount => _database.getTotalCount();
|
||||
|
||||
/// Check if we have data
|
||||
bool get hasData => _processedEntries.isNotEmpty;
|
||||
Future<bool> get hasData => _database.hasData();
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
import 'dart:convert';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter_map/flutter_map.dart';
|
||||
import 'package:latlong2/latlong.dart';
|
||||
import 'package:sqflite/sqflite.dart';
|
||||
import 'package:path/path.dart' as path;
|
||||
|
||||
import '../models/suspected_location.dart';
|
||||
|
||||
/// Database service for suspected location data
|
||||
/// Replaces the SharedPreferences-based cache to handle large datasets efficiently
|
||||
class SuspectedLocationDatabase {
|
||||
static final SuspectedLocationDatabase _instance = SuspectedLocationDatabase._();
|
||||
factory SuspectedLocationDatabase() => _instance;
|
||||
SuspectedLocationDatabase._();
|
||||
|
||||
Database? _database;
|
||||
static const String _dbName = 'suspected_locations.db';
|
||||
static const int _dbVersion = 1;
|
||||
|
||||
// Table and column names
|
||||
static const String _tableName = 'suspected_locations';
|
||||
static const String _columnTicketNo = 'ticket_no';
|
||||
static const String _columnCentroidLat = 'centroid_lat';
|
||||
static const String _columnCentroidLng = 'centroid_lng';
|
||||
static const String _columnBounds = 'bounds';
|
||||
static const String _columnGeoJson = 'geo_json';
|
||||
static const String _columnAllFields = 'all_fields';
|
||||
|
||||
// Metadata table for tracking last fetch time
|
||||
static const String _metaTableName = 'metadata';
|
||||
static const String _metaColumnKey = 'key';
|
||||
static const String _metaColumnValue = 'value';
|
||||
static const String _lastFetchKey = 'last_fetch_time';
|
||||
|
||||
/// Initialize the database
|
||||
Future<void> init() async {
|
||||
if (_database != null) return;
|
||||
|
||||
try {
|
||||
final dbPath = await getDatabasesPath();
|
||||
final fullPath = path.join(dbPath, _dbName);
|
||||
|
||||
debugPrint('[SuspectedLocationDatabase] Initializing database at $fullPath');
|
||||
|
||||
_database = await openDatabase(
|
||||
fullPath,
|
||||
version: _dbVersion,
|
||||
onCreate: _createTables,
|
||||
onUpgrade: _upgradeTables,
|
||||
);
|
||||
|
||||
debugPrint('[SuspectedLocationDatabase] Database initialized successfully');
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationDatabase] Error initializing database: $e');
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
|
||||
/// Create database tables
|
||||
Future<void> _createTables(Database db, int version) async {
|
||||
debugPrint('[SuspectedLocationDatabase] Creating tables...');
|
||||
|
||||
// Main suspected locations table
|
||||
await db.execute('''
|
||||
CREATE TABLE $_tableName (
|
||||
$_columnTicketNo TEXT PRIMARY KEY,
|
||||
$_columnCentroidLat REAL NOT NULL,
|
||||
$_columnCentroidLng REAL NOT NULL,
|
||||
$_columnBounds TEXT,
|
||||
$_columnGeoJson TEXT,
|
||||
$_columnAllFields TEXT NOT NULL
|
||||
)
|
||||
''');
|
||||
|
||||
// Create spatial index for efficient bounds queries
|
||||
await db.execute('''
|
||||
CREATE INDEX idx_centroid ON $_tableName ($_columnCentroidLat, $_columnCentroidLng)
|
||||
''');
|
||||
|
||||
// Metadata table for tracking last fetch time and other info
|
||||
await db.execute('''
|
||||
CREATE TABLE $_metaTableName (
|
||||
$_metaColumnKey TEXT PRIMARY KEY,
|
||||
$_metaColumnValue TEXT NOT NULL
|
||||
)
|
||||
''');
|
||||
|
||||
debugPrint('[SuspectedLocationDatabase] Tables created successfully');
|
||||
}
|
||||
|
||||
/// Handle database upgrades
|
||||
Future<void> _upgradeTables(Database db, int oldVersion, int newVersion) async {
|
||||
debugPrint('[SuspectedLocationDatabase] Upgrading database from version $oldVersion to $newVersion');
|
||||
// Future migrations would go here
|
||||
}
|
||||
|
||||
/// Get database instance, initializing if needed
|
||||
Future<Database> get database async {
|
||||
if (_database == null) {
|
||||
await init();
|
||||
}
|
||||
return _database!;
|
||||
}
|
||||
|
||||
/// Clear all data and recreate tables
|
||||
Future<void> clearAllData() async {
|
||||
try {
|
||||
final db = await database;
|
||||
|
||||
debugPrint('[SuspectedLocationDatabase] Clearing all data...');
|
||||
|
||||
// Drop and recreate tables (simpler than DELETE for large datasets)
|
||||
await db.execute('DROP TABLE IF EXISTS $_tableName');
|
||||
await db.execute('DROP TABLE IF EXISTS $_metaTableName');
|
||||
await _createTables(db, _dbVersion);
|
||||
|
||||
debugPrint('[SuspectedLocationDatabase] All data cleared successfully');
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationDatabase] Error clearing data: $e');
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert suspected locations in batch
|
||||
Future<void> insertBatch(List<Map<String, dynamic>> rawDataList, DateTime fetchTime) async {
|
||||
try {
|
||||
final db = await database;
|
||||
|
||||
debugPrint('[SuspectedLocationDatabase] Starting batch insert of ${rawDataList.length} entries...');
|
||||
|
||||
// Clear existing data first
|
||||
await clearAllData();
|
||||
|
||||
// Process entries in batches to avoid memory issues
|
||||
const batchSize = 1000;
|
||||
int totalInserted = 0;
|
||||
int validCount = 0;
|
||||
int errorCount = 0;
|
||||
|
||||
// Start transaction for better performance
|
||||
await db.transaction((txn) async {
|
||||
for (int i = 0; i < rawDataList.length; i += batchSize) {
|
||||
final batch = txn.batch();
|
||||
final endIndex = (i + batchSize < rawDataList.length) ? i + batchSize : rawDataList.length;
|
||||
final currentBatch = rawDataList.sublist(i, endIndex);
|
||||
|
||||
for (final rowData in currentBatch) {
|
||||
try {
|
||||
// Create temporary SuspectedLocation to extract centroid and bounds
|
||||
final tempLocation = SuspectedLocation.fromCsvRow(rowData);
|
||||
|
||||
// Skip entries with zero coordinates
|
||||
if (tempLocation.centroid.latitude == 0 && tempLocation.centroid.longitude == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Prepare data for database insertion
|
||||
final dbRow = {
|
||||
_columnTicketNo: tempLocation.ticketNo,
|
||||
_columnCentroidLat: tempLocation.centroid.latitude,
|
||||
_columnCentroidLng: tempLocation.centroid.longitude,
|
||||
_columnBounds: tempLocation.bounds.isNotEmpty
|
||||
? jsonEncode(tempLocation.bounds.map((p) => [p.latitude, p.longitude]).toList())
|
||||
: null,
|
||||
_columnGeoJson: tempLocation.geoJson != null ? jsonEncode(tempLocation.geoJson!) : null,
|
||||
_columnAllFields: jsonEncode(tempLocation.allFields),
|
||||
};
|
||||
|
||||
batch.insert(_tableName, dbRow, conflictAlgorithm: ConflictAlgorithm.replace);
|
||||
validCount++;
|
||||
|
||||
} catch (e) {
|
||||
errorCount++;
|
||||
// Skip invalid entries
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Commit this batch
|
||||
await batch.commit(noResult: true);
|
||||
totalInserted += currentBatch.length;
|
||||
|
||||
// Log progress every few batches
|
||||
if ((i ~/ batchSize) % 5 == 0) {
|
||||
debugPrint('[SuspectedLocationDatabase] Processed ${i + currentBatch.length}/${rawDataList.length} entries...');
|
||||
}
|
||||
}
|
||||
|
||||
// Insert metadata
|
||||
await txn.insert(
|
||||
_metaTableName,
|
||||
{
|
||||
_metaColumnKey: _lastFetchKey,
|
||||
_metaColumnValue: fetchTime.millisecondsSinceEpoch.toString(),
|
||||
},
|
||||
conflictAlgorithm: ConflictAlgorithm.replace,
|
||||
);
|
||||
});
|
||||
|
||||
debugPrint('[SuspectedLocationDatabase] Batch insert complete - Valid: $validCount, Errors: $errorCount');
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationDatabase] Error in batch insert: $e');
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
|
||||
/// Get suspected locations within bounding box
|
||||
Future<List<SuspectedLocation>> getLocationsInBounds(LatLngBounds bounds) async {
|
||||
try {
|
||||
final db = await database;
|
||||
|
||||
// Query with spatial bounds (simple lat/lng box filtering)
|
||||
final result = await db.query(
|
||||
_tableName,
|
||||
where: '''
|
||||
$_columnCentroidLat <= ? AND $_columnCentroidLat >= ? AND
|
||||
$_columnCentroidLng <= ? AND $_columnCentroidLng >= ?
|
||||
''',
|
||||
whereArgs: [bounds.north, bounds.south, bounds.east, bounds.west],
|
||||
);
|
||||
|
||||
// Convert database rows to SuspectedLocation objects
|
||||
final locations = <SuspectedLocation>[];
|
||||
for (final row in result) {
|
||||
try {
|
||||
final allFields = Map<String, dynamic>.from(jsonDecode(row[_columnAllFields] as String));
|
||||
|
||||
// Reconstruct bounds if available
|
||||
List<LatLng> boundsList = [];
|
||||
final boundsJson = row[_columnBounds] as String?;
|
||||
if (boundsJson != null) {
|
||||
final boundsData = jsonDecode(boundsJson) as List;
|
||||
boundsList = boundsData.map((b) => LatLng(
|
||||
(b[0] as num).toDouble(),
|
||||
(b[1] as num).toDouble(),
|
||||
)).toList();
|
||||
}
|
||||
|
||||
// Reconstruct GeoJSON if available
|
||||
Map<String, dynamic>? geoJson;
|
||||
final geoJsonString = row[_columnGeoJson] as String?;
|
||||
if (geoJsonString != null) {
|
||||
geoJson = Map<String, dynamic>.from(jsonDecode(geoJsonString));
|
||||
}
|
||||
|
||||
final location = SuspectedLocation(
|
||||
ticketNo: row[_columnTicketNo] as String,
|
||||
centroid: LatLng(
|
||||
row[_columnCentroidLat] as double,
|
||||
row[_columnCentroidLng] as double,
|
||||
),
|
||||
bounds: boundsList,
|
||||
geoJson: geoJson,
|
||||
allFields: allFields,
|
||||
);
|
||||
|
||||
locations.add(location);
|
||||
} catch (e) {
|
||||
// Skip invalid database entries
|
||||
debugPrint('[SuspectedLocationDatabase] Error parsing row: $e');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return locations;
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationDatabase] Error querying bounds: $e');
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/// Get last fetch time
|
||||
Future<DateTime?> getLastFetchTime() async {
|
||||
try {
|
||||
final db = await database;
|
||||
|
||||
final result = await db.query(
|
||||
_metaTableName,
|
||||
where: '$_metaColumnKey = ?',
|
||||
whereArgs: [_lastFetchKey],
|
||||
);
|
||||
|
||||
if (result.isNotEmpty) {
|
||||
final value = result.first[_metaColumnValue] as String;
|
||||
return DateTime.fromMillisecondsSinceEpoch(int.parse(value));
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationDatabase] Error getting last fetch time: $e');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// Get total count of entries
|
||||
Future<int> getTotalCount() async {
|
||||
try {
|
||||
final db = await database;
|
||||
final result = await db.rawQuery('SELECT COUNT(*) as count FROM $_tableName');
|
||||
return Sqflite.firstIntValue(result) ?? 0;
|
||||
} catch (e) {
|
||||
debugPrint('[SuspectedLocationDatabase] Error getting total count: $e');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if database has data
|
||||
Future<bool> hasData() async {
|
||||
final count = await getTotalCount();
|
||||
return count > 0;
|
||||
}
|
||||
|
||||
/// Close database connection
|
||||
Future<void> close() async {
|
||||
if (_database != null) {
|
||||
await _database!.close();
|
||||
_database = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -18,13 +18,13 @@ class SuspectedLocationService {
|
||||
|
||||
static const String _prefsKeyEnabled = 'suspected_locations_enabled';
|
||||
static const Duration _maxAge = Duration(days: 7);
|
||||
static const Duration _timeout = Duration(seconds: 30);
|
||||
static const Duration _timeout = Duration(minutes: 5); // Increased for large CSV files (100MB+)
|
||||
|
||||
final SuspectedLocationCache _cache = SuspectedLocationCache();
|
||||
bool _isEnabled = false;
|
||||
|
||||
/// Get last fetch time
|
||||
DateTime? get lastFetchTime => _cache.lastFetchTime;
|
||||
Future<DateTime?> get lastFetchTime => _cache.lastFetchTime;
|
||||
|
||||
/// Check if suspected locations are enabled
|
||||
bool get isEnabled => _isEnabled;
|
||||
@@ -37,11 +37,12 @@ class SuspectedLocationService {
|
||||
await _cache.loadFromStorage();
|
||||
|
||||
// Only auto-fetch if enabled, data is stale or missing, and we are not offline
|
||||
if (_isEnabled && _shouldRefresh() && !offlineMode) {
|
||||
if (_isEnabled && (await _shouldRefresh()) && !offlineMode) {
|
||||
debugPrint('[SuspectedLocationService] Auto-refreshing CSV data on startup (older than $_maxAge or missing)');
|
||||
await _fetchData();
|
||||
} else if (_isEnabled && _shouldRefresh() && offlineMode) {
|
||||
debugPrint('[SuspectedLocationService] Skipping auto-refresh due to offline mode - data is ${_cache.lastFetchTime != null ? 'outdated' : 'missing'}');
|
||||
} else if (_isEnabled && (await _shouldRefresh()) && offlineMode) {
|
||||
final lastFetch = await _cache.lastFetchTime;
|
||||
debugPrint('[SuspectedLocationService] Skipping auto-refresh due to offline mode - data is ${lastFetch != null ? 'outdated' : 'missing'}');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,20 +54,20 @@ class SuspectedLocationService {
|
||||
|
||||
// If disabling, clear the cache
|
||||
if (!enabled) {
|
||||
_cache.clear();
|
||||
await _cache.clear();
|
||||
}
|
||||
// Note: If enabling and no data, the state layer will call fetchDataIfNeeded()
|
||||
}
|
||||
|
||||
/// Check if cache has any data
|
||||
bool get hasData => _cache.hasData;
|
||||
Future<bool> get hasData => _cache.hasData;
|
||||
|
||||
/// Get last fetch time
|
||||
DateTime? get lastFetch => _cache.lastFetchTime;
|
||||
Future<DateTime?> get lastFetch => _cache.lastFetchTime;
|
||||
|
||||
/// Fetch data if needed (for enabling suspected locations when no data exists)
|
||||
Future<bool> fetchDataIfNeeded() async {
|
||||
if (!_shouldRefresh()) {
|
||||
if (!(await _shouldRefresh())) {
|
||||
debugPrint('[SuspectedLocationService] Data is fresh, skipping fetch');
|
||||
return true; // Already have fresh data
|
||||
}
|
||||
@@ -79,10 +80,11 @@ class SuspectedLocationService {
|
||||
}
|
||||
|
||||
/// Check if data should be refreshed
|
||||
bool _shouldRefresh() {
|
||||
if (!_cache.hasData) return true;
|
||||
if (_cache.lastFetchTime == null) return true;
|
||||
return DateTime.now().difference(_cache.lastFetchTime!) > _maxAge;
|
||||
Future<bool> _shouldRefresh() async {
|
||||
if (!(await _cache.hasData)) return true;
|
||||
final lastFetch = await _cache.lastFetchTime;
|
||||
if (lastFetch == null) return true;
|
||||
return DateTime.now().difference(lastFetch) > _maxAge;
|
||||
}
|
||||
|
||||
/// Load settings from shared preferences
|
||||
@@ -101,110 +103,145 @@ class SuspectedLocationService {
|
||||
|
||||
/// Fetch data from the CSV URL
|
||||
Future<bool> _fetchData() async {
|
||||
try {
|
||||
debugPrint('[SuspectedLocationService] Fetching CSV data from $kSuspectedLocationsCsvUrl');
|
||||
const maxRetries = 3;
|
||||
|
||||
for (int attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
debugPrint('[SuspectedLocationService] Fetching CSV data from $kSuspectedLocationsCsvUrl (attempt $attempt/$maxRetries)');
|
||||
if (attempt == 1) {
|
||||
debugPrint('[SuspectedLocationService] This may take up to ${_timeout.inMinutes} minutes for large datasets...');
|
||||
}
|
||||
|
||||
final response = await http.get(
|
||||
Uri.parse(kSuspectedLocationsCsvUrl),
|
||||
headers: {
|
||||
'User-Agent': 'DeFlock/1.0 (OSM surveillance mapping app)',
|
||||
},
|
||||
).timeout(_timeout);
|
||||
|
||||
final response = await http.get(
|
||||
Uri.parse(kSuspectedLocationsCsvUrl),
|
||||
headers: {
|
||||
'User-Agent': 'DeFlock/1.0 (OSM surveillance mapping app)',
|
||||
},
|
||||
).timeout(_timeout);
|
||||
if (response.statusCode != 200) {
|
||||
debugPrint('[SuspectedLocationService] HTTP error ${response.statusCode}');
|
||||
throw Exception('HTTP ${response.statusCode}');
|
||||
}
|
||||
|
||||
final responseSize = response.contentLength ?? response.bodyBytes.length;
|
||||
debugPrint('[SuspectedLocationService] Downloaded ${responseSize} bytes, parsing CSV...');
|
||||
|
||||
// Parse CSV with proper field separator and quote handling
|
||||
final csvData = await compute(_parseCSV, response.body);
|
||||
debugPrint('[SuspectedLocationService] Parsed ${csvData.length} rows from CSV');
|
||||
|
||||
if (csvData.isEmpty) {
|
||||
debugPrint('[SuspectedLocationService] Empty CSV data');
|
||||
throw Exception('Empty CSV data');
|
||||
}
|
||||
|
||||
// First row should be headers
|
||||
final headers = csvData.first.map((h) => h.toString().toLowerCase()).toList();
|
||||
debugPrint('[SuspectedLocationService] Headers: $headers');
|
||||
final dataRows = csvData.skip(1);
|
||||
debugPrint('[SuspectedLocationService] Data rows count: ${dataRows.length}');
|
||||
|
||||
// Find required column indices - we only need ticket_no and location
|
||||
final ticketNoIndex = headers.indexOf('ticket_no');
|
||||
final locationIndex = headers.indexOf('location');
|
||||
|
||||
debugPrint('[SuspectedLocationService] Column indices - ticket_no: $ticketNoIndex, location: $locationIndex');
|
||||
|
||||
if (ticketNoIndex == -1 || locationIndex == -1) {
|
||||
debugPrint('[SuspectedLocationService] Required columns not found in CSV. Headers: $headers');
|
||||
throw Exception('Required columns not found in CSV');
|
||||
}
|
||||
|
||||
if (response.statusCode != 200) {
|
||||
debugPrint('[SuspectedLocationService] HTTP error ${response.statusCode}');
|
||||
return false;
|
||||
}
|
||||
|
||||
// Parse CSV with proper field separator and quote handling
|
||||
final csvData = await compute(_parseCSV, response.body);
|
||||
debugPrint('[SuspectedLocationService] Parsed ${csvData.length} rows from CSV');
|
||||
|
||||
if (csvData.isEmpty) {
|
||||
debugPrint('[SuspectedLocationService] Empty CSV data');
|
||||
return false;
|
||||
}
|
||||
|
||||
// First row should be headers
|
||||
final headers = csvData.first.map((h) => h.toString().toLowerCase()).toList();
|
||||
debugPrint('[SuspectedLocationService] Headers: $headers');
|
||||
final dataRows = csvData.skip(1);
|
||||
debugPrint('[SuspectedLocationService] Data rows count: ${dataRows.length}');
|
||||
|
||||
// Find required column indices - we only need ticket_no and location
|
||||
final ticketNoIndex = headers.indexOf('ticket_no');
|
||||
final locationIndex = headers.indexOf('location');
|
||||
|
||||
debugPrint('[SuspectedLocationService] Column indices - ticket_no: $ticketNoIndex, location: $locationIndex');
|
||||
|
||||
if (ticketNoIndex == -1 || locationIndex == -1) {
|
||||
debugPrint('[SuspectedLocationService] Required columns not found in CSV. Headers: $headers');
|
||||
return false;
|
||||
}
|
||||
|
||||
// Parse rows and store all data dynamically
|
||||
final List<Map<String, dynamic>> rawDataList = [];
|
||||
int rowIndex = 0;
|
||||
int validRows = 0;
|
||||
for (final row in dataRows) {
|
||||
rowIndex++;
|
||||
try {
|
||||
final Map<String, dynamic> rowData = {};
|
||||
|
||||
// Store all columns dynamically
|
||||
for (int i = 0; i < headers.length && i < row.length; i++) {
|
||||
final headerName = headers[i];
|
||||
final cellValue = row[i];
|
||||
if (cellValue != null) {
|
||||
rowData[headerName] = cellValue;
|
||||
|
||||
// Parse rows and store all data dynamically
|
||||
final List<Map<String, dynamic>> rawDataList = [];
|
||||
int rowIndex = 0;
|
||||
int validRows = 0;
|
||||
for (final row in dataRows) {
|
||||
rowIndex++;
|
||||
try {
|
||||
final Map<String, dynamic> rowData = {};
|
||||
|
||||
// Store all columns dynamically
|
||||
for (int i = 0; i < headers.length && i < row.length; i++) {
|
||||
final headerName = headers[i];
|
||||
final cellValue = row[i];
|
||||
if (cellValue != null) {
|
||||
rowData[headerName] = cellValue;
|
||||
}
|
||||
}
|
||||
|
||||
// Basic validation - must have ticket_no and location
|
||||
if (rowData['ticket_no']?.toString().isNotEmpty == true &&
|
||||
rowData['location']?.toString().isNotEmpty == true) {
|
||||
rawDataList.add(rowData);
|
||||
validRows++;
|
||||
}
|
||||
|
||||
} catch (e, stackTrace) {
|
||||
// Skip rows that can't be parsed
|
||||
debugPrint('[SuspectedLocationService] Error parsing row $rowIndex: $e');
|
||||
continue;
|
||||
}
|
||||
|
||||
// Basic validation - must have ticket_no and location
|
||||
if (rowData['ticket_no']?.toString().isNotEmpty == true &&
|
||||
rowData['location']?.toString().isNotEmpty == true) {
|
||||
rawDataList.add(rowData);
|
||||
validRows++;
|
||||
}
|
||||
|
||||
} catch (e, stackTrace) {
|
||||
// Skip rows that can't be parsed
|
||||
debugPrint('[SuspectedLocationService] Error parsing row $rowIndex: $e');
|
||||
continue;
|
||||
}
|
||||
|
||||
debugPrint('[SuspectedLocationService] Parsed $validRows valid rows from ${dataRows.length} total rows');
|
||||
|
||||
final fetchTime = DateTime.now();
|
||||
|
||||
// Process raw data and save (calculates centroids once)
|
||||
await _cache.processAndSave(rawDataList, fetchTime);
|
||||
|
||||
debugPrint('[SuspectedLocationService] Successfully fetched and stored $validRows valid raw entries (${rawDataList.length} total)');
|
||||
return true;
|
||||
} catch (e, stackTrace) {
|
||||
debugPrint('[SuspectedLocationService] Attempt $attempt failed: $e');
|
||||
|
||||
if (attempt == maxRetries) {
|
||||
debugPrint('[SuspectedLocationService] All $maxRetries attempts failed');
|
||||
debugPrint('[SuspectedLocationService] Stack trace: $stackTrace');
|
||||
return false;
|
||||
} else {
|
||||
// Wait before retrying (exponential backoff)
|
||||
final delay = Duration(seconds: attempt * 10);
|
||||
debugPrint('[SuspectedLocationService] Retrying in ${delay.inSeconds} seconds...');
|
||||
await Future.delayed(delay);
|
||||
}
|
||||
}
|
||||
|
||||
debugPrint('[SuspectedLocationService] Parsed $validRows valid rows from ${dataRows.length} total rows');
|
||||
|
||||
final fetchTime = DateTime.now();
|
||||
|
||||
// Process raw data and save (calculates centroids once)
|
||||
await _cache.processAndSave(rawDataList, fetchTime);
|
||||
|
||||
debugPrint('[SuspectedLocationService] Successfully fetched and stored $validRows valid raw entries (${rawDataList.length} total)');
|
||||
return true;
|
||||
|
||||
} catch (e, stackTrace) {
|
||||
debugPrint('[SuspectedLocationService] Error fetching data: $e');
|
||||
debugPrint('[SuspectedLocationService] Stack trace: $stackTrace');
|
||||
return false;
|
||||
}
|
||||
|
||||
return false; // Should never reach here
|
||||
}
|
||||
|
||||
/// Get suspected locations within a bounding box
|
||||
List<SuspectedLocation> getLocationsInBounds({
|
||||
/// Get suspected locations within a bounding box (async)
|
||||
Future<List<SuspectedLocation>> getLocationsInBounds({
|
||||
required double north,
|
||||
required double south,
|
||||
required double east,
|
||||
required double west,
|
||||
}) async {
|
||||
return await _cache.getLocationsForBounds(LatLngBounds(
|
||||
LatLng(north, west),
|
||||
LatLng(south, east),
|
||||
));
|
||||
}
|
||||
|
||||
/// Get suspected locations within a bounding box (sync, for UI)
|
||||
List<SuspectedLocation> getLocationsInBoundsSync({
|
||||
required double north,
|
||||
required double south,
|
||||
required double east,
|
||||
required double west,
|
||||
}) {
|
||||
return _cache.getLocationsForBounds(LatLngBounds(
|
||||
return _cache.getLocationsForBoundsSync(LatLngBounds(
|
||||
LatLng(north, west),
|
||||
LatLng(south, east),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Simple CSV parser for compute() - must be top-level function
|
||||
List<List<dynamic>> _parseCSV(String csvBody) {
|
||||
return const CsvToListConverter(
|
||||
|
||||
Reference in New Issue
Block a user