From 0680fcb908cba97865654ee9e991bc24158daa80 Mon Sep 17 00:00:00 2001 From: Will Freeman Date: Tue, 29 Oct 2024 19:45:30 -0600 Subject: [PATCH] primitive clustering, cache nominatim --- clustering/cluster.py | 62 ++++++++++++++ .../me/deflock/shotgun/ShotgunServer.scala | 17 +++- .../main/scala/services/NominatimClient.scala | 47 ++++++++--- webapp/src/components/DFMapMarker.vue | 2 +- webapp/src/components/DFMapPopup.vue | 2 +- webapp/src/components/DFMarkerCluster.vue | 21 +++++ webapp/src/services/apiService.ts | 16 ++++ webapp/src/views/HomeView.vue | 80 +++++++++++++------ 8 files changed, 204 insertions(+), 43 deletions(-) create mode 100644 clustering/cluster.py create mode 100644 webapp/src/components/DFMarkerCluster.vue diff --git a/clustering/cluster.py b/clustering/cluster.py new file mode 100644 index 0000000..36aaddc --- /dev/null +++ b/clustering/cluster.py @@ -0,0 +1,62 @@ +import requests +import json +from sklearn.cluster import DBSCAN +from geopy.distance import great_circle +from geopy.point import Point +import numpy as np + +# Set up the Overpass API query +# TODO: remove the bbox +query = """ +[out:json]; +node["man_made"="surveillance"]["surveillance:type"="ALPR"]; +out body; +""" + +# Request data from Overpass API +print("Requesting data from Overpass API...") +url = "http://overpass-api.de/api/interpreter" +response = requests.get(url, params={'data': query}, headers={'User-Agent': 'DeFlock/1.0'}) +data = response.json() +print("Data received. Parsing nodes...") + +# Parse nodes and extract lat/lon for clustering +coordinates = [] +node_ids = [] +for element in data['elements']: + if element['type'] == 'node': + coordinates.append([element['lat'], element['lon']]) + node_ids.append(element['id']) + +# Convert coordinates to NumPy array for DBSCAN +coordinates = np.array(coordinates) + +# Define the clustering radius (10 miles in meters) +radius_miles = 50 +radius_km = radius_miles * 1.60934 # 1 mile = 1.60934 km +radius_in_radians = radius_km / 6371.0 # Earth's radius in km + +# Perform DBSCAN clustering +db = DBSCAN(eps=radius_in_radians, min_samples=1, algorithm='ball_tree', metric='haversine').fit(np.radians(coordinates)) +labels = db.labels_ + +# Prepare clusters and calculate centroids +clusters = {} +for label in set(labels): + cluster_points = coordinates[labels == label] + centroid = np.mean(cluster_points, axis=0) + first_node_id = node_ids[labels.tolist().index(label)] + + # Store in clusters dict with centroid and first node ID + clusters[label] = { + "lat": centroid[0], + "lon": centroid[1], + "id": first_node_id + } + +# Save clusters to JSON +output = {"clusters": list(clusters.values())} +with open("alpr_clusters.json", "w") as outfile: + json.dump(output, outfile, indent=2) + +print("Clustering complete. Results saved to alpr_clusters.json.") diff --git a/shotgun/src/main/scala/me/deflock/shotgun/ShotgunServer.scala b/shotgun/src/main/scala/me/deflock/shotgun/ShotgunServer.scala index d25791e..4af1026 100644 --- a/shotgun/src/main/scala/me/deflock/shotgun/ShotgunServer.scala +++ b/shotgun/src/main/scala/me/deflock/shotgun/ShotgunServer.scala @@ -97,10 +97,19 @@ object ShotgunServer { val bindingFuture = Http().newServerAt("0.0.0.0", 8080).bind(routes) - println(s"Server now online. Please navigate to http://localhost:8080\nPress RETURN to stop...") - StdIn.readLine() // let it run until user presses return + // Handle the binding future properly + bindingFuture.foreach { binding => + println(s"Server online at http://localhost:${binding.localAddress.getPort}/") + println("Press RETURN to stop...") + } + + StdIn.readLine() + bindingFuture - .flatMap(_.unbind()) // trigger unbinding from the port - .onComplete(_ => system.terminate()) // and shutdown when done + .flatMap(_.unbind()) + .onComplete { _ => + println("Server shutting down...") + system.terminate() + } } } diff --git a/shotgun/src/main/scala/services/NominatimClient.scala b/shotgun/src/main/scala/services/NominatimClient.scala index 585b26b..48ecf99 100644 --- a/shotgun/src/main/scala/services/NominatimClient.scala +++ b/shotgun/src/main/scala/services/NominatimClient.scala @@ -6,27 +6,48 @@ import pekko.http.scaladsl.Http import pekko.http.scaladsl.model._ import pekko.http.scaladsl.unmarshalling.Unmarshal import spray.json._ + +import scala.collection.mutable import scala.concurrent.{ExecutionContextExecutor, Future} class NominatimClient(implicit val system: ActorSystem, implicit val executionContext: ExecutionContextExecutor) { val baseUrl = "https://nominatim.openstreetmap.org/search" + private val cache: mutable.LinkedHashMap[String, JsValue] = new mutable.LinkedHashMap[String, JsValue]() + private val maxCacheSize = 300 + + private def cleanUpCache(): Unit = { + if (cache.size > maxCacheSize) { + val oldest = cache.head + cache.remove(oldest._1) + } + } def geocodePhrase(query: String): Future[JsValue] = { - val request = HttpRequest( - uri = s"$baseUrl?q=$query&format=json", - headers = List(headers.`User-Agent`("DeFlock/1.0")) - ) + cleanUpCache() + cache.get(query) match { + case Some(cachedResult) => + println(s"Cache hit for $query") + Future.successful(cachedResult) + case _ => + println(s"Cache miss for $query") + val request = HttpRequest( + uri = s"$baseUrl?q=$query&format=json", + headers = List(headers.`User-Agent`("DeFlock/1.0")) + ) - Http().singleRequest(request).flatMap { response => - response.status match { - case StatusCodes.OK => - Unmarshal(response.entity).to[String].map { jsonString => - jsonString.parseJson + Http().singleRequest(request).flatMap { response => + response.status match { + case StatusCodes.OK => + Unmarshal(response.entity).to[String].map { jsonString => + val json = jsonString.parseJson + cache.put(query, json) + json + } + case _ => + response.discardEntityBytes() + Future.failed(new Exception(s"Failed to geocode phrase: ${response.status}")) } - case _ => - response.discardEntityBytes() - Future.failed(new Exception(s"Failed to geocode phrase: ${response.status}")) - } + } } } } diff --git a/webapp/src/components/DFMapMarker.vue b/webapp/src/components/DFMapMarker.vue index b4158df..eb2f991 100644 --- a/webapp/src/components/DFMapMarker.vue +++ b/webapp/src/components/DFMapMarker.vue @@ -18,7 +18,7 @@ diff --git a/webapp/src/services/apiService.ts b/webapp/src/services/apiService.ts index 49ce155..35d6bd1 100644 --- a/webapp/src/services/apiService.ts +++ b/webapp/src/services/apiService.ts @@ -1,5 +1,11 @@ import axios from "axios"; +export interface Cluster { + id: string; + lat: number; + lon: number; +} + export interface BoundingBoxLiteral { minLat: number; maxLat: number; @@ -20,6 +26,10 @@ export class BoundingBox implements BoundingBoxLiteral { this.maxLng = maxLng; } + containsPoint(lat: number, lng: number) { + return lat >= this.minLat && lat <= this.maxLat && lng >= this.minLng && lng <= this.maxLng; + } + updateFromOther(boundingBoxLiteral: BoundingBoxLiteral) { this.minLat = boundingBoxLiteral.minLat; this.maxLat = boundingBoxLiteral.maxLat; @@ -55,6 +65,12 @@ export const getALPRs = async (boundingBox: BoundingBox) => { return response.data; } +export const getClusters = async () => { + const s3Url = "https://deflock-clusters.s3.us-east-1.amazonaws.com/alpr_clusters.json"; + const response = await apiService.get(s3Url); + return response.data; +} + export const geocodeQuery = async (query: string, currentLocation: any) => { const encodedQuery = encodeURIComponent(query); const results = (await apiService.get(`/geocode?query=${encodedQuery}`)).data; diff --git a/webapp/src/views/HomeView.vue b/webapp/src/views/HomeView.vue index a4670a7..950da89 100644 --- a/webapp/src/views/HomeView.vue +++ b/webapp/src/views/HomeView.vue @@ -18,20 +18,23 @@ :options="{ zoomControl: false, attributionControl: false }" > - - - - - mdi-circle Directional ALPR - - - - - mdi-circle Omni w/ Face Recognition - - - - + + + Legend + + + + + mdi-circle Directional ALPR + + + mdi-circle Omnidirectional w/ Face Recognition + + + + + +
@@ -64,7 +67,9 @@ name="OpenStreetMap" /> - + + +
Loading Map @@ -76,17 +81,21 @@