primitive clustering, cache nominatim

clustering/cluster.py

@@ -0,0 +1,62 @@
+import requests
+import json
+from sklearn.cluster import DBSCAN
+from geopy.distance import great_circle
+from geopy.point import Point
+import numpy as np
+
+# Set up the Overpass API query
+# TODO: remove the bbox
+query = """
+[out:json];
+node["man_made"="surveillance"]["surveillance:type"="ALPR"];
+out body;
+"""
+
+# Request data from Overpass API
+print("Requesting data from Overpass API...")
+url = "http://overpass-api.de/api/interpreter"
+response = requests.get(url, params={'data': query}, headers={'User-Agent': 'DeFlock/1.0'})
+data = response.json()
+print("Data received. Parsing nodes...")
+
+# Parse nodes and extract lat/lon for clustering
+coordinates = []
+node_ids = []
+for element in data['elements']:
+    if element['type'] == 'node':
+        coordinates.append([element['lat'], element['lon']])
+        node_ids.append(element['id'])
+
+# Convert coordinates to NumPy array for DBSCAN
+coordinates = np.array(coordinates)
+
+# Define the clustering radius (10 miles in meters)
+radius_miles = 50
+radius_km = radius_miles * 1.60934  # 1 mile = 1.60934 km
+radius_in_radians = radius_km / 6371.0  # Earth's radius in km
+
+# Perform DBSCAN clustering
+db = DBSCAN(eps=radius_in_radians, min_samples=1, algorithm='ball_tree', metric='haversine').fit(np.radians(coordinates))
+labels = db.labels_
+
+# Prepare clusters and calculate centroids
+clusters = {}
+for label in set(labels):
+    cluster_points = coordinates[labels == label]
+    centroid = np.mean(cluster_points, axis=0)
+    first_node_id = node_ids[labels.tolist().index(label)]
+    
+    # Store in clusters dict with centroid and first node ID
+    clusters[label] = {
+        "lat": centroid[0],
+        "lon": centroid[1],
+        "id": first_node_id
+    }
+
+# Save clusters to JSON
+output = {"clusters": list(clusters.values())}
+with open("alpr_clusters.json", "w") as outfile:
+    json.dump(output, outfile, indent=2)
+
+print("Clustering complete. Results saved to alpr_clusters.json.")

shotgun/src/main/scala/me/deflock/shotgun/ShotgunServer.scala

@@ -97,10 +97,19 @@ object ShotgunServer {
 
     val bindingFuture = Http().newServerAt("0.0.0.0", 8080).bind(routes)
 
-    println(s"Server now online. Please navigate to http://localhost:8080\nPress RETURN to stop...")
-    StdIn.readLine() // let it run until user presses return
+    // Handle the binding future properly
+    bindingFuture.foreach { binding =>
+      println(s"Server online at http://localhost:${binding.localAddress.getPort}/")
+      println("Press RETURN to stop...")
+    }
+    
+    StdIn.readLine()
+
     bindingFuture
-      .flatMap(_.unbind()) // trigger unbinding from the port
-      .onComplete(_ => system.terminate()) // and shutdown when done
+      .flatMap(_.unbind())
+      .onComplete { _ =>
+        println("Server shutting down...")
+        system.terminate()
+      }
   }
 }

shotgun/src/main/scala/services/NominatimClient.scala

@@ -6,27 +6,48 @@ import pekko.http.scaladsl.Http
 import pekko.http.scaladsl.model._
 import pekko.http.scaladsl.unmarshalling.Unmarshal
 import spray.json._
+
+import scala.collection.mutable
 import scala.concurrent.{ExecutionContextExecutor, Future}
 
 class NominatimClient(implicit val system: ActorSystem, implicit val executionContext: ExecutionContextExecutor) {
   val baseUrl = "https://nominatim.openstreetmap.org/search"
+  private val cache: mutable.LinkedHashMap[String, JsValue] = new mutable.LinkedHashMap[String, JsValue]()
+  private val maxCacheSize = 300
+
+  private def cleanUpCache(): Unit = {
+    if (cache.size > maxCacheSize) {
+      val oldest = cache.head
+      cache.remove(oldest._1)
+    }
+  }
 
   def geocodePhrase(query: String): Future[JsValue] = {
-    val request = HttpRequest(
-      uri = s"$baseUrl?q=$query&format=json",
-      headers = List(headers.`User-Agent`("DeFlock/1.0"))
-    )
+    cleanUpCache()
+    cache.get(query) match {
+      case Some(cachedResult) =>
+        println(s"Cache hit for $query")
+        Future.successful(cachedResult)
+      case _ =>
+        println(s"Cache miss for $query")
+        val request = HttpRequest(
+          uri = s"$baseUrl?q=$query&format=json",
+          headers = List(headers.`User-Agent`("DeFlock/1.0"))
+        )
 
-    Http().singleRequest(request).flatMap { response =>
-      response.status match {
-        case StatusCodes.OK =>
-          Unmarshal(response.entity).to[String].map { jsonString =>
-            jsonString.parseJson
+        Http().singleRequest(request).flatMap { response =>
+          response.status match {
+            case StatusCodes.OK =>
+              Unmarshal(response.entity).to[String].map { jsonString =>
+                val json = jsonString.parseJson
+                cache.put(query, json)
+                json
+              }
+            case _ =>
+              response.discardEntityBytes()
+              Future.failed(new Exception(s"Failed to geocode phrase: ${response.status}"))
           }
-        case _ =>
-          response.discardEntityBytes()
-          Future.failed(new Exception(s"Failed to geocode phrase: ${response.status}"))
-      }
+        }
     }
   }
 }

webapp/src/components/DFMapMarker.vue

@@ -18,7 +18,7 @@
 </template>
 
 <script setup lang="ts">
-import { LCircleMarker, LCircle, LPolygon, LPopup } from '@vue-leaflet/vue-leaflet';
+import { LCircleMarker, LPolygon, LPopup } from '@vue-leaflet/vue-leaflet';
 import DFMapPopup from '@/components/DFMapPopup.vue';
 import type { ALPR } from '@/types';
 import type { PropType } from 'vue';

webapp/src/components/DFMapPopup.vue

@@ -14,7 +14,7 @@
         <v-icon start>mdi-cctv</v-icon> <b>Directional {{ alpr.tags.direction ? `(${degreesToCardinal(parseInt(alpr.tags.direction))})` : '' }}</b>
       </v-list-item>
       <v-list-item>
-        <v-icon start>mdi-domain</v-icon> <b>{{ alpr.tags.brand ?? 'Unknown' }}</b>
+        <v-icon start>mdi-domain</v-icon> <b>{{ alpr.tags.brand ?? 'Unknown Brand' }}</b>
       </v-list-item>
     </v-list>
     <!-- <v-data-table density="compact" hide-default-header hide-default-footer disable-sort :items="kvTags" /> -->

webapp/src/components/DFMarkerCluster.vue

@@ -0,0 +1,21 @@
+<template>
+  <l-circle-marker
+    @click="$emit('click', props)"
+    :lat-lng="[props.lat, props.lon]"
+    :radius="26"
+    :center="[props.lat, props.lon]"
+    color="#3f54f3"
+  />
+</template>
+
+<script setup lang="ts">
+import { LCircleMarker } from '@vue-leaflet/vue-leaflet';
+import { defineProps, defineEmits } from 'vue';
+
+defineEmits(['click']);
+
+const props = defineProps({
+  lat: Number,
+  lon: Number,
+});
+</script>

webapp/src/services/apiService.ts

@@ -1,5 +1,11 @@
 import axios from "axios";
 
+export interface Cluster {
+  id: string;
+  lat: number;
+  lon: number;
+}
+
 export interface BoundingBoxLiteral {
   minLat: number;
   maxLat: number;
@@ -20,6 +26,10 @@ export class BoundingBox implements BoundingBoxLiteral {
     this.maxLng = maxLng;
   }
 
+  containsPoint(lat: number, lng: number) {
+    return lat >= this.minLat && lat <= this.maxLat && lng >= this.minLng && lng <= this.maxLng;
+  }
+
   updateFromOther(boundingBoxLiteral: BoundingBoxLiteral) {
     this.minLat = boundingBoxLiteral.minLat;
     this.maxLat = boundingBoxLiteral.maxLat;
@@ -55,6 +65,12 @@ export const getALPRs = async (boundingBox: BoundingBox) => {
   return response.data;
 }
 
+export const getClusters = async () => {
+  const s3Url = "https://deflock-clusters.s3.us-east-1.amazonaws.com/alpr_clusters.json";
+  const response = await apiService.get(s3Url);
+  return response.data;
+}
+
 export const geocodeQuery = async (query: string, currentLocation: any) => {
   const encodedQuery = encodeURIComponent(query);
   const results = (await apiService.get(`/geocode?query=${encodedQuery}`)).data;

webapp/src/views/HomeView.vue

@@ -18,20 +18,23 @@
       :options="{ zoomControl: false, attributionControl: false }"
     >
       <l-control position="bottomleft">
-        <v-card>
-          <v-list density="compact">
-            <v-list-item>
-              <v-list-item-content>
-                <v-icon start color="#3f54f3">mdi-circle</v-icon> Directional ALPR
-              </v-list-item-content>
-            </v-list-item>
-            <v-list-item>
-              <v-list-item-content>
-                <v-icon start color="#ff5722">mdi-circle</v-icon> Omni w/ Face Recognition
-              </v-list-item-content>
-            </v-list-item>
-          </v-list>
-        </v-card>
+        <v-expansion-panels>
+          <v-expansion-panel>
+            <v-expansion-panel-title>Legend</v-expansion-panel-title>
+            <v-expansion-panel-text>
+              <v-card flat>
+                <v-list density="compact">
+                  <v-list-item class="px-0">
+                    <v-icon start color="#3f54f3">mdi-circle</v-icon> Directional ALPR
+                  </v-list-item>
+                  <v-list-item class="px-0">
+                    <v-icon start color="#ff5722">mdi-circle</v-icon> Omnidirectional w/ Face Recognition
+                  </v-list-item>
+                </v-list>
+              </v-card>
+            </v-expansion-panel-text>
+          </v-expansion-panel>
+        </v-expansion-panels>
       </l-control>
       <l-control position="topleft">
         <form @submit.prevent="onSearch">
@@ -64,7 +67,9 @@
         name="OpenStreetMap"
       />
       <l-control-zoom position="bottomright" />
-      <DFMapMarker v-for="alpr in alprsInView" :key="alpr.id" :alpr :show-fov="zoom >= 16" />
+
+      <DFMarkerCluster v-if="showClusters" @click="zoomToCluster" v-for="cluster in clusters" :key="cluster.id" :lat="cluster.lat" :lon="cluster.lon" />
+      <DFMapMarker v-else v-for="alpr in visibleALPRs" :key="alpr.id" :alpr :show-fov="zoom >= 16" />
     </l-map>
     <div class="loader" v-else>
       <span class="mb-4 text-grey">Loading Map</span>
@@ -76,17 +81,21 @@
 <script setup lang="ts">
 import 'leaflet/dist/leaflet.css';
 import { LMap, LTileLayer, LControlZoom, LControl } from '@vue-leaflet/vue-leaflet';
-import { ref, onMounted, computed } from 'vue';
+import { ref, onMounted, computed, watch } from 'vue';
 import { useRouter } from 'vue-router'
 import type { Ref } from 'vue';
 import { BoundingBox } from '@/services/apiService';
-import { getALPRs, geocodeQuery } from '@/services/apiService';
+import type { Cluster } from '@/services/apiService';
+import { getALPRs, geocodeQuery, getClusters } from '@/services/apiService';
 import { useDisplay, useTheme } from 'vuetify';
 import DFMapMarker from '@/components/DFMapMarker.vue';
+import DFMarkerCluster from '@/components/DFMarkerCluster.vue';
 import NewVisitor from '@/components/NewVisitor.vue';
 import type { ALPR } from '@/types';
 
 const DEFAULT_ZOOM = 12;
+const MIN_ZOOM_FOR_REFRESH = 4;
+const CLUSTER_ZOOM_THRESHOLD = 8;
 
 const theme = useTheme();
 const zoom: Ref<number> = ref(DEFAULT_ZOOM);
@@ -97,16 +106,28 @@ const searchQuery: Ref<string> = ref('');
 const router = useRouter();
 const { xs } = useDisplay();
 
-const canRefreshMarkers = computed(() => zoom.value >= 10);
+const canRefreshMarkers = computed(() => zoom.value >= MIN_ZOOM_FOR_REFRESH);
 const mapTileUrl = computed(() =>
   theme.global.name.value === 'dark' ?
     'https://tiles.stadiamaps.com/tiles/alidade_smooth_dark/{z}/{x}/{y}{r}.png' :
     'https://tiles.stadiamaps.com/tiles/osm_bright/{z}/{x}/{y}{r}.png'
 );
 
-const alprsInView: Ref<ALPR[]> = ref([]);
+const alprs: Ref<ALPR[]> = ref([]);
+const clusters: Ref<Cluster[]> = ref([]);
 const bboxForLastRequest: Ref<BoundingBox|null> = ref(null);
 
+const showClusters = computed(() => zoom.value <= CLUSTER_ZOOM_THRESHOLD);
+
+const visibleALPRs = computed(() => {
+  return alprs.value.filter(alpr => bounds.value?.containsPoint(alpr.lat, alpr.lon));
+});
+
+function zoomToCluster({ lat, lon }: { lat: number, lon: number }) {
+  center.value = { lat: lat, lng: lon };
+  zoom.value = CLUSTER_ZOOM_THRESHOLD + 1;
+}
+
 function handleKeyUp(event: KeyboardEvent) {
   if (event.key === '/' && searchField.value.value !== document.activeElement) {
     searchField.value.focus();
@@ -186,27 +207,38 @@ function updateURL() {
   });
 }
 
+watch(showClusters, (newValue, oldValue) => {
+  if (newValue && !oldValue) {
+    bboxForLastRequest.value = bounds.value;
+  }
+});
+
 function updateMarkers() {
   // Fetch ALPRs in the current view
   if (!bounds.value) {
     return;
   }
 
-  if (!canRefreshMarkers.value) {
+  if (showClusters.value || !canRefreshMarkers.value) {
     return;
   }
 
   getALPRs(bounds.value)
-    .then((alprs: any) => {
+    .then((result: any) => {
       // merge incoming with existing, so that moving the map doesn't remove markers
-      const existingIds = new Set(alprsInView.value.map(alpr => alpr.id));
-      const newAlprs = alprs.elements.filter((alpr: any) => !existingIds.has(alpr.id));
-      alprsInView.value = [...alprsInView.value, ...newAlprs];
+      const existingIds = new Set(alprs.value.map(alpr => alpr.id));
+      const newAlprs = result.elements.filter((alpr: any) => !existingIds.has(alpr.id));
+      alprs.value = [...alprs.value, ...newAlprs];
       bboxForLastRequest.value = bounds.value;
     });
 }
 
 onMounted(() => {
+  getClusters()
+    .then((result: any) => {
+      clusters.value = result.clusters;
+    });
+
   const hash = router.currentRoute.value.hash;
   if (hash) {
     const parts = hash.split('/');