mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-07-09 21:58:41 +02:00
feat: v0.4 — satellite imagery, KiwiSDR radio, LOCATE bar & security cleanup
New features: - NASA GIBS (MODIS Terra) daily satellite imagery with 30-day time slider - Esri World Imagery high-res satellite layer (sub-meter, zoom 18+) - KiwiSDR SDR receivers on map with embedded radio tuner - Sentinel-2 intel card — right-click for recent satellite photo popup - LOCATE bar — search by coordinates or place name (Nominatim geocoding) - SATELLITE style preset in bottom bar cycling - v0.4 changelog modal on first launch Fixes: - Satellite imagery renders below data icons (imagery-ceiling anchor) - Sentinel-2 opens full-res PNG directly (not STAC catalog JSON) - Light/dark theme: UI stays dark, only map basemap changes Security: - Removed test files with hardcoded API keys from tracking - Removed .git_backup directory from tracking - Updated .gitignore to exclude test files, dev scripts, cache files Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
anoracleofra-code
File diff suppressed because one or more lines are too long
@@ -100,7 +100,8 @@ latest_data = {
|
||||
"uavs": [],
|
||||
"frontlines": None,
|
||||
"gdelt": [],
|
||||
"liveuamap": []
|
||||
"liveuamap": [],
|
||||
"kiwisdr": []
|
||||
}
|
||||
|
||||
# Thread lock for safe reads/writes to latest_data
|
||||
@@ -1250,6 +1251,14 @@ def fetch_cctv():
|
||||
logger.error(f"Error fetching cctv from DB: {e}")
|
||||
latest_data["cctv"] = []
|
||||
|
||||
def fetch_kiwisdr():
|
||||
try:
|
||||
from services.kiwisdr_fetcher import fetch_kiwisdr_nodes
|
||||
latest_data["kiwisdr"] = fetch_kiwisdr_nodes()
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching KiwiSDR nodes: {e}")
|
||||
latest_data["kiwisdr"] = []
|
||||
|
||||
def fetch_bikeshare():
|
||||
bikes = []
|
||||
try:
|
||||
@@ -1805,6 +1814,7 @@ def update_slow_data():
|
||||
fetch_cctv,
|
||||
fetch_earthquakes,
|
||||
fetch_geopolitics,
|
||||
fetch_kiwisdr,
|
||||
]
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(slow_funcs)) as executor:
|
||||
futures = [executor.submit(func) for func in slow_funcs]
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""
|
||||
KiwiSDR public receiver list fetcher.
|
||||
Scrapes the kiwisdr.com public page for active SDR receivers worldwide.
|
||||
Data is embedded as HTML comments inside each entry div.
|
||||
"""
|
||||
|
||||
import re
|
||||
import logging
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
kiwisdr_cache = TTLCache(maxsize=1, ttl=600) # 10-minute cache
|
||||
|
||||
|
||||
def _parse_comment(html: str, field: str) -> str:
|
||||
"""Extract a field value from HTML comment like <!-- field=value -->"""
|
||||
m = re.search(rf'<!--\s*{field}=(.*?)\s*-->', html)
|
||||
return m.group(1).strip() if m else ""
|
||||
|
||||
|
||||
def _parse_gps(html: str):
|
||||
"""Extract lat/lon from <!-- gps=(lat, lon) --> comment."""
|
||||
m = re.search(r'<!--\s*gps=\(([^,]+),\s*([^)]+)\)\s*-->', html)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(1)), float(m.group(2))
|
||||
except ValueError:
|
||||
return None, None
|
||||
return None, None
|
||||
|
||||
|
||||
@cached(kiwisdr_cache)
|
||||
def fetch_kiwisdr_nodes() -> list[dict]:
|
||||
"""Fetch and parse the KiwiSDR public receiver list."""
|
||||
from services.network_utils import smart_request
|
||||
|
||||
try:
|
||||
res = smart_request("http://kiwisdr.com/.public/", timeout=20)
|
||||
if not res or res.status_code != 200:
|
||||
logger.error(f"KiwiSDR fetch failed: HTTP {res.status_code if res else 'no response'}")
|
||||
return []
|
||||
|
||||
html = res.text
|
||||
# Split by entry divs
|
||||
entries = re.findall(r"<div class='cl-entry[^']*'>(.*?)</div>\s*</div>", html, re.DOTALL)
|
||||
|
||||
nodes = []
|
||||
for entry in entries:
|
||||
lat, lon = _parse_gps(entry)
|
||||
if lat is None or lon is None:
|
||||
continue
|
||||
if abs(lat) > 90 or abs(lon) > 180:
|
||||
continue
|
||||
|
||||
offline = _parse_comment(entry, "offline")
|
||||
if offline == "yes":
|
||||
continue
|
||||
|
||||
name = _parse_comment(entry, "name") or "Unknown SDR"
|
||||
users_str = _parse_comment(entry, "users")
|
||||
users_max_str = _parse_comment(entry, "users_max")
|
||||
bands = _parse_comment(entry, "bands")
|
||||
antenna = _parse_comment(entry, "antenna")
|
||||
location = _parse_comment(entry, "loc")
|
||||
|
||||
# Extract the URL from the href
|
||||
url_match = re.search(r"href='(https?://[^']+)'", entry)
|
||||
url = url_match.group(1) if url_match else ""
|
||||
|
||||
try:
|
||||
users = int(users_str) if users_str else 0
|
||||
except ValueError:
|
||||
users = 0
|
||||
try:
|
||||
users_max = int(users_max_str) if users_max_str else 0
|
||||
except ValueError:
|
||||
users_max = 0
|
||||
|
||||
nodes.append({
|
||||
"name": name[:120], # Truncate long names
|
||||
"lat": round(lat, 5),
|
||||
"lon": round(lon, 5),
|
||||
"url": url,
|
||||
"users": users,
|
||||
"users_max": users_max,
|
||||
"bands": bands,
|
||||
"antenna": antenna[:200] if antenna else "",
|
||||
"location": location[:100] if location else "",
|
||||
})
|
||||
|
||||
logger.info(f"KiwiSDR: parsed {len(nodes)} online receivers")
|
||||
return nodes
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"KiwiSDR fetch exception: {e}")
|
||||
return []
|
||||
@@ -0,0 +1,81 @@
|
||||
"""
|
||||
Sentinel-2 satellite imagery search via Microsoft Planetary Computer STAC API.
|
||||
Free, keyless search for metadata + thumbnails. Used in the right-click dossier.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from cachetools import TTLCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Cache by rounded lat/lon (0.02° grid ~= 2km), TTL 1 hour
|
||||
_sentinel_cache = TTLCache(maxsize=200, ttl=3600)
|
||||
|
||||
|
||||
def search_sentinel2_scene(lat: float, lng: float) -> dict:
|
||||
"""Search for the latest Sentinel-2 L2A scene covering a point."""
|
||||
cache_key = f"{round(lat, 2)}_{round(lng, 2)}"
|
||||
if cache_key in _sentinel_cache:
|
||||
return _sentinel_cache[cache_key]
|
||||
|
||||
try:
|
||||
from pystac_client import Client
|
||||
|
||||
catalog = Client.open("https://planetarycomputer.microsoft.com/api/stac/v1")
|
||||
end = datetime.utcnow()
|
||||
start = end - timedelta(days=30)
|
||||
|
||||
search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
intersects={"type": "Point", "coordinates": [lng, lat]},
|
||||
datetime=f"{start.isoformat()}Z/{end.isoformat()}Z",
|
||||
sortby=[{"field": "datetime", "direction": "desc"}],
|
||||
max_items=3,
|
||||
query={"eo:cloud_cover": {"lt": 30}},
|
||||
)
|
||||
|
||||
items = list(search.items())
|
||||
if not items:
|
||||
result = {"found": False, "message": "No clear scenes in last 30 days"}
|
||||
_sentinel_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
item = items[0]
|
||||
# Try to sign item first for Azure blob URLs
|
||||
try:
|
||||
import planetary_computer
|
||||
item = planetary_computer.sign_item(item)
|
||||
except ImportError:
|
||||
pass # planetary_computer not installed, try unsigned URLs
|
||||
except Exception as e:
|
||||
logger.warning(f"Sentinel-2 signing failed: {e}")
|
||||
|
||||
# Get the rendered_preview (full-res PNG) and thumbnail separately
|
||||
rendered = item.assets.get("rendered_preview")
|
||||
thumbnail = item.assets.get("thumbnail")
|
||||
|
||||
# Full-res image URL — what opens when user clicks
|
||||
fullres_url = rendered.href if rendered else (thumbnail.href if thumbnail else None)
|
||||
# Thumbnail URL — what shows in the popup card
|
||||
thumb_url = thumbnail.href if thumbnail else (rendered.href if rendered else None)
|
||||
|
||||
result = {
|
||||
"found": True,
|
||||
"scene_id": item.id,
|
||||
"datetime": item.datetime.isoformat() if item.datetime else None,
|
||||
"cloud_cover": item.properties.get("eo:cloud_cover"),
|
||||
"thumbnail_url": thumb_url,
|
||||
"fullres_url": fullres_url,
|
||||
"bbox": list(item.bbox) if item.bbox else None,
|
||||
"platform": item.properties.get("platform", "Sentinel-2"),
|
||||
}
|
||||
_sentinel_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
except ImportError:
|
||||
logger.warning("pystac-client not installed — Sentinel-2 search unavailable")
|
||||
return {"found": False, "error": "pystac-client not installed"}
|
||||
except Exception as e:
|
||||
logger.error(f"Sentinel-2 search failed for ({lat}, {lng}): {e}")
|
||||
return {"found": False, "error": str(e)}
|
||||
@@ -1,17 +0,0 @@
|
||||
import sys
|
||||
import logging
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
# Add backend directory to sys path so we can import modules
|
||||
sys.path.append(r'f:\Codebase\Oracle\live-risk-dashboard\backend')
|
||||
|
||||
from services.data_fetcher import fetch_flights, latest_data
|
||||
|
||||
print("Testing fetch_flights...")
|
||||
try:
|
||||
fetch_flights()
|
||||
print("Commercial flights count:", len(latest_data.get('commercial_flights', [])))
|
||||
print("Private jets count:", len(latest_data.get('private_jets', [])))
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
@@ -1,38 +0,0 @@
|
||||
import json
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
def scrape_liveuamap():
|
||||
print("Launching playwright...")
|
||||
with sync_playwright() as p:
|
||||
# User agents are important for headless browsing
|
||||
browser = p.chromium.launch(headless=True)
|
||||
page = browser.new_page(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
|
||||
def handle_response(response):
|
||||
try:
|
||||
if not response.url.endswith(('js', 'css', 'png', 'jpg', 'woff2', 'svg', 'ico')):
|
||||
print(f"Intercepted API Call: {response.url}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
page.on("response", handle_response)
|
||||
|
||||
print("Navigating to liveuamap...")
|
||||
try:
|
||||
page.goto("https://liveuamap.com/", timeout=30000, wait_until="domcontentloaded")
|
||||
page.wait_for_timeout(5000)
|
||||
|
||||
print("Grabbing all script tags...")
|
||||
scripts = page.evaluate("() => Array.from(document.querySelectorAll('script')).map(s => s.innerText)")
|
||||
for i, s in enumerate(scripts):
|
||||
if 'JSON.parse' in s or 'markers' in s or 'JSON' in s:
|
||||
with open(f"script_{i}.txt", "w", encoding="utf-8") as f:
|
||||
f.write(s)
|
||||
except Exception as e:
|
||||
print("Playwright timeout or error:", e)
|
||||
|
||||
print("Closing browser...")
|
||||
browser.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
scrape_liveuamap()
|
||||
@@ -1,59 +0,0 @@
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
import cloudscraper
|
||||
|
||||
def scrape_openmhz_systems():
|
||||
print("Testing OpenMHZ undocumented API with Cloudscraper...")
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
}
|
||||
|
||||
scraper = cloudscraper.create_scraper(browser={'browser': 'chrome', 'platform': 'windows', 'desktop': True})
|
||||
|
||||
try:
|
||||
# Step 1: Hit the public systems list that the front-end map uses
|
||||
res = scraper.get("https://api.openmhz.com/systems", headers=headers, timeout=15)
|
||||
json_data = res.json()
|
||||
systems = json_data.get('systems', []) if isinstance(json_data, dict) else []
|
||||
print(f"Successfully spoofed OpenMHZ frontend. Found {len(systems)} active police/fire systems.")
|
||||
|
||||
if not systems:
|
||||
return
|
||||
|
||||
# Inspect the first system (usually a major city)
|
||||
city = systems[0]
|
||||
sys_name = city.get('shortName')
|
||||
print(f"Targeting System: {city.get('name')} ({sys_name})")
|
||||
|
||||
if not sys_name:
|
||||
return
|
||||
|
||||
time.sleep(2) # Ethical delay
|
||||
|
||||
# Step 2: Query the recent calls for this specific system
|
||||
# The frontend queries: https://api.openmhz.com/<system_name>/calls
|
||||
calls_url = f"https://api.openmhz.com/{sys_name}/calls"
|
||||
print(f"Fetching recent bursts: {calls_url}")
|
||||
|
||||
call_res = scraper.get(calls_url, headers=headers, timeout=15)
|
||||
|
||||
if call_res.status_code == 200:
|
||||
call_json = call_res.json()
|
||||
calls = call_json.get('calls', []) if isinstance(call_json, dict) else []
|
||||
if calls and len(calls) > 0:
|
||||
print(f"Intercepted {len(calls)} audio bursts.")
|
||||
latest = calls[0]
|
||||
print("LATEST INTERCEPT:")
|
||||
print(f"Talkgroup: {latest.get('talkgroupNum')}")
|
||||
print(f"Audio URL: {latest.get('url')}")
|
||||
else:
|
||||
print("No recent calls found for this system.")
|
||||
else:
|
||||
print(f"Failed to fetch calls. HTTP {call_res.status_code}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Scrape Exception: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
scrape_openmhz_systems()
|
||||
@@ -1,19 +0,0 @@
|
||||
import requests
|
||||
|
||||
def test_openmhz():
|
||||
print("Testing OpenMHZ...")
|
||||
res = requests.get("https://api.openmhz.com/systems")
|
||||
if res.status_code == 200:
|
||||
data = res.json()
|
||||
print(f"OpenMHZ returned {len(data)} systems.")
|
||||
print(f"Example: {data[0]['name']} ({data[0]['shortName']})")
|
||||
else:
|
||||
print(f"OpenMHZ Failed: {res.status_code}")
|
||||
|
||||
def test_scanner_radio():
|
||||
print("Testing Scanner Radio...")
|
||||
# Gordon Edwards app often uses something like this
|
||||
# We will just try broadcastify public page scrape as a secondary fallback
|
||||
pass
|
||||
|
||||
test_openmhz()
|
||||
@@ -1,55 +0,0 @@
|
||||
import feedparser
|
||||
import requests
|
||||
import re
|
||||
|
||||
feeds = {
|
||||
"NPR": "https://feeds.npr.org/1004/rss.xml",
|
||||
"BBC": "http://feeds.bbci.co.uk/news/world/rss.xml"
|
||||
}
|
||||
|
||||
keyword_coords = {
|
||||
"venezuela": (7.119, -66.589), "brazil": (-14.235, -51.925), "argentina": (-38.416, -63.616),
|
||||
"colombia": (4.570, -74.297), "mexico": (23.634, -102.552), "united states": (38.907, -77.036),
|
||||
" usa ": (38.907, -77.036), " us ": (38.907, -77.036), "washington": (38.907, -77.036),
|
||||
"canada": (56.130, -106.346), "ukraine": (49.487, 31.272), "kyiv": (50.450, 30.523),
|
||||
"russia": (61.524, 105.318), "moscow": (55.755, 37.617), "israel": (31.046, 34.851),
|
||||
"gaza": (31.416, 34.333), "iran": (32.427, 53.688), "lebanon": (33.854, 35.862),
|
||||
"syria": (34.802, 38.996), "yemen": (15.552, 48.516), "china": (35.861, 104.195),
|
||||
"beijing": (39.904, 116.407), "taiwan": (23.697, 120.960), "north korea": (40.339, 127.510),
|
||||
"south korea": (35.907, 127.766), "pyongyang": (39.039, 125.762), "seoul": (37.566, 126.978),
|
||||
"japan": (36.204, 138.252), "afghanistan": (33.939, 67.709), "pakistan": (30.375, 69.345),
|
||||
"india": (20.593, 78.962), " uk ": (55.378, -3.435), "london": (51.507, -0.127),
|
||||
"france": (46.227, 2.213), "paris": (48.856, 2.352), "germany": (51.165, 10.451),
|
||||
"berlin": (52.520, 13.405), "sudan": (12.862, 30.217), "congo": (-4.038, 21.758),
|
||||
"south africa": (-30.559, 22.937), "nigeria": (9.082, 8.675), "egypt": (26.820, 30.802),
|
||||
"zimbabwe": (-19.015, 29.154), "australia": (-25.274, 133.775), "middle east": (31.500, 34.800),
|
||||
"europe": (48.800, 2.300), "africa": (0.000, 25.000), "america": (38.900, -77.000),
|
||||
"south america": (-14.200, -51.900), "asia": (34.000, 100.000),
|
||||
"california": (36.778, -119.417), "texas": (31.968, -99.901), "florida": (27.994, -81.760),
|
||||
"new york": (40.712, -74.006), "virginia": (37.431, -78.656),
|
||||
"british columbia": (53.726, -127.647), "ontario": (51.253, -85.323), "quebec": (52.939, -73.549),
|
||||
"delhi": (28.704, 77.102), "new delhi": (28.613, 77.209), "mumbai": (19.076, 72.877),
|
||||
"shanghai": (31.230, 121.473), "hong kong": (22.319, 114.169), "istanbul": (41.008, 28.978),
|
||||
"dubai": (25.204, 55.270), "singapore": (1.352, 103.819)
|
||||
}
|
||||
|
||||
for name, url in feeds.items():
|
||||
r = requests.get(url)
|
||||
feed = feedparser.parse(r.text)
|
||||
for entry in feed.entries[:10]:
|
||||
title = entry.get('title', '')
|
||||
summary = entry.get('summary', '')
|
||||
text = (title + " " + summary).lower()
|
||||
padded_text = f" {text} "
|
||||
|
||||
matched_kw = None
|
||||
for kw, coords in keyword_coords.items():
|
||||
if kw.startswith(" ") or kw.endswith(" "):
|
||||
if kw in padded_text:
|
||||
matched_kw = kw
|
||||
break
|
||||
else:
|
||||
if re.search(r'\b' + re.escape(kw) + r'\b', text):
|
||||
matched_kw = kw
|
||||
break
|
||||
print(f"[{name}] {title}\n Matched: {matched_kw}\n Text: {text}\n")
|
||||
@@ -1,67 +0,0 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
|
||||
def scrape_broadcastify_top():
|
||||
print("Scraping Broadcastify Top Feeds...")
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
try:
|
||||
# The top 50 feeds page provides a wealth of listening data
|
||||
res = requests.get("https://www.broadcastify.com/listen/top", headers=headers, timeout=10)
|
||||
if res.status_code != 200:
|
||||
print(f"Failed HTTP {res.status_code}")
|
||||
return []
|
||||
|
||||
soup = BeautifulSoup(res.text, 'html.parser')
|
||||
|
||||
# The table of feeds is in a standard class
|
||||
table = soup.find('table', {'class': 'btable'})
|
||||
if not table:
|
||||
print("Could not find feeds table.")
|
||||
return []
|
||||
|
||||
feeds = []
|
||||
rows = table.find_all('tr')[1:] # Skip header
|
||||
|
||||
for row in rows:
|
||||
cols = row.find_all('td')
|
||||
if len(cols) >= 5:
|
||||
# Top layout: [Listeners, Feed ID (hidden), Location, Feed Name, Category, Genre]
|
||||
listeners_str = cols[0].text.strip().replace(',', '')
|
||||
listeners = int(listeners_str) if listeners_str.isdigit() else 0
|
||||
|
||||
# The link is usually in the Feed Name column
|
||||
link_tag = cols[2].find('a')
|
||||
if not link_tag:
|
||||
continue
|
||||
|
||||
href = link_tag.get('href', '')
|
||||
feed_id = href.split('/')[-1] if '/listen/feed/' in href else None
|
||||
|
||||
if not feed_id:
|
||||
continue
|
||||
|
||||
location = cols[1].text.strip()
|
||||
name = cols[2].text.strip()
|
||||
|
||||
feeds.append({
|
||||
"id": feed_id,
|
||||
"listeners": listeners,
|
||||
"location": location,
|
||||
"name": name,
|
||||
"stream_url": f"https://broadcastify.cdnstream1.com/{feed_id}"
|
||||
})
|
||||
|
||||
print(f"Successfully scraped {len(feeds)} top feeds.")
|
||||
return feeds
|
||||
|
||||
except Exception as e:
|
||||
print(f"Scrape error: {e}")
|
||||
return []
|
||||
|
||||
if __name__ == "__main__":
|
||||
top_feeds = scrape_broadcastify_top()
|
||||
print(json.dumps(top_feeds[:3], indent=2))
|
||||
Reference in New Issue
Block a user