mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-09-24 01:40:41 +02:00
feat: real-time gate messages via SSE + faster push/pull intervals
- Add Server-Sent Events endpoint at GET /api/mesh/gate/stream that broadcasts ALL gate events to connected frontends (privacy: no per-gate subscriptions, clients filter locally) - Hook SSE broadcast into all gate event entry points: local append, peer push receiver, and pull loop - Reduce push/pull intervals from 30s to 10s for faster relay sync - Add useGateSSE hook for frontend EventSource integration - GateView + MeshChat use SSE for instant refresh, polling demoted to 30s fallback Latency: same-node instant, cross-node ~10s avg (was ~34s)
This commit is contained in:
+78
-3
@@ -694,7 +694,7 @@ def _schedule_public_event_propagation(event_dict: dict[str, Any]) -> None:
|
|||||||
# Runs alongside the sync loop. Every PUSH_INTERVAL seconds, batches new
|
# Runs alongside the sync loop. Every PUSH_INTERVAL seconds, batches new
|
||||||
# Infonet events and sends them via HMAC-authenticated POST to push peers.
|
# Infonet events and sends them via HMAC-authenticated POST to push peers.
|
||||||
|
|
||||||
_PEER_PUSH_INTERVAL_S = 30
|
_PEER_PUSH_INTERVAL_S = 10
|
||||||
_PEER_PUSH_BATCH_SIZE = 50
|
_PEER_PUSH_BATCH_SIZE = 50
|
||||||
_peer_push_last_index: dict[str, int] = {} # peer_url → last pushed event index
|
_peer_push_last_index: dict[str, int] = {} # peer_url → last pushed event index
|
||||||
|
|
||||||
@@ -782,7 +782,7 @@ def _http_peer_push_loop() -> None:
|
|||||||
# Complements the push loop: push sends OUR events to peers, pull fetches
|
# Complements the push loop: push sends OUR events to peers, pull fetches
|
||||||
# THEIR events from peers (needed when this node is behind NAT).
|
# THEIR events from peers (needed when this node is behind NAT).
|
||||||
|
|
||||||
_GATE_PULL_INTERVAL_S = 30
|
_GATE_PULL_INTERVAL_S = 10
|
||||||
_gate_pull_last_count: dict[str, dict[str, int]] = {} # peer → {gate_id → known count}
|
_gate_pull_last_count: dict[str, dict[str, int]] = {} # peer → {gate_id → known count}
|
||||||
|
|
||||||
|
|
||||||
@@ -893,6 +893,7 @@ def _http_gate_pull_loop() -> None:
|
|||||||
accepted = int(result.get("accepted", 0) or 0)
|
accepted = int(result.get("accepted", 0) or 0)
|
||||||
dups = int(result.get("duplicates", 0) or 0)
|
dups = int(result.get("duplicates", 0) or 0)
|
||||||
if accepted > 0:
|
if accepted > 0:
|
||||||
|
_broadcast_gate_events(gate_id, events[:accepted])
|
||||||
logger.info(
|
logger.info(
|
||||||
"Gate pull: %d new event(s) for %s from %s",
|
"Gate pull: %d new event(s) for %s from %s",
|
||||||
accepted, gate_id[:12], normalized[:40],
|
accepted, gate_id[:12], normalized[:40],
|
||||||
@@ -907,6 +908,35 @@ def _http_gate_pull_loop() -> None:
|
|||||||
_NODE_SYNC_STOP.wait(_GATE_PULL_INTERVAL_S)
|
_NODE_SYNC_STOP.wait(_GATE_PULL_INTERVAL_S)
|
||||||
|
|
||||||
|
|
||||||
|
# ─── SSE Gate Event Broadcast ─────────────────────────────────────────────
|
||||||
|
# All connected SSE clients receive every gate event (encrypted blobs).
|
||||||
|
# Clients filter locally by gate_id — the server never learns which gates
|
||||||
|
# a client cares about (privacy-preserving broadcast).
|
||||||
|
|
||||||
|
_gate_sse_clients: set[asyncio.Queue] = set()
|
||||||
|
_gate_sse_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _broadcast_gate_events(gate_id: str, events: list[dict]) -> None:
|
||||||
|
"""Notify all connected SSE clients about new gate events (non-blocking)."""
|
||||||
|
if not events:
|
||||||
|
return
|
||||||
|
payload = json_mod.dumps(
|
||||||
|
{"gate_id": gate_id, "count": len(events), "ts": time.time()},
|
||||||
|
separators=(",", ":"),
|
||||||
|
ensure_ascii=False,
|
||||||
|
)
|
||||||
|
with _gate_sse_lock:
|
||||||
|
dead: list[asyncio.Queue] = []
|
||||||
|
for q in _gate_sse_clients:
|
||||||
|
try:
|
||||||
|
q.put_nowait(payload)
|
||||||
|
except (asyncio.QueueFull, Exception):
|
||||||
|
dead.append(q)
|
||||||
|
for q in dead:
|
||||||
|
_gate_sse_clients.discard(q)
|
||||||
|
|
||||||
|
|
||||||
# ─── Background Gate Message Push Worker ─────────────────────────────────
|
# ─── Background Gate Message Push Worker ─────────────────────────────────
|
||||||
|
|
||||||
_gate_push_last_count: dict[str, dict[str, int]] = {} # peer → {gate_id → count}
|
_gate_push_last_count: dict[str, dict[str, int]] = {} # peer → {gate_id → count}
|
||||||
@@ -4008,6 +4038,7 @@ def _submit_gate_message_envelope(request: Request, gate_id: str, body: dict[str
|
|||||||
if reply_to:
|
if reply_to:
|
||||||
store_payload["reply_to"] = reply_to
|
store_payload["reply_to"] = reply_to
|
||||||
stored_event = gate_store.append(gate_id, gate_event)
|
stored_event = gate_store.append(gate_id, gate_event)
|
||||||
|
_broadcast_gate_events(gate_id, [gate_event])
|
||||||
chain_event_id = chain_event_id or str(stored_event.get("event_id", ""))
|
chain_event_id = chain_event_id or str(stored_event.get("event_id", ""))
|
||||||
try:
|
try:
|
||||||
from services.mesh.mesh_rns import rns_bridge
|
from services.mesh.mesh_rns import rns_bridge
|
||||||
@@ -4429,9 +4460,12 @@ async def gate_peer_push(request: Request):
|
|||||||
rejected = 0
|
rejected = 0
|
||||||
for event_gate_id, items in grouped_events.items():
|
for event_gate_id, items in grouped_events.items():
|
||||||
result = gate_store.ingest_peer_events(event_gate_id, items)
|
result = gate_store.ingest_peer_events(event_gate_id, items)
|
||||||
accepted += int(result.get("accepted", 0) or 0)
|
a = int(result.get("accepted", 0) or 0)
|
||||||
|
accepted += a
|
||||||
duplicates += int(result.get("duplicates", 0) or 0)
|
duplicates += int(result.get("duplicates", 0) or 0)
|
||||||
rejected += int(result.get("rejected", 0) or 0)
|
rejected += int(result.get("rejected", 0) or 0)
|
||||||
|
if a > 0:
|
||||||
|
_broadcast_gate_events(event_gate_id, items[:a])
|
||||||
return {"ok": True, "accepted": accepted, "duplicates": duplicates, "rejected": rejected}
|
return {"ok": True, "accepted": accepted, "duplicates": duplicates, "rejected": rejected}
|
||||||
|
|
||||||
|
|
||||||
@@ -4489,6 +4523,47 @@ async def gate_peer_pull(request: Request):
|
|||||||
return {"ok": True, "events": batch, "total": total, "gate_id": gate_id}
|
return {"ok": True, "events": batch, "total": total, "gate_id": gate_id}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# SSE Gate Event Stream — real-time push of gate activity to frontends.
|
||||||
|
# Delivers ALL gate events (encrypted blobs) to every connected client.
|
||||||
|
# The client filters locally by gate_id — the server never learns which
|
||||||
|
# gates a client cares about (privacy-preserving broadcast).
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/mesh/gate/stream")
|
||||||
|
async def gate_event_stream(request: Request):
|
||||||
|
"""SSE stream of all gate events for real-time delivery."""
|
||||||
|
client_queue: asyncio.Queue = asyncio.Queue(maxsize=256)
|
||||||
|
with _gate_sse_lock:
|
||||||
|
_gate_sse_clients.add(client_queue)
|
||||||
|
|
||||||
|
async def event_generator():
|
||||||
|
try:
|
||||||
|
yield ": connected\n\n"
|
||||||
|
while True:
|
||||||
|
if await request.is_disconnected():
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
payload = await asyncio.wait_for(client_queue.get(), timeout=15.0)
|
||||||
|
yield f"data: {payload}\n\n"
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
yield ": keepalive\n\n"
|
||||||
|
finally:
|
||||||
|
with _gate_sse_lock:
|
||||||
|
_gate_sse_clients.discard(client_queue)
|
||||||
|
|
||||||
|
return StreamingResponse(
|
||||||
|
event_generator(),
|
||||||
|
media_type="text/event-stream",
|
||||||
|
headers={
|
||||||
|
"Cache-Control": "no-cache",
|
||||||
|
"Connection": "keep-alive",
|
||||||
|
"X-Accel-Buffering": "no",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Peer Management API — operator endpoints for adding / removing / listing
|
# Peer Management API — operator endpoints for adding / removing / listing
|
||||||
# peers without editing peer_store.json by hand.
|
# peers without editing peer_store.json by hand.
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import {
|
|||||||
} from '@/mesh/wormholeIdentityClient';
|
} from '@/mesh/wormholeIdentityClient';
|
||||||
import { gateEnvelopeDisplayText, gateEnvelopeState, isEncryptedGateEnvelope } from '@/mesh/gateEnvelope';
|
import { gateEnvelopeDisplayText, gateEnvelopeState, isEncryptedGateEnvelope } from '@/mesh/gateEnvelope';
|
||||||
import { validateEventPayload } from '@/mesh/meshSchema';
|
import { validateEventPayload } from '@/mesh/meshSchema';
|
||||||
|
import { useGateSSE } from '@/hooks/useGateSSE';
|
||||||
|
|
||||||
const GATE_INTROS: Record<string, string> = {
|
const GATE_INTROS: Record<string, string> = {
|
||||||
infonet:
|
infonet:
|
||||||
@@ -357,11 +358,21 @@ export default function GateView({
|
|||||||
}
|
}
|
||||||
}, [gateId, hydrateMessages]);
|
}, [gateId, hydrateMessages]);
|
||||||
|
|
||||||
|
// SSE: instant delivery when new gate events arrive
|
||||||
|
const handleSSEEvent = useCallback(
|
||||||
|
(eventGateId: string) => {
|
||||||
|
if (eventGateId === gateId) void refreshGate();
|
||||||
|
},
|
||||||
|
[gateId, refreshGate],
|
||||||
|
);
|
||||||
|
useGateSSE(handleSSEEvent);
|
||||||
|
|
||||||
|
// Fallback poll (30s) in case SSE disconnects
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
void refreshGate();
|
void refreshGate();
|
||||||
const timer = window.setInterval(() => {
|
const timer = window.setInterval(() => {
|
||||||
void refreshGate();
|
void refreshGate();
|
||||||
}, 8000);
|
}, 30_000);
|
||||||
return () => {
|
return () => {
|
||||||
window.clearInterval(timer);
|
window.clearInterval(timer);
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import {
|
|||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import { API_BASE } from '@/lib/api';
|
import { API_BASE } from '@/lib/api';
|
||||||
import { controlPlaneJson } from '@/lib/controlPlane';
|
import { controlPlaneJson } from '@/lib/controlPlane';
|
||||||
|
import { useGateSSE } from '@/hooks/useGateSSE';
|
||||||
import { requestSecureMeshTerminalLauncherOpen } from '@/lib/meshTerminalLauncher';
|
import { requestSecureMeshTerminalLauncherOpen } from '@/lib/meshTerminalLauncher';
|
||||||
import {
|
import {
|
||||||
loadIdentityBoundSensitiveValue,
|
loadIdentityBoundSensitiveValue,
|
||||||
@@ -1111,6 +1112,17 @@ const MeshChat = React.memo(function MeshChat({
|
|||||||
const [reps, setReps] = useState<Record<string, number>>({});
|
const [reps, setReps] = useState<Record<string, number>>({});
|
||||||
const repsRef = useRef(reps);
|
const repsRef = useRef(reps);
|
||||||
const [votedOn, setVotedOn] = useState<Record<string, 1 | -1>>({});
|
const [votedOn, setVotedOn] = useState<Record<string, 1 | -1>>({});
|
||||||
|
|
||||||
|
// SSE: bump tick counter to trigger immediate re-poll on gate events
|
||||||
|
const [sseGateTick, setSseGateTick] = useState(0);
|
||||||
|
const selectedGateRef = useRef(selectedGate);
|
||||||
|
selectedGateRef.current = selectedGate;
|
||||||
|
const handleSSEGateEvent = useCallback((eventGateId: string) => {
|
||||||
|
if (eventGateId === selectedGateRef.current.trim().toLowerCase()) {
|
||||||
|
setSseGateTick((t) => t + 1);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
useGateSSE(handleSSEGateEvent);
|
||||||
const [gateReplyContext, setGateReplyContext] = useState<GateReplyContext | null>(null);
|
const [gateReplyContext, setGateReplyContext] = useState<GateReplyContext | null>(null);
|
||||||
const [showCreateGate, setShowCreateGate] = useState(false);
|
const [showCreateGate, setShowCreateGate] = useState(false);
|
||||||
const [newGateId, setNewGateId] = useState('');
|
const [newGateId, setNewGateId] = useState('');
|
||||||
@@ -1699,7 +1711,7 @@ const MeshChat = React.memo(function MeshChat({
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
poll();
|
poll();
|
||||||
const iv = setInterval(poll, 10000);
|
const iv = setInterval(poll, 30_000); // SSE handles fast path; this is fallback
|
||||||
return () => {
|
return () => {
|
||||||
cancelled = true;
|
cancelled = true;
|
||||||
clearInterval(iv);
|
clearInterval(iv);
|
||||||
@@ -1712,6 +1724,7 @@ const MeshChat = React.memo(function MeshChat({
|
|||||||
gatePersonaBusy,
|
gatePersonaBusy,
|
||||||
gatePersonaPromptOpen,
|
gatePersonaPromptOpen,
|
||||||
hydrateInfonetMessages,
|
hydrateInfonetMessages,
|
||||||
|
sseGateTick, // SSE event triggers immediate re-poll
|
||||||
]);
|
]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { useEffect, useRef } from 'react';
|
||||||
|
import { API_BASE } from '@/lib/api';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Subscribe to the backend SSE gate-event stream.
|
||||||
|
* Delivers ALL gate events (encrypted blobs) — the client filters by gate_id locally.
|
||||||
|
* The server never learns which gates a client cares about (privacy-preserving broadcast).
|
||||||
|
*
|
||||||
|
* Falls back gracefully: if the stream fails the browser's EventSource auto-reconnects.
|
||||||
|
*/
|
||||||
|
export function useGateSSE(onEvent: (gateId: string) => void) {
|
||||||
|
const callbackRef = useRef(onEvent);
|
||||||
|
callbackRef.current = onEvent;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const es = new EventSource(`${API_BASE}/api/mesh/gate/stream`);
|
||||||
|
|
||||||
|
es.onmessage = (e) => {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(e.data);
|
||||||
|
if (data.gate_id && typeof data.gate_id === 'string') {
|
||||||
|
callbackRef.current(data.gate_id);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* ignore parse errors */
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Browser auto-reconnects EventSource on error — no manual retry needed.
|
||||||
|
|
||||||
|
return () => es.close();
|
||||||
|
}, []);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user