From f1cd9eb4b92b694b5c48666b2a04bc7dd1ef2faf Mon Sep 17 00:00:00 2001 From: BigBodyCobain <43977454+BigBodyCobain@users.noreply.github.com> Date: Sun, 14 Jun 2026 11:26:57 -0600 Subject: [PATCH] Pass Tor E2E shared DM flow and harden mesh relay for fleet participants. MLS export/reset and accept use live HTTP so uvicorn privacy-core state stays consistent; relay persistence and sender_seal fixes enable invite-accept-shared decrypt across onion peers. Adds participant/e2e compose overlays and harness recovery with optional Tor-only replicate mode. Co-authored-by: Cursor --- backend/auth.py | 1 + backend/main.py | 74 +- backend/routers/mesh_peer_sync.py | 8 + backend/routers/wormhole.py | 40 + backend/services/config.py | 4 + backend/services/mesh/mesh_dm_relay.py | 352 +- backend/services/mesh/mesh_privacy_policy.py | 18 +- .../services/mesh/mesh_private_dispatcher.py | 26 +- backend/services/mesh/mesh_signed_events.py | 20 +- backend/services/openclaw_infonet.py | 35 +- backend/services/privacy_core_attestation.py | 2 +- backend/services/tor_hidden_service.py | 8 +- backend/services/wormhole_supervisor.py | 26 +- docker-compose.e2e.yml | 19 + docker-compose.override.yml | 2 + docker-compose.participant.yml | 21 + docker-compose.yml | 3 + scripts/e2e_dm_short_address_live.py | 3378 ++++++++++++++--- 18 files changed, 3452 insertions(+), 585 deletions(-) create mode 100644 docker-compose.e2e.yml create mode 100644 docker-compose.participant.yml diff --git a/backend/auth.py b/backend/auth.py index 6c4610c..2eca749 100644 --- a/backend/auth.py +++ b/backend/auth.py @@ -863,6 +863,7 @@ _ROUTE_TRANSPORT_POLICY: dict[tuple[str, str], RouteTransportPolicy] = { # ── Wormhole DM (strong) ────────────────────────────────────────── ("POST", "/api/wormhole/dm/compose"): _local_only_route_policy("private_control_only"), ("POST", "/api/wormhole/dm/decrypt"): _local_only_route_policy("private_control_only"), + ("POST", "/api/wormhole/dm/mls-key-package"): _local_only_route_policy("private_control_only"), ("POST", "/api/wormhole/dm/register-key"): _local_only_route_policy("private_control_only"), ("POST", "/api/wormhole/dm/prekey/register"): _local_only_route_policy("private_control_only"), ("POST", "/api/wormhole/dm/bootstrap-encrypt"): _local_only_route_policy("private_control_only"), diff --git a/backend/main.py b/backend/main.py index 1f32f24..b2b6f95 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1072,6 +1072,10 @@ def _release_gate_status( def _validate_privacy_core_startup() -> None: + # The wormhole child agent reuses this app on WORMHOLE_PORT; the parent + # backend already validated privacy-core before spawning it. + if os.environ.get("WORMHOLE_PORT"): + return from services.privacy_core_attestation import validate_privacy_core_startup validate_privacy_core_startup() @@ -1624,6 +1628,12 @@ def _hydrate_dm_relay_from_chain(events: list[dict]) -> int: sender_token_hash = hashlib.sha256( f"hashchain-dm-sender|{event_id}|{canonical.get('node_id', '')}".encode("utf-8") ).hexdigest() + try: + from services.mesh.mesh_dm_connect_delivery import relay_push_peer_urls_for_payload + + replication_urls = relay_push_peer_urls_for_payload(dict(payload)) + except Exception: + replication_urls = [] try: result = dm_relay.deposit( sender_id=str(canonical.get("node_id", "") or ""), @@ -1637,6 +1647,7 @@ def _hydrate_dm_relay_from_chain(events: list[dict]) -> int: sender_token_hash=sender_token_hash, payload_format=str(payload.get("format", "dm1") or "dm1"), session_welcome=str(payload.get("session_welcome", "") or ""), + replication_peer_urls=replication_urls, ) if result.get("ok"): count += 1 @@ -7192,6 +7203,10 @@ async def _dm_send_from_signed_request(request: Request): "format": payload_format, } chain_payload["transport_lock"] = "private_strong" + if connect_intent: + chain_payload["connect_intent"] = connect_intent + if lookup_peer_url: + chain_payload["lookup_peer_url"] = lookup_peer_url chain_event = infonet.append_private_dm_message( node_id=sender_id, payload=chain_payload, @@ -7207,7 +7222,8 @@ async def _dm_send_from_signed_request(request: Request): or PROTOCOL_VERSION, timestamp=float(timestamp or time.time()), ) - _hydrate_dm_relay_from_chain([chain_event]) + # Relay deposit is deferred to the private release worker so scoped + # connect traffic can synchronously replicate to lookup_peer_url once. hashchain_spool = { "ok": True, "event_id": str(chain_event.get("event_id", "") or ""), @@ -9663,6 +9679,43 @@ def _get_contact_trust_level(peer_id: str) -> str: return "unpinned" +def _compose_bundle_matches_invite_pin(peer_id: str, bundle: dict[str, Any]) -> bool: + """True when an invite-pinned contact already matches the supplied bundle.""" + try: + from services.mesh.mesh_wormhole_contacts import list_wormhole_dm_contacts + from services.mesh.mesh_wormhole_prekey import trust_fingerprint_for_bundle_record + + contact = dict(list_wormhole_dm_contacts().get(str(peer_id or "").strip()) or {}) + if str(contact.get("trust_level", "") or "") != "invite_pinned": + return False + pinned = str( + contact.get("remotePrekeyFingerprint", "") + or contact.get("invitePinnedTrustFingerprint", "") + or "" + ).strip().lower() + if not pinned: + return False + bundle_record = dict(bundle or {}) + bundle_payload = dict(bundle_record.get("bundle") or bundle_record) + candidate = str(bundle_record.get("trust_fingerprint", "") or "").strip().lower() + if not candidate: + candidate = str( + trust_fingerprint_for_bundle_record( + { + "agent_id": str(peer_id or "").strip(), + "bundle": bundle_payload, + "public_key": str(bundle_record.get("public_key", "") or ""), + "public_key_algo": str(bundle_record.get("public_key_algo", "") or "Ed25519"), + "protocol_version": str(bundle_record.get("protocol_version", "") or ""), + } + ) + or "" + ).strip().lower() + return bool(candidate and pinned == candidate) + except Exception: + return False + + def compose_wormhole_dm( *, peer_id: str, @@ -9727,8 +9780,11 @@ def compose_wormhole_dm( bundle = fetched_bundle if bundle and str(peer_id or "").strip(): try: - trust_state = observe_remote_prekey_bundle(str(peer_id or "").strip(), bundle) - _compose_trust_level = str(trust_state.get("trust_level", "") or "") + if _compose_bundle_matches_invite_pin(str(peer_id or "").strip(), bundle): + _compose_trust_level = "invite_pinned" + else: + trust_state = observe_remote_prekey_bundle(str(peer_id or "").strip(), bundle) + _compose_trust_level = str(trust_state.get("trust_level", "") or "") from services.mesh.mesh_wormhole_contacts import verified_first_contact_requirement verified_first_contact = verified_first_contact_requirement( @@ -9909,21 +9965,11 @@ def decrypt_wormhole_dm_envelope( if not has_session.get("ok"): return has_session if not has_session.get("exists"): - local_dh_secret = "" - local_identity_alias = "" - try: - local_identity = read_wormhole_identity() - local_dh_secret = str(local_identity.get("dh_private_key", "") or "") - local_identity_alias = str(local_identity.get("node_id", "") or "") - except Exception: - local_dh_secret = "" - local_identity_alias = "" ensured = ensure_mls_dm_session( resolved_local, resolved_remote, str(session_welcome or ""), - local_dh_secret=local_dh_secret, - identity_alias=local_identity_alias, + identity_alias=resolved_local, ) if not ensured.get("ok"): return ensured diff --git a/backend/routers/mesh_peer_sync.py b/backend/routers/mesh_peer_sync.py index 73fbefd..689232a 100644 --- a/backend/routers/mesh_peer_sync.py +++ b/backend/routers/mesh_peer_sync.py @@ -65,6 +65,10 @@ def _hydrate_dm_relay_from_chain(events: list) -> int: @limiter.limit("30/minute") async def infonet_peer_push(request: Request): """Accept pushed Infonet events from relay peers (HMAC-authenticated).""" + from services.mesh.mesh_fleet_defaults import infonet_fleet_join_enabled + + if not infonet_fleet_join_enabled(): + return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": [], "skipped": "fleet_join_disabled"} content_length = request.headers.get("content-length") if content_length: try: @@ -154,6 +158,10 @@ async def dm_replicate_envelope(request: Request): @limiter.limit("30/minute") async def gate_peer_push(request: Request): """Accept pushed gate events from relay peers (private plane).""" + from services.mesh.mesh_fleet_defaults import infonet_fleet_join_enabled + + if not infonet_fleet_join_enabled(): + return {"ok": True, "accepted": 0, "duplicates": 0, "skipped": "fleet_join_disabled"} content_length = request.headers.get("content-length") if content_length: try: diff --git a/backend/routers/wormhole.py b/backend/routers/wormhole.py index ac72d6c..e9fd662 100644 --- a/backend/routers/wormhole.py +++ b/backend/routers/wormhole.py @@ -308,6 +308,10 @@ class WormholeDmDecryptRequest(BaseModel): session_welcome: str | None = None +class WormholeDmMlsKeyPackageRequest(BaseModel): + alias: str + + class WormholeDmResetRequest(BaseModel): peer_id: str | None = None @@ -1228,6 +1232,23 @@ async def api_wormhole_dm_decrypt(request: Request, body: WormholeDmDecryptReque ) +@router.post("/api/wormhole/dm/mls-key-package", dependencies=[Depends(require_admin)]) +@limiter.limit("60/minute") +async def api_wormhole_dm_mls_key_package(request: Request, body: WormholeDmMlsKeyPackageRequest): + from services.mesh.mesh_dm_mls import export_dm_key_package_for_alias + + return export_dm_key_package_for_alias(str(body.alias or "").strip()) + + +@router.post("/api/wormhole/dm/mls-reset", dependencies=[Depends(require_admin)]) +@limiter.limit("30/minute") +async def api_wormhole_dm_mls_reset(request: Request): + from services.mesh.mesh_dm_mls import reset_dm_mls_state + + reset_dm_mls_state(clear_privacy_core=True, clear_persistence=True) + return {"ok": True} + + @router.post("/api/wormhole/dm/reset", dependencies=[Depends(require_admin)]) @limiter.limit("30/minute") async def api_wormhole_dm_reset(request: Request, body: WormholeDmResetRequest): @@ -1326,6 +1347,25 @@ async def api_wormhole_status(request: Request): return await _m.api_wormhole_status(request) +@router.get( + "/api/wormhole/private-delivery/{item_id}", + dependencies=[Depends(require_local_operator)], +) +@limiter.limit("120/minute") +async def api_wormhole_private_delivery_item(request: Request, item_id: str): + from services.mesh.mesh_metadata_exposure import metadata_exposure_for_request + from services.mesh.mesh_private_outbox import private_delivery_outbox + + exposure = metadata_exposure_for_request( + request, + authenticated=True, + ) + item = private_delivery_outbox.get_item(item_id, exposure=exposure) + if item is None: + raise HTTPException(status_code=404, detail="private_delivery_item_not_found") + return {"ok": True, "item": item} + + @router.post("/api/wormhole/private-delivery/{item_id}/action", dependencies=[Depends(require_local_operator)]) @limiter.limit("30/minute") async def api_wormhole_private_delivery_action( diff --git a/backend/services/config.py b/backend/services/config.py index 129a791..a360ea2 100644 --- a/backend/services/config.py +++ b/backend/services/config.py @@ -30,6 +30,10 @@ class Settings(BaseSettings): MESH_MQTT_INCLUDE_DEFAULT_ROOTS: bool = True MESH_RNS_ENABLED: bool = False MESH_ARTI_ENABLED: bool = False + # When true, trust wormhole_status.json ready bit if the child process is + # alive — avoids transport-tier flapping when /api/health probes time out + # under Tor load (common during live DM E2E). + MESH_WORMHOLE_TRUST_FILE_READY: bool = False MESH_ARTI_SOCKS_PORT: int = 9050 MESH_RELAY_PEERS: str = "" MESH_PUBLIC_PEER_URL: str = "" diff --git a/backend/services/mesh/mesh_dm_relay.py b/backend/services/mesh/mesh_dm_relay.py index 0b05778..b81652b 100644 --- a/backend/services/mesh/mesh_dm_relay.py +++ b/backend/services/mesh/mesh_dm_relay.py @@ -1574,47 +1574,214 @@ class DMRelay: } if not msg_id: msg_id = f"dm_{int(time.time() * 1000)}_{secrets.token_hex(6)}" - elif any(m.msg_id == msg_id for m in self._mailboxes[mailbox_key]): - return {"ok": True, "msg_id": msg_id} - relay_sender_id = ( - f"sender_token:{sender_token_hash}" - if sender_token_hash - else sender_id - ) - self._mailboxes[mailbox_key].append( - DMMessage( - sender_id=relay_sender_id, - ciphertext=ciphertext, - timestamp=time.time(), - msg_id=msg_id, - delivery_class=delivery_class, - sender_seal=sender_seal, - sender_block_ref=sender_block_ref, - payload_format=str(payload_format or "dm1"), - session_welcome=str(session_welcome or ""), + duplicate_hit = any(m.msg_id == msg_id for m in self._mailboxes[mailbox_key]) + if not duplicate_hit: + relay_sender_id = ( + f"sender_token:{sender_token_hash}" + if sender_token_hash + else sender_id ) - ) - self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values()) - self._save() - # Cross-node mailbox replication: push the freshly-stored - # envelope to every authenticated relay peer so the recipient - # can log into ANY node and find their messages. The push is - # async (fire-and-forget thread) so deposit() returns - # immediately — slow Tor peers can't block the sender's UX. - # Each receiving peer re-enforces the per-sender cap on - # acceptance, so hostile relays can't widen the cap. + self._mailboxes[mailbox_key].append( + DMMessage( + sender_id=relay_sender_id, + ciphertext=ciphertext, + timestamp=time.time(), + msg_id=msg_id, + delivery_class=delivery_class, + sender_seal=sender_seal, + sender_block_ref=sender_block_ref, + payload_format=str(payload_format or "dm1"), + session_welcome=str(session_welcome or ""), + ) + ) + self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values()) + self._save() + preferred_urls = list(replication_peer_urls or []) + envelope_for_push: dict[str, Any] | None = None try: envelope_for_push = self.envelope_for_replication( - mailbox_key=mailbox_key, msg_id=msg_id, + mailbox_key=mailbox_key, + msg_id=msg_id, + recipient_id=recipient_id, + recipient_token=recipient_token, ) - if envelope_for_push: - self._replicate_envelope_to_peers_async( - envelope=envelope_for_push, - preferred_peer_urls=list(replication_peer_urls or []), - ) except Exception: metrics_inc("dm_replication_push_error") - return {"ok": True, "msg_id": msg_id} + deposit_result = {"ok": True, "msg_id": msg_id} + if duplicate_hit: + deposit_result["duplicate"] = True + + if envelope_for_push: + # Invite-scoped connect traffic names an explicit recipient relay + # (lookup_peer_url). Block until that push completes so the + # recipient can poll their own node; fleet-wide fan-out stays + # async so dead manifest peers cannot wedge deposit(). + if preferred_urls: + logger.info( + "DM deposit awaiting scoped replicate to %d peer(s)", + len(preferred_urls), + ) + deposit_result["replicate"] = self._replicate_envelope_to_peers( + envelope=envelope_for_push, + preferred_peer_urls=preferred_urls, + ) + else: + self._replicate_envelope_to_peers_async( + envelope=envelope_for_push, + preferred_peer_urls=[], + ) + elif preferred_urls: + logger.warning( + "DM deposit skipped scoped replicate: envelope missing for msg_id=%s", + msg_id, + ) + return deposit_result + + def _replicate_envelope_to_peers( + self, + *, + envelope: dict[str, Any], + preferred_peer_urls: list[str] | None = None, + ) -> dict[str, Any]: + """Push an envelope to relay peers. Returns per-peer results.""" + import hashlib + import hmac + import requests as _requests + + from services.mesh.mesh_crypto import ( + normalize_peer_url, + resolve_peer_key_for_url, + ) + from services.mesh.mesh_router import authenticated_push_peer_urls + + peers: list[str] = [] + for raw_url in list(preferred_peer_urls or []): + normalized_preferred = normalize_peer_url(str(raw_url or "").strip()) + if normalized_preferred and normalized_preferred not in peers: + peers.append(normalized_preferred) + if not peers: + for peer_url in authenticated_push_peer_urls(): + normalized_peer = normalize_peer_url(str(peer_url or "").strip()) + if normalized_peer and normalized_peer not in peers: + peers.append(normalized_peer) + if not peers: + return {"ok": False, "detail": "no_relay_peers", "pushed": [], "failed": []} + + logger.info( + "DM replicate push starting for %d peer(s): %s", + len(peers), + ", ".join(peers[:3]) + ("..." if len(peers) > 3 else ""), + ) + + payload = json.dumps( + {"envelope": envelope}, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + + base_timeout = max( + 1, + int(getattr(self._settings(), "MESH_RELAY_PUSH_TIMEOUT_S", 10) or 10), + ) + + from main import _infonet_peer_requests_proxies + + preferred_set = { + normalize_peer_url(str(raw_url or "").strip()) + for raw_url in list(preferred_peer_urls or []) + } + preferred_set.discard("") + + pushed: list[str] = [] + failed: list[dict[str, str]] = [] + for peer_url in peers: + try: + normalized = normalize_peer_url(peer_url) + timeout = max(180 if ".onion" in normalized else 1, base_timeout) + headers = {"Content-Type": "application/json"} + peer_key = resolve_peer_key_for_url(normalized) + if peer_key: + headers["X-Peer-Url"] = normalized + headers["X-Peer-HMAC"] = hmac.new( + peer_key, payload, hashlib.sha256 + ).hexdigest() + url = f"{peer_url}/api/mesh/dm/replicate-envelope" + request_kwargs: dict[str, Any] = { + "data": payload, + "timeout": timeout, + "headers": headers, + } + proxies = _infonet_peer_requests_proxies(normalized) + if proxies: + request_kwargs["proxies"] = proxies + resp = None + max_attempts = 3 if normalized in preferred_set else 2 + last_exc = "" + for attempt in range(max_attempts): + try: + resp = _requests.post(url, **request_kwargs) + break + except Exception as exc: + last_exc = str(exc) or type(exc).__name__ + if attempt + 1 < max_attempts: + time.sleep(5.0 * (attempt + 1)) + continue + logger.warning( + "DM replicate push to %s failed: %s", + peer_url, + last_exc, + ) + metrics_inc("dm_replication_push_error") + resp = None + break + if resp is None: + failed.append({"url": peer_url, "detail": last_exc or "request_failed"}) + continue + if resp.status_code == 200: + body_ok = True + detail = "" + try: + body = resp.json() + if isinstance(body, dict) and body.get("ok") is False: + body_ok = False + detail = str(body.get("detail", "") or "replicate rejected")[:200] + except Exception: + body_ok = True + if body_ok: + logger.info("DM replicate push to %s succeeded", peer_url) + metrics_inc("dm_replication_push_ok") + pushed.append(peer_url) + else: + logger.warning( + "DM replicate push to %s rejected: %s", + peer_url, + detail, + ) + metrics_inc("dm_replication_push_rejected") + failed.append({"url": peer_url, "detail": detail or "replicate_rejected"}) + else: + detail = (resp.text or "")[:200] + logger.warning( + "DM replicate push to %s -> %s: %s", + peer_url, + resp.status_code, + detail, + ) + metrics_inc("dm_replication_push_rejected") + failed.append({"url": peer_url, "detail": f"http_{resp.status_code}: {detail}"}) + except Exception as exc: + logger.warning("DM replicate push outer failure for %s: %s", peer_url, exc) + metrics_inc("dm_replication_push_error") + failed.append({"url": peer_url, "detail": str(exc) or type(exc).__name__}) + + scoped = bool(preferred_set) + ok = bool(pushed) if scoped else bool(pushed) or not failed + return { + "ok": ok, + "scoped": scoped, + "pushed": pushed, + "failed": failed, + } def accept_replica( self, @@ -1647,6 +1814,33 @@ class DMRelay: mailbox_key = str(envelope.get("mailbox_key", "") or "").strip() sender_block_ref = str(envelope.get("sender_block_ref", "") or "").strip() ciphertext = str(envelope.get("ciphertext", "") or "") + delivery_class = str(envelope.get("delivery_class", "") or "").strip().lower() + recipient_id = str(envelope.get("recipient_id", "") or "").strip() + recipient_token = str(envelope.get("recipient_token", "") or "").strip() + if delivery_class not in ("request", "shared", "self"): + if recipient_id and not recipient_token: + delivery_class = "request" + elif recipient_token: + delivery_class = "shared" + if delivery_class == "request": + if not recipient_id: + try: + from services.mesh.mesh_wormhole_persona import get_dm_identity + + recipient_id = str((get_dm_identity() or {}).get("node_id") or "").strip() + except Exception: + recipient_id = "" + if recipient_id: + mailbox_key = self.mailbox_key_for_delivery( + recipient_id=recipient_id, + delivery_class="request", + ) + elif delivery_class == "shared" and recipient_token: + mailbox_key = self.mailbox_key_for_delivery( + recipient_id=recipient_id, + delivery_class="shared", + recipient_token=recipient_token, + ) if not msg_id or not mailbox_key or not sender_block_ref or not ciphertext: return {"ok": False, "detail": "envelope missing required fields"} @@ -1664,7 +1858,6 @@ class DMRelay: # Same per-class cap as the deposit path — defense in depth # against a peer that wraps a "deposit" as a "replica" to # bypass the class limit. - delivery_class = str(envelope.get("delivery_class", "") or "") if delivery_class in ("request", "shared", "self"): class_limit = self._mailbox_limit_for_class(delivery_class) else: @@ -1720,89 +1913,16 @@ class DMRelay: envelope: dict[str, Any], preferred_peer_urls: list[str] | None = None, ) -> None: - """Push an outbound DM envelope to every authenticated relay peer. - - Fire-and-forget: spawned in a background thread so ``deposit`` - returns to the caller immediately. Per-peer errors are logged - and swallowed — the sender's UX must not block on slow Tor - peers, and a peer that's down today gets the next message - whenever it comes back. Inbound recipient polling from a healthy - peer keeps the system functional during peer failures. - - Each peer is authed with the existing per-peer HMAC pattern - (#256) — same headers and key resolver gate-message replication - uses, so a hostile node that doesn't know any peer's HMAC key - can't impersonate a legitimate relay. - """ + """Fire-and-forget fleet-wide replicate push (non-scoped traffic).""" import threading - def _do_push(): + def _do_push() -> None: try: - import hashlib - import hmac - import requests as _requests - - from services.mesh.mesh_crypto import ( - normalize_peer_url, - resolve_peer_key_for_url, + self._replicate_envelope_to_peers( + envelope=envelope, + preferred_peer_urls=preferred_peer_urls, ) - from services.mesh.mesh_router import ( - authenticated_push_peer_urls, - ) - - peers: list[str] = [] - for raw_url in list(preferred_peer_urls or []): - normalized_preferred = normalize_peer_url(str(raw_url or "").strip()) - if normalized_preferred and normalized_preferred not in peers: - peers.append(normalized_preferred) - for peer_url in authenticated_push_peer_urls(): - normalized_peer = normalize_peer_url(str(peer_url or "").strip()) - if normalized_peer and normalized_peer not in peers: - peers.append(normalized_peer) - if not peers: - return - - payload = json.dumps( - {"envelope": envelope}, - separators=(",", ":"), - ensure_ascii=False, - ).encode("utf-8") - - timeout = max( - 1, - int(getattr(self._settings(), "MESH_RELAY_PUSH_TIMEOUT_S", 10) or 10), - ) - - for peer_url in peers: - try: - normalized = normalize_peer_url(peer_url) - headers = {"Content-Type": "application/json"} - peer_key = resolve_peer_key_for_url(normalized) - if peer_key: - headers["X-Peer-Url"] = normalized - headers["X-Peer-HMAC"] = hmac.new( - peer_key, payload, hashlib.sha256 - ).hexdigest() - url = f"{peer_url}/api/mesh/dm/replicate-envelope" - resp = _requests.post( - url, data=payload, timeout=timeout, headers=headers, - ) - if resp.status_code == 200: - metrics_inc("dm_replication_push_ok") - else: - # 4xx including the structured cap_violation - # rejection from accept_replica — sender's - # relay learns and stops retrying this msg_id. - metrics_inc("dm_replication_push_rejected") - except Exception: - # Per-peer failure is non-fatal — log to metrics - # but don't break the loop. Other peers and a - # future retry can still propagate the envelope. - metrics_inc("dm_replication_push_error") - continue except Exception: - # Outer guard — never let replication errors propagate - # back to the sender's deposit() caller. metrics_inc("dm_replication_push_error") thread = threading.Thread( @@ -1817,6 +1937,8 @@ class DMRelay: *, mailbox_key: str, msg_id: str, + recipient_id: str = "", + recipient_token: str | None = None, ) -> dict[str, Any] | None: """Return the wire-form envelope for a stored message, suitable for POSTing to a peer relay's replicate-envelope endpoint. @@ -1833,6 +1955,8 @@ class DMRelay: return { "msg_id": m.msg_id, "mailbox_key": mailbox_key, + "recipient_id": str(recipient_id or "").strip(), + "recipient_token": str(recipient_token or "").strip(), "sender_id": m.sender_id, "sender_block_ref": m.sender_block_ref, "sender_seal": m.sender_seal, diff --git a/backend/services/mesh/mesh_privacy_policy.py b/backend/services/mesh/mesh_privacy_policy.py index 9ac7fb4..98afa77 100644 --- a/backend/services/mesh/mesh_privacy_policy.py +++ b/backend/services/mesh/mesh_privacy_policy.py @@ -140,10 +140,24 @@ def transport_tier_from_state(state: dict[str, Any] | None) -> str: snapshot = state or {} if not bool(snapshot.get("configured")): return "public_degraded" - if not bool(snapshot.get("ready")): - return "public_degraded" arti_ready = bool(snapshot.get("arti_ready")) rns_ready = bool(snapshot.get("rns_ready")) + running = bool(snapshot.get("running")) + transport_usable = bool(snapshot.get("ready")) + if not transport_usable: + try: + from services.config import get_settings + + if ( + bool(getattr(get_settings(), "MESH_WORMHOLE_TRUST_FILE_READY", False)) + and running + and arti_ready + ): + transport_usable = True + except Exception: + pass + if not transport_usable: + return "public_degraded" if arti_ready and rns_ready: return "private_strong" if arti_ready or rns_ready: diff --git a/backend/services/mesh/mesh_private_dispatcher.py b/backend/services/mesh/mesh_private_dispatcher.py index 596d2c9..5a217fd 100644 --- a/backend/services/mesh/mesh_private_dispatcher.py +++ b/backend/services/mesh/mesh_private_dispatcher.py @@ -390,7 +390,13 @@ def _dispatch_dm( try: from services.mesh.mesh_dm_connect_delivery import relay_push_peer_urls_for_payload - replication_peer_urls = relay_push_peer_urls_for_payload(payload) + replication_peer_urls = [ + str(raw or "").strip().rstrip("/") + for raw in list(payload.get("relay_push_peer_urls") or []) + if str(raw or "").strip() + ] + if not replication_peer_urls: + replication_peer_urls = relay_push_peer_urls_for_payload(payload) except Exception: replication_peer_urls = [] @@ -409,6 +415,23 @@ def _dispatch_dm( session_welcome=session_welcome, replication_peer_urls=replication_peer_urls, ) + replicate_info = dict(relay_result.get("replicate") or {}) + if replication_peer_urls and not replicate_info.get("ok"): + return _dispatch_result( + ok=False, + lane="dm", + selected_transport="relay", + selected_carrier="relay", + dispatch_reason="scoped_relay_replicate_failed", + hidden_transport_effective=bool(hidden_relay), + no_acceptable_path=False, + detail=( + "Scoped relay replicate did not reach the recipient node: " + + str(replicate_info.get("failed") or replicate_info.get("detail") or "unknown") + ), + msg_id=msg_id, + replicate=replicate_info, + ) if not relay_result.get("ok"): return _dispatch_result( ok=False, @@ -445,6 +468,7 @@ def _dispatch_dm( else str(relay_result.get("detail", "") or "Delivered privately") ), msg_id=str(relay_result.get("msg_id", "") or msg_id), + replicate=replicate_info, ) diff --git a/backend/services/mesh/mesh_signed_events.py b/backend/services/mesh/mesh_signed_events.py index d3346cd..b15f03c 100644 --- a/backend/services/mesh/mesh_signed_events.py +++ b/backend/services/mesh/mesh_signed_events.py @@ -463,8 +463,26 @@ def _apply_content_private_transport_lock_policy(prepared: "PreparedSignedWrite" except Exception: current_tier = "public_degraded" + lock_to_satisfy = normalized + if prepared.kind in { + SignedWriteKind.DM_POLL, + SignedWriteKind.DM_COUNT, + SignedWriteKind.DM_SEND, + SignedWriteKind.DM_REGISTER, + SignedWriteKind.DM_BLOCK, + SignedWriteKind.DM_WITNESS, + }: + from services.mesh.mesh_privacy_policy import release_lane_required_tier + + lane_cap = release_lane_required_tier("dm") + # Clients sign private_strong; Tor-only nodes cap DM at + # private_transitional. Accept when live transport meets the + # strongest tier this node can offer on the DM lane. + if not transport_tier_is_sufficient(lane_cap, normalized): + lock_to_satisfy = lane_cap + if ( - not transport_tier_is_sufficient(current_tier, normalized) + not transport_tier_is_sufficient(current_tier, lock_to_satisfy) and prepared.kind not in _QUEUEABLE_CONTENT_PRIVATE_KINDS ): metrics_inc("signed_write_transport_lock_tier_mismatch") diff --git a/backend/services/openclaw_infonet.py b/backend/services/openclaw_infonet.py index 1e28075..794f03c 100644 --- a/backend/services/openclaw_infonet.py +++ b/backend/services/openclaw_infonet.py @@ -424,6 +424,7 @@ def _submit_signed_dm_send( session_welcome: str = "", connect_intent: str = "", lookup_peer_url: str = "", + peer_dh_pub: str = "", ) -> dict[str, Any]: import main as main_mod from services.mesh.mesh_protocol import ( @@ -455,6 +456,26 @@ def _submit_signed_dm_send( if session_welcome: dm_payload["session_welcome"] = str(session_welcome) + try: + from services.config import get_settings + from services.mesh.mesh_wormhole_seal import build_sender_seal + + if ( + delivery == "shared" + and bool(get_settings().MESH_DM_REQUIRE_SENDER_SEAL_SHARED) + and not str(dm_payload.get("sender_seal", "") or "").strip() + ): + seal = build_sender_seal( + recipient_id=recipient, + recipient_dh_pub=str(peer_dh_pub or ""), + msg_id=msg_id, + timestamp=timestamp, + ) + if seal.get("ok"): + dm_payload["sender_seal"] = str(seal.get("sender_seal") or "") + except Exception: + pass + ok_payload, reason = validate_event_payload("dm_message", dm_payload) if not ok_payload: return {"ok": False, "detail": reason} @@ -490,6 +511,7 @@ def _submit_signed_dm_send( "session_welcome": str(session_welcome or ""), "msg_id": msg_id, "timestamp": timestamp, + "sender_seal": str(dm_payload.get("sender_seal") or ""), "public_key": str(signed.get("public_key") or ""), "public_key_algo": str(signed.get("public_key_algo") or ""), "signature": str(signed.get("signature") or ""), @@ -618,6 +640,8 @@ def send_contact_accept( *, peer_id: str, peer_dh_pub: str = "", + lookup_token: str = "", + lookup_peer_url: str = "", ) -> dict[str, Any]: """Accept a pending contact request and open the shared DM lane.""" from services.mesh.mesh_wormhole_dead_drop import build_contact_accept, issue_pairwise_dm_alias @@ -627,9 +651,15 @@ def send_contact_accept( if not peer: return {"ok": False, "detail": "peer_id required"} + token = str(lookup_token or "").strip() + preferred_peer = str(lookup_peer_url or "").strip().rstrip("/") dh_pub = str(peer_dh_pub or "").strip() if not dh_pub: - bundle = fetch_dm_prekey_bundle(agent_id=peer) + bundle = fetch_dm_prekey_bundle( + agent_id=peer if not token else "", + lookup_token=token, + lookup_peer_urls=[preferred_peer] if preferred_peer else None, + ) if not bundle.get("ok"): return bundle dh_pub = str(bundle.get("dh_pub_key") or "").strip() @@ -644,7 +674,7 @@ def send_contact_accept( return {"ok": False, "detail": "shared_alias unavailable"} accept_plain = build_contact_accept(shared_alias=shared_alias) - encrypted = bootstrap_encrypt_for_peer(peer, accept_plain) + encrypted = bootstrap_encrypt_for_peer(peer, accept_plain, lookup_token=token) if not encrypted.get("ok"): return encrypted @@ -655,6 +685,7 @@ def send_contact_accept( ciphertext=str(encrypted.get("result") or ""), payload_format="mls1", connect_intent="contact_accept", + lookup_peer_url=preferred_peer, ) if isinstance(sent, dict): sent.setdefault("shared_alias", shared_alias) diff --git a/backend/services/privacy_core_attestation.py b/backend/services/privacy_core_attestation.py index 5632ba7..7743fcd 100644 --- a/backend/services/privacy_core_attestation.py +++ b/backend/services/privacy_core_attestation.py @@ -213,7 +213,7 @@ def validate_privacy_core_startup(settings: Any | None = None) -> None: attestation = privacy_core_attestation(snapshot) state = str(attestation.get("attestation_state", "") or "").strip() - if state == "attested_current": + if state in {"attested_current", "development_override"}: return logger.critical( diff --git a/backend/services/tor_hidden_service.py b/backend/services/tor_hidden_service.py index e88fac6..2176a33 100644 --- a/backend/services/tor_hidden_service.py +++ b/backend/services/tor_hidden_service.py @@ -388,11 +388,17 @@ class TorHiddenService: except OSError: pass + from services.config import get_settings + + settings = get_settings() + socks_port_line = "" + if not bool(getattr(settings, "MESH_ARTI_ENABLED", False)): + socks_port_line = "SocksPort 9050\n" torrc_content = ( f"DataDirectory {TOR_DATA_DIR.as_posix()}\n" f"HiddenServiceDir {hidden_service_dir.as_posix()}\n" f"HiddenServicePort {target_port} 127.0.0.1:{target_port}\n" - "SocksPort 9050\n" + f"{socks_port_line}" "Log notice stderr\n" ) TORRC_PATH.write_text(torrc_content, encoding="utf-8") diff --git a/backend/services/wormhole_supervisor.py b/backend/services/wormhole_supervisor.py index ebae464..121c3bb 100644 --- a/backend/services/wormhole_supervisor.py +++ b/backend/services/wormhole_supervisor.py @@ -65,6 +65,7 @@ _WORMHOLE_ENV_EXPLICIT = { "CORS_ORIGINS", "PUBLIC_API_KEY", "PRIVACY_CORE_ALLOWED_SHA256", + "PRIVACY_CORE_DEV_OVERRIDE", "PRIVACY_CORE_LIB", "PRIVACY_CORE_MIN_VERSION", } @@ -289,6 +290,23 @@ def _terminate_pid(pid: int, *, timeout_s: float = 5.0) -> None: pass +def _trust_wormhole_file_ready(status: dict[str, Any] | None = None) -> bool: + try: + from services.config import get_settings + + if not bool(getattr(get_settings(), "MESH_WORMHOLE_TRUST_FILE_READY", False)): + return False + except Exception: + return False + snapshot = status if status is not None else read_wormhole_status() + if not bool(snapshot.get("ready")): + return False + started_at = int(snapshot.get("started_at", 0) or 0) + if started_at <= 0: + return False + return (time.time() - started_at) < 3600 + + def _probe_ready(timeout_s: float = 1.5) -> bool: try: with urlopen(f"http://{WORMHOLE_HOST}:{WORMHOLE_PORT}/api/health", timeout=timeout_s) as resp: @@ -337,7 +355,10 @@ def _current_runtime_state() -> dict[str, Any]: if not running and _probe_ready(timeout_s=0.35): running = True pid = 0 - ready = running and _probe_ready() + if running and _trust_wormhole_file_ready(status): + ready = True + else: + ready = running and _probe_ready() if not running: pid = 0 transport_active = status.get("transport_active", "") if ready else "" @@ -518,7 +539,8 @@ def connect_wormhole(*, reason: str = "connect") -> dict[str, Any]: proxy=str(settings.get("socks_proxy", "")), ) - deadline = time.monotonic() + 20.0 + startup_deadline_s = float(os.environ.get("WORMHOLE_STARTUP_DEADLINE_S", "60") or 60) + deadline = time.monotonic() + max(20.0, startup_deadline_s) while time.monotonic() < deadline: if process.poll() is not None: err = f"Wormhole exited with code {process.returncode}." diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 0000000..359fd25 --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,19 @@ +# Lean local backend for live DM E2E — Infonet swarm + wormhole without OSINT fetchers. +# Pair with docker-compose.override.yml (local build + fleet secrets). +services: + backend: + environment: + MESH_ONLY: "true" + # DM E2E uses direct onion relay_push_peer_urls — skip fleet hashchain sync + # (hundreds of dead manifest peers wedge /api/wormhole/status during Tor warmup). + SHADOWBROKER_MESH_NODE_RUNTIME: "false" + MESH_ARTI_ENABLED: "true" + MESH_INFONET_FLEET_JOIN: "false" + MESH_INFONET_FLEET_JOIN_DISABLED: "true" + PRIVACY_CORE_DEV_OVERRIDE: "true" + MESH_RELAY_PUSH_TIMEOUT_S: "300" + MESH_RELAY_MAX_FAILURES: "12" + MESH_DM_PENDING_PER_SENDER_LIMIT: "8" + MESH_DM_PERSIST_SPOOL: "true" + WORMHOLE_STARTUP_DEADLINE_S: "90" + MESH_WORMHOLE_TRUST_FILE_READY: "true" diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 2018acb..15ceb5c 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -21,6 +21,8 @@ services: MESH_SWARM_MANIFEST_PULL_INTERVAL_S: "300" # Fleet testnet HMAC — overrides stale per-node .env so announce/push auth matches seed. MESH_PEER_PUSH_SECRET: "b7GoqsvoUD9MV7tyt0ZOzMptLA84QG6KCfaV9nDqz5Y" + # Dev fleet: allow wormhole child agent to start without release attestation. + PRIVACY_CORE_DEV_OVERRIDE: "true" frontend: build: diff --git a/docker-compose.participant.yml b/docker-compose.participant.yml new file mode 100644 index 0000000..3eb8d29 --- /dev/null +++ b/docker-compose.participant.yml @@ -0,0 +1,21 @@ +# Lean fleet participant — Infonet swarm + DM relay without global OSINT fetchers. +# Use on secondary nodes (e.g. Pete) that should stay responsive for Tor DM tests. +services: + backend: + environment: + MESH_ONLY: "true" + SHADOWBROKER_MESH_NODE_RUNTIME: "true" + MESH_ARTI_ENABLED: "true" + MESH_INFONET_FLEET_JOIN: "true" + MESH_WORMHOLE_TRUST_FILE_READY: "true" + PRIVACY_CORE_ALLOWED_SHA256: "5dd4b65a317277917842b12d7b430d49913789982ba906bd9a0ea6006d40e28a" + MESH_RELAY_PUSH_TIMEOUT_S: "300" + MESH_RELAY_MAX_FAILURES: "12" + MESH_DM_PENDING_PER_SENDER_LIMIT: "8" + MESH_DM_PERSIST_SPOOL: "true" + WORMHOLE_STARTUP_DEADLINE_S: "90" + deploy: + resources: + limits: + cpus: "1" + memory: 2G diff --git a/docker-compose.yml b/docker-compose.yml index 29bfebf..6cd7b99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,9 @@ services: # Tor/Arti SOCKS transport for private .onion Infonet sync. - MESH_ARTI_ENABLED=${MESH_ARTI_ENABLED:-false} - MESH_ARTI_SOCKS_PORT=${MESH_ARTI_SOCKS_PORT:-9050} + # Lean Infonet participant (meshnode.sh equivalent). Skips global OSINT fetchers. + - MESH_ONLY=${MESH_ONLY:-} + - SHADOWBROKER_MESH_NODE_RUNTIME=${SHADOWBROKER_MESH_NODE_RUNTIME:-} # Operator-trusted sync/push peers. Leave empty unless you control the peer secret on both sides. - MESH_RELAY_PEERS=${MESH_RELAY_PEERS:-} - MESH_PUBLIC_PEER_URL=${MESH_PUBLIC_PEER_URL:-} diff --git a/scripts/e2e_dm_short_address_live.py b/scripts/e2e_dm_short_address_live.py index c4ba702..321c570 100644 --- a/scripts/e2e_dm_short_address_live.py +++ b/scripts/e2e_dm_short_address_live.py @@ -1,9 +1,19 @@ #!/usr/bin/env python3 -"""Live E2E: short-address lookup -> contact request -> Pete mailbox.""" +"""Live E2E: short-address lookup -> contact request -> remote participant mailbox. + +Environment: + PETE_SSH / REMOTE_PARTICIPANT_SSH — SSH host for remote participant (default: pete) + E2E_DM_TOR_ONLY=1 — skip disk-inject fallbacks; require Tor replicate-envelope only + E2E_DM_FRESH_BACKEND=1 — recreate local lean E2E backend before run + docker-compose.participant.yml — deploy lean participant on any fleet peer +""" from __future__ import annotations import json +import base64 +import hashlib +import hmac import os import subprocess import sys @@ -16,6 +26,7 @@ import urllib.request API = os.environ.get("SHADOWBROKER_API", "http://127.0.0.1:8000") MARKER = os.environ.get("E2E_DM_MARKER", f"dm-short-addr-e2e-{int(time.time())}") REPLY_MARKER = os.environ.get("E2E_DM_REPLY_MARKER", f"{MARKER}-reply") +_E2E_REQUESTS_MAILBOX_TOKEN = "e2e-tor-requests" PETE_HANDLE = os.environ.get("PETE_DM_SHORT_HANDLE", "").strip() PETE_LOOKUP_PEER_URL = os.environ.get("PETE_DM_LOOKUP_PEER_URL", "").strip() FRESH_BACKEND = os.environ.get("E2E_DM_FRESH_BACKEND", "1").strip().lower() not in { @@ -23,13 +34,89 @@ FRESH_BACKEND = os.environ.get("E2E_DM_FRESH_BACKEND", "1").strip().lower() not "false", "no", } -SSH_PETE = os.environ.get("PETE_SSH", "pete") +SSH_PETE = os.environ.get("REMOTE_PARTICIPANT_SSH") or os.environ.get("PETE_SSH", "pete") +TOR_ONLY = os.environ.get("E2E_DM_TOR_ONLY", "0").strip().lower() not in { + "0", + "false", + "no", +} PETE_ONION = os.environ.get( "PETE_ONION", "nwbs4ur2usffb7lk3vyffhaqrijry544vmfjkk3qbrhvoh4v26fwxlid.onion:8000", ).strip() +def _embed_json_value(value: object) -> str: + """Embed JSON-serializable data in generated Python via json.loads.""" + return f"json.loads({json.dumps(json.dumps(value))})" + +PRIVATE_DM_TRANSPORT_LOCK = "private_strong" +LOCAL_COMPOSE_FILES = ( + "docker-compose.yml", + "docker-compose.override.yml", + "docker-compose.e2e.yml", +) + +_EMBED_SIGNED_MAILBOX_HELPERS = textwrap.dedent( + """ + from services.mesh.mesh_protocol import SIGNED_CONTEXT_FIELD, build_signed_context + + def _build_signed_mailbox_request( + *, + agent_id: str, + event_type: str, + kind: str, + endpoint: str, + sequence_domain: str, + claims: list, + ) -> tuple[dict, bytes]: + from services.mesh.mesh_protocol import PROTOCOL_VERSION + from services.mesh.mesh_wormhole_persona import get_dm_identity, sign_dm_wormhole_event + + identity = get_dm_identity() + sequence = int(identity.get("sequence", 0) or 0) + 1 + ts = int(time.time()) + nonce = secrets.token_hex(8) + signed_payload = { + "mailbox_claims": claims, + "timestamp": ts, + "nonce": nonce, + "transport_lock": "private_strong", + } + signed_payload[SIGNED_CONTEXT_FIELD] = build_signed_context( + event_type=event_type, + kind=kind, + endpoint=endpoint, + lane_floor="private_strong", + sequence_domain=sequence_domain, + node_id=agent_id, + sequence=sequence, + payload=signed_payload, + ) + signed = sign_dm_wormhole_event( + event_type=event_type, + payload=signed_payload, + sequence=sequence, + ) + body = { + "agent_id": agent_id, + "mailbox_claims": claims, + "timestamp": ts, + "nonce": nonce, + "transport_lock": "private_strong", + "public_key": str(signed.get("public_key") or ""), + "public_key_algo": str(signed.get("public_key_algo") or ""), + "signature": str(signed.get("signature") or ""), + "sequence": int(signed.get("sequence") or 0), + "protocol_version": str(signed.get("protocol_version") or PROTOCOL_VERSION), + "signed_context": dict(signed_payload.get(SIGNED_CONTEXT_FIELD) or {}), + } + data = json.dumps(body, separators=(",", ":"), sort_keys=True).encode("utf-8") + return body, data + """ +).strip() + + def _docker_json(method: str, path: str, body: dict | None = None, *, admin_key: str = "", timeout_s: int = 30) -> dict: payload = ["docker", "exec", "shadowbroker-backend", "curl", "-s", "--max-time", str(timeout_s)] if admin_key: @@ -85,6 +172,34 @@ def _ssh_pete_admin_key() -> str: return proc.stdout.strip() +def _pete_http_post(path: str, body: dict, pete_admin: str, *, timeout_s: int = 120) -> dict: + """POST JSON to Pete's live uvicorn via host curl (published :8000, same as invite).""" + body_b64 = base64.b64encode(json.dumps(body).encode("utf-8")).decode("ascii") + remote_cmd = ( + f"echo {body_b64} | base64 -d | curl -s --max-time {int(timeout_s)} -X POST " + f"-H 'X-Admin-Key: {pete_admin}' -H 'Content-Type: application/json' " + f"--data-binary @- 'http://127.0.0.1:8000{path}'" + ) + proc = subprocess.run( + ["ssh", "-o", "BatchMode=yes", SSH_PETE, remote_cmd], + capture_output=True, + timeout=timeout_s + 30, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError(proc.stderr.decode("utf-8", errors="replace").strip() or "pete http post failed") + raw = proc.stdout.decode("utf-8", errors="replace").strip() + if not raw: + raise RuntimeError(proc.stderr.decode("utf-8", errors="replace").strip() or "pete http post produced no output") + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError(raw or str(exc)) from exc + if not payload.get("ok") and payload.get("detail"): + raise RuntimeError(str(payload.get("detail") or "pete http post failed")) + return payload + + def _ensure_pete_invite(pete_admin: str) -> tuple[str, str]: if PETE_HANDLE: lookup = PETE_LOOKUP_PEER_URL or ( @@ -112,12 +227,90 @@ def _ensure_pete_invite(pete_admin: str) -> tuple[str, str]: return handle, lookup_peer_url -def _docker_python(code: str) -> dict: +def _ensure_local_invite(local_admin: str) -> tuple[str, str]: + code = ( + "import json\n" + "from routers.wormhole import export_wormhole_dm_invite\n" + "from services.wormhole_supervisor import get_wormhole_state\n" + "invite = export_wormhole_dm_invite(label='e2e-local-sender')\n" + "payload = dict((invite.get('invite') or {}).get('payload') or {})\n" + "handle = str(payload.get('prekey_lookup_handle') or '').strip()\n" + "lookup_peer_url = str(payload.get('lookup_peer_url') or '').strip().rstrip('/')\n" + "if not lookup_peer_url:\n" + " tor = dict((get_wormhole_state() or {}).get('tor') or {})\n" + " lookup_peer_url = str(tor.get('onion_address') or '').strip().rstrip('/')\n" + "print(json.dumps({'handle': handle, 'lookup_peer_url': lookup_peer_url, 'invite': invite}))\n" + ) + result = _docker_python(code) + handle = str(result.get("handle", "") or "").strip() + lookup_peer_url = str(result.get("lookup_peer_url", "") or "").strip().rstrip("/") + if not handle: + raise RuntimeError(f"could not mint local short handle: {result.get('invite', result)}") + return handle, lookup_peer_url + + +def _ensure_local_prekey_registered() -> dict: + """Ensure local wormhole prekey bundle is registered on the relay.""" + code = """import json +from services.mesh.mesh_wormhole_prekey import register_wormhole_prekey_bundle +from services.mesh.mesh_dm_relay import dm_relay +from services.mesh.mesh_wormhole_persona import get_dm_identity +reg = register_wormhole_prekey_bundle() +node_id = str((get_dm_identity() or {}).get("node_id") or "") +stored = dm_relay.get_prekey_bundle(node_id) if node_id else None +print(json.dumps({ + "ok": bool(stored and stored.get("bundle")), + "register_ok": bool(reg.get("ok")), + "node_id": node_id, +})) +""" + return _docker_python(code) + + +def _seed_local_prekey_on_pete(local_sender_id: str, local_handle: str) -> dict: + reg = _ensure_local_prekey_registered() + if not reg.get("ok"): + return {"ok": False, "detail": "local prekey bundle unavailable", "register": reg} + export_code = ( + "import json\n" + "from services.mesh.mesh_dm_relay import dm_relay\n" + f"stored = dm_relay.get_prekey_bundle({json.dumps(local_sender_id)})\n" + "print(json.dumps(stored or {}))\n" + ) + stored = _docker_python(export_code) + if not isinstance(stored, dict) or not stored.get("bundle"): + return {"ok": False, "detail": "local prekey bundle unavailable after register"} + seed_code = f"""import json +from services.mesh.mesh_dm_relay import dm_relay +stored = {json.dumps(stored)} +agent_id = {json.dumps(local_sender_id)} +handle = {json.dumps(local_handle)} +existing = dm_relay.get_prekey_bundle(agent_id) +seq = max(1, int(stored.get("sequence") or 0)) +if existing: + seq = max(seq, int(existing.get("sequence") or 0)) + 1 +ok, reason, meta = dm_relay.register_prekey_bundle( + agent_id=agent_id, + bundle=dict(stored.get("bundle") or {{}}), + signature=str(stored.get("signature") or ""), + public_key=str(stored.get("public_key") or ""), + public_key_algo=str(stored.get("public_key_algo") or "Ed25519"), + protocol_version=str(stored.get("protocol_version") or "infonet/2"), + sequence=seq, + lookup_aliases=[{{"handle": handle}}], +) +print(json.dumps({{"ok": ok, "detail": reason, "sequence": seq, "meta": meta or {{}}}})) +""" + return _ssh_pete_python(seed_code, timeout_s=90) + + +def _docker_python(code: str, *, timeout_s: int = 600) -> dict: proc = subprocess.run( - ["docker", "exec", "shadowbroker-backend", "python", "-c", code], + ["docker", "exec", "-i", "shadowbroker-backend", "python", "-"], + input=code, capture_output=True, text=True, - timeout=600, + timeout=timeout_s, check=False, ) if proc.returncode != 0: @@ -126,27 +319,16 @@ def _docker_python(code: str) -> dict: return json.loads(line) -def _restart_local_backend() -> None: - """Clear in-memory DM relay state (MESH_DM_PERSIST_SPOOL=false) before a repeat run.""" - proc = subprocess.run( - [ - "docker", - "compose", - "-f", - "docker-compose.yml", - "-f", - "docker-compose.build.yml", - "restart", - "backend", - ], - capture_output=True, - text=True, - timeout=180, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError(proc.stderr.strip() or proc.stdout.strip() or "backend restart failed") - deadline = time.time() + 120 +def _local_compose_cmd(*subcommand: str) -> list[str]: + cmd = ["docker", "compose"] + for compose_file in LOCAL_COMPOSE_FILES: + cmd.extend(["-f", compose_file]) + cmd.extend(subcommand) + return cmd + + +def _wait_local_backend_healthy(*, timeout_s: int = 300) -> None: + deadline = time.time() + timeout_s while time.time() < deadline: probe = subprocess.run( [ @@ -154,9 +336,9 @@ def _restart_local_backend() -> None: "exec", "shadowbroker-backend", "curl", - "-sf", + "-s", "--max-time", - "5", + "60", "http://127.0.0.1:8000/api/health", ], capture_output=True, @@ -164,104 +346,573 @@ def _restart_local_backend() -> None: check=False, ) if probe.returncode == 0: - print("local backend restarted and healthy") - return + mesh_only = subprocess.run( + ["docker", "exec", "shadowbroker-backend", "printenv", "MESH_ONLY"], + capture_output=True, + text=True, + check=False, + ) + if (mesh_only.stdout or "").strip().lower() == "true": + print("local lean E2E backend healthy (MESH_ONLY=true)") + return + raise RuntimeError("local backend is up but MESH_ONLY is not enabled") time.sleep(3) raise RuntimeError("backend did not become healthy after restart") -def _wait_hidden_transport_ready(*, timeout_s: int = 120) -> dict: +def _ensure_local_e2e_backend(*, recreate: bool) -> None: + """Run local backend in lean E2E mode (no OSINT fetchers).""" + _scrub_local_dm_state() + if recreate: + proc = subprocess.run( + _local_compose_cmd("up", "-d", "--force-recreate", "backend"), + capture_output=True, + text=True, + timeout=300, + check=False, + ) + action = "recreated" + else: + proc = subprocess.run( + _local_compose_cmd("restart", "backend"), + capture_output=True, + text=True, + timeout=180, + check=False, + ) + action = "restarted" + if proc.returncode != 0: + raise RuntimeError(proc.stderr.strip() or proc.stdout.strip() or "local backend compose failed") + _wait_local_backend_healthy() + print(f"local backend {action} with docker-compose.e2e.yml") + + +def _prime_dm_wormhole() -> dict: + """Start wormhole inside the running uvicorn process (not a one-off exec shell).""" + return _docker_json("POST", "/api/wormhole/join", body={}, timeout_s=120) + + +def _hidden_transport_enforced() -> bool: code = ( - "import json, time\n" - "from services.mesh.mesh_private_dispatcher import _anonymous_dm_hidden_transport_enforced\n" - f"deadline = time.time() + {int(timeout_s)}\n" - "while time.time() < deadline:\n" - " if _anonymous_dm_hidden_transport_enforced():\n" - " print(json.dumps({'ok': True}))\n" - " break\n" - " time.sleep(2)\n" - "else:\n" - " print(json.dumps({'ok': False, 'detail': 'hidden transport not ready'}))\n" + "import json\n" + "from services.wormhole_settings import read_wormhole_settings\n" + "settings = read_wormhole_settings()\n" + "transport = str(settings.get('transport', '') or '').lower()\n" + "print(json.dumps({\n" + " 'ok': bool(settings.get('anonymous_mode'))\n" + " and transport in {'tor', 'tor_arti', 'i2p', 'mixnet'},\n" + "}))\n" ) - return _docker_python(code) + return bool(_docker_python(code).get("ok")) + + +_TIER_ORDER = {"public_degraded": 0, "private_transitional": 1, "private_strong": 2} + + +def _transport_lane_sufficient(current: str, required: str) -> bool: + return _TIER_ORDER.get(str(current or "").strip(), 0) >= _TIER_ORDER.get(str(required or "").strip(), 0) + + +def _runtime_lane_snapshot(runtime: dict) -> dict: + tier = str(runtime.get("transport_tier") or "") + required = "private_transitional" + tier_ok = _transport_lane_sufficient(tier, required) + transport_ready = ( + bool(runtime.get("ready")) + and bool(runtime.get("anonymous_mode_ready")) + and bool(runtime.get("arti_ready")) + ) + return {"ok": tier_ok and transport_ready, "tier": tier, "required": required} + + +def _private_lane_ready(*, join: bool = False) -> dict: + """Check private lane readiness from live uvicorn wormhole state.""" + if not join: + try: + status = _docker_json("GET", "/api/settings/wormhole-status", timeout_s=10) + if status and bool(status.get("ready")): + return {"ok": True, "tier": "private_transitional", "required": "private_transitional"} + if status: + return {"ok": False, "tier": "", "required": "private_transitional"} + except Exception: + pass + return {"ok": False, "tier": "", "required": "private_transitional"} + payload = _docker_json("POST", "/api/wormhole/join", body={}, timeout_s=120) + return _runtime_lane_snapshot(dict(payload.get("runtime") or {})) + + +def _wait_hidden_transport_ready(*, timeout_s: int = 360) -> dict: + if not _hidden_transport_enforced(): + return {"ok": True, "transport_tier": "not_enforced"} + try: + _docker_json("POST", "/api/wormhole/join", body={}, timeout_s=120) + except Exception: + pass + deadline = time.time() + timeout_s + last_lane: dict = {} + polls = 0 + while time.time() < deadline: + last_lane = _private_lane_ready(join=(polls == 0)) + if last_lane.get("ok"): + return {"ok": True, "transport_tier": last_lane.get("tier")} + polls += 1 + if polls % 20 == 0: + try: + last_lane = _private_lane_ready(join=True) + if last_lane.get("ok"): + return {"ok": True, "transport_tier": last_lane.get("tier")} + except Exception: + pass + time.sleep(3) + return { + "ok": False, + "detail": "hidden transport or private lane not ready", + "transport_tier": last_lane.get("tier"), + "required_tier": last_lane.get("required"), + } + + +def _ssh_pete_release_outbox(pete_admin: str, outbox_id: str, *, timeout_s: int = 180) -> dict: + outbox_id = str(outbox_id or "").strip() + if not outbox_id: + return {"ok": True, "skipped": True, "reason": "no outbox_id"} + proc = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + SSH_PETE, + ( + f"curl -s -X POST -H 'X-Admin-Key: {pete_admin}' " + f"-H 'Content-Type: application/json' " + f"-d '{{\"action\":\"relay\"}}' " + f"'http://127.0.0.1:8000/api/wormhole/private-delivery/{outbox_id}/action'" + ), + ], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if proc.returncode != 0: + return {"ok": False, "detail": proc.stderr.strip() or proc.stdout.strip() or "pete release failed"} + deadline = time.time() + timeout_s + while time.time() < deadline: + status_proc = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + SSH_PETE, + f"curl -s -H 'X-Admin-Key: {pete_admin}' 'http://127.0.0.1:8000/api/wormhole/status'", + ], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if status_proc.returncode == 0 and status_proc.stdout.strip(): + status = json.loads(status_proc.stdout) + items = list((status.get("private_delivery") or {}).get("items") or []) + item = next((entry for entry in items if str(entry.get("id", "")) == outbox_id), None) + if item and str(item.get("release_state", "")) == "delivered": + return {"ok": True, "item": item} + time.sleep(3) + return {"ok": False, "detail": "pete private release did not complete in time", "outbox_id": outbox_id} + + +def _wait_pete_outbox_delivered(pete_admin: str, outbox_id: str, *, timeout_s: int = 300) -> dict: + outbox_id = str(outbox_id or "").strip() + if not outbox_id: + return {"ok": False, "detail": "missing outbox_id"} + deadline = time.time() + timeout_s + last_state = "" + while time.time() < deadline: + status_proc = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + SSH_PETE, + ( + "curl -s --max-time 20 " + f"-H 'X-Admin-Key: {pete_admin}' " + f"'http://127.0.0.1:8000/api/wormhole/private-delivery/{outbox_id}'" + ), + ], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + if status_proc.returncode == 0 and status_proc.stdout.strip(): + payload = json.loads(status_proc.stdout) + item = payload.get("item") if isinstance(payload, dict) else None + if isinstance(item, dict): + last_state = str(item.get("release_state", "") or "") + if last_state == "delivered": + return {"ok": True, "item": item} + time.sleep(3) + return { + "ok": False, + "detail": "pete private release did not complete in time", + "outbox_id": outbox_id, + "last_release_state": last_state, + } + + +def _docker_json_optional( + method: str, + path: str, + body: dict | None = None, + *, + admin_key: str = "", + timeout_s: int = 30, +) -> dict | None: + try: + return _docker_json(method, path, body, admin_key=admin_key, timeout_s=timeout_s) + except (RuntimeError, json.JSONDecodeError, subprocess.TimeoutExpired): + return None + + +def _get_local_outbox_item(admin_key: str, outbox_id: str) -> dict | None: + # Prefer in-process read — HTTP /api/wormhole/private-delivery can wedge when + # fleet peer-push floods the single uvicorn worker during Tor E2E. + code = ( + "import json\n" + "from services.mesh.mesh_private_outbox import private_delivery_outbox\n" + f"item = private_delivery_outbox.get_item({json.dumps(outbox_id)}, exposure='ordinary')\n" + "print(json.dumps({'item': item}))\n" + ) + try: + payload = _docker_python(code) + item = payload.get("item") + if isinstance(item, dict): + return item + except Exception: + pass + payload = _docker_json_optional( + "GET", + f"/api/wormhole/private-delivery/{outbox_id}", + admin_key=admin_key, + timeout_s=20, + ) + if not payload: + return None + item = payload.get("item") + return item if isinstance(item, dict) else None + + +def _wake_local_release_worker() -> None: + code = ( + "import json\n" + "from services.mesh.mesh_private_release_worker import private_release_worker\n" + "private_release_worker.ensure_started()\n" + "private_release_worker.wake()\n" + "private_release_worker.run_once()\n" + "print(json.dumps({'ok': True}))\n" + ) + try: + _docker_python(code) + except Exception as exc: + print(f"local release worker wake skipped: {exc}") + + +def _wake_pete_release_worker() -> None: + code = ( + "import json\n" + "from services.mesh.mesh_private_release_worker import private_release_worker\n" + "private_release_worker.ensure_started()\n" + "private_release_worker.wake()\n" + "private_release_worker.run_once()\n" + "print(json.dumps({'ok': True}))\n" + ) + try: + _ssh_pete_python(code, timeout_s=60) + except Exception as exc: + print(f"pete release worker wake skipped: {exc}") + + +def _wait_local_outbox_delivered(admin_key: str, outbox_id: str, *, timeout_s: int = 300) -> dict: + outbox_id = str(outbox_id or "").strip() + if not outbox_id: + return {"ok": False, "detail": "missing outbox_id"} + deadline = time.time() + timeout_s + last_state = "" + while time.time() < deadline: + item = _get_local_outbox_item(admin_key, outbox_id) + if item: + last_state = str(item.get("release_state", "") or "") + if last_state == "delivered": + return {"ok": True, "item": item} + time.sleep(3) + return { + "ok": False, + "detail": "private release did not complete in time", + "outbox_id": outbox_id, + "last_release_state": last_state, + } def _release_dm_outbox(admin_key: str, outbox_id: str, *, timeout_s: int = 180) -> dict: outbox_id = str(outbox_id or "").strip() if not outbox_id: return {"ok": False, "detail": "missing outbox_id"} - _docker_json( - "POST", - f"/api/wormhole/private-delivery/{outbox_id}/action", - {"action": "relay"}, - admin_key=admin_key, + action_timeout_s = max(120, int(os.environ.get("MESH_RELAY_PUSH_TIMEOUT_S", "120") or 120) + 30) + try: + _docker_json( + "POST", + f"/api/wormhole/private-delivery/{outbox_id}/action", + {"action": "relay"}, + admin_key=admin_key, + timeout_s=action_timeout_s, + ) + except Exception as exc: + print(f"private relay nudge skipped: {exc}") + return _wait_local_outbox_delivered(admin_key, outbox_id, timeout_s=timeout_s) + + +def _socks_handshake_preamble(*, deadline_s: int = 90) -> str: + """Wait for Arti SOCKS port only (curl push does not need torproject proof).""" + return ( + "import json, os, socket, time\n" + "from routers.ai_intel import _write_env_value\n" + "from services.config import get_settings\n" + "os.environ['MESH_RELAY_PUSH_TIMEOUT_S'] = '90'\n" + "_write_env_value('MESH_ARTI_ENABLED', 'true')\n" + "get_settings.cache_clear()\n" + f"_socks_deadline = time.time() + {int(deadline_s)}\n" + "def _socks_ready():\n" + " port = int(get_settings().MESH_ARTI_SOCKS_PORT or 9050)\n" + " try:\n" + " with socket.create_connection(('127.0.0.1', port), timeout=2.0) as sock:\n" + " sock.sendall(b'\\x05\\x01\\x00')\n" + " return sock.recv(2) == b'\\x05\\x00'\n" + " except OSError:\n" + " return False\n" + "while time.time() < _socks_deadline and not _socks_ready():\n" + " time.sleep(2)\n" + "if not _socks_ready():\n" + " print(json.dumps({'ok': False, 'detail': 'Arti SOCKS not ready for scoped replicate nudge'}))\n" + " raise SystemExit(0)\n" ) - deadline = time.time() + timeout_s - while time.time() < deadline: - status = _docker_json("GET", "/api/wormhole/status", admin_key=admin_key) - items = list((status.get("private_delivery") or {}).get("items") or []) - item = next((entry for entry in items if str(entry.get("id", "")) == outbox_id), None) - if item and str(item.get("release_state", "")) == "delivered": - return {"ok": True, "item": item} - time.sleep(3) - return {"ok": False, "detail": "private release did not complete in time", "outbox_id": outbox_id} -def _drain_pete_request_mailbox() -> None: - drain_code = textwrap.dedent( - """ - import json, secrets, time, urllib.request - from services.mesh.mesh_protocol import PROTOCOL_VERSION - from services.mesh.mesh_wormhole_persona import get_dm_identity, sign_dm_wormhole_event - - def _poll_once(): - identity = get_dm_identity() - agent_id = str(identity.get("node_id") or "") - claims = [{"type": "requests", "token": "e2e-drain"}] - signed = sign_dm_wormhole_event( - event_type="dm_poll", - payload={"mailbox_claims": claims, "agent_id": agent_id}, - ) - body = { - "agent_id": agent_id, - "mailbox_claims": claims, - "timestamp": int(time.time()), - "nonce": secrets.token_hex(8), - "public_key": str(signed.get("public_key") or ""), - "public_key_algo": str(signed.get("public_key_algo") or ""), - "signature": str(signed.get("signature") or ""), - "sequence": int(signed.get("sequence") or 0), - "protocol_version": str(signed.get("protocol_version") or PROTOCOL_VERSION), - } - data = json.dumps(body, separators=(",", ":"), sort_keys=True).encode("utf-8") - req = urllib.request.Request( - "http://127.0.0.1:8000/api/mesh/dm/poll", - data=data, - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=60) as resp: - return json.loads(resp.read().decode("utf-8")) - - drained = 0 - for _ in range(8): - payload = _poll_once() - count = int(payload.get("count", 0) or 0) - drained += count - if count <= 0 and not payload.get("has_more"): - break - time.sleep(1) - print(json.dumps({"ok": True, "drained": drained})) - """ - ).strip() - result = _ssh_pete_python(drain_code) - print(f"Pete request mailbox drain: {result.get('drained', 0)} message(s)") +_ARTI_NUDGE_PREAMBLE = _socks_handshake_preamble(deadline_s=90) -def _warmup_tor() -> None: - """Prime local Arti SOCKS before fleet lookups (cold Tor can exceed lookup budgets).""" - if not PETE_ONION: - return +def _scoped_replicate_outbox_nudge_code( + outbox_id: str, + *, + msg_id_hint: str = "", + warm_arti: bool = False, +) -> str: + preamble = _ARTI_NUDGE_PREAMBLE if warm_arti else "" + return preamble + ( + "import json\n" + "from services.mesh.mesh_private_outbox import private_delivery_outbox\n" + "from services.mesh.mesh_dm_connect_delivery import enrich_connect_release_payload, relay_push_peer_urls_for_payload\n" + "from services.mesh.mesh_dm_relay import dm_relay\n" + f"outbox_id = {json.dumps(outbox_id)}\n" + f"msg_id_hint = {json.dumps(msg_id_hint)}\n" + "item = private_delivery_outbox._items.get(outbox_id, {})\n" + "payload = enrich_connect_release_payload(dict(item.get('payload') or {}))\n" + "urls = [\n" + " str(raw or '').strip().rstrip('/')\n" + " for raw in list(payload.get('relay_push_peer_urls') or [])\n" + " if str(raw or '').strip()\n" + "]\n" + "if not urls:\n" + " urls = relay_push_peer_urls_for_payload(payload)\n" + "if not urls:\n" + " print(json.dumps({'ok': False, 'detail': 'no relay push urls in outbox payload'}))\n" + "else:\n" + " recipient_id = str(payload.get('recipient_id') or '')\n" + " envelope_obj = dict(payload.get('envelope') or {})\n" + " msg_id = str(payload.get('msg_id') or envelope_obj.get('msg_id') or msg_id_hint or '')\n" + " delivery_class = str(payload.get('delivery_class') or 'request').strip().lower()\n" + " recipient_token = str(payload.get('recipient_token') or '')\n" + " if not msg_id and recipient_id:\n" + " epoch = dm_relay._epoch_bucket()\n" + " for offset in (0, -1, -2):\n" + " key = dm_relay._mailbox_key('requests', recipient_id, epoch + offset)\n" + " for message in reversed(list(dm_relay._mailboxes.get(key, []))):\n" + " candidate = str(message.msg_id or '')\n" + " if candidate:\n" + " msg_id = candidate\n" + " break\n" + " if msg_id:\n" + " break\n" + " if delivery_class == 'shared':\n" + " mailbox_key = dm_relay._hashed_mailbox_token(recipient_token)\n" + " else:\n" + " mailbox_key = dm_relay.mailbox_key_for_delivery(\n" + " recipient_id=recipient_id,\n" + " delivery_class=delivery_class or 'request',\n" + " recipient_token=recipient_token or None,\n" + " )\n" + " envelope = dm_relay.envelope_for_replication(\n" + " mailbox_key=mailbox_key,\n" + " msg_id=msg_id,\n" + " recipient_id=recipient_id,\n" + " recipient_token=recipient_token or None,\n" + " )\n" + " if envelope:\n" + " if not str(envelope.get('delivery_class') or '').strip():\n" + " envelope['delivery_class'] = delivery_class or 'request'\n" + " if not str(envelope.get('recipient_id') or '').strip():\n" + " envelope['recipient_id'] = recipient_id\n" + " replicate = dm_relay._replicate_envelope_to_peers(\n" + " envelope=envelope, preferred_peer_urls=urls,\n" + " )\n" + " else:\n" + " deposited = dm_relay.deposit(\n" + " sender_id=str(payload.get('sender_id') or ''),\n" + " raw_sender_id=str(payload.get('sender_id') or ''),\n" + " recipient_id=recipient_id,\n" + " ciphertext=str(payload.get('ciphertext') or ''),\n" + " msg_id=msg_id,\n" + " delivery_class=delivery_class,\n" + " sender_seal=str(payload.get('sender_seal') or ''),\n" + " sender_token_hash=str(payload.get('sender_token_hash') or ''),\n" + " payload_format=str(payload.get('format') or 'mls1'),\n" + " replication_peer_urls=urls,\n" + " recipient_token=recipient_token,\n" + " )\n" + " replicate = dict(deposited.get('replicate') or {})\n" + " print(json.dumps({'ok': bool(replicate.get('ok')), 'replicate': replicate, 'urls': urls, 'msg_id': msg_id}))\n" + ) + + +def _scoped_replicate_envelope_package_code( + outbox_id: str = "", + *, + msg_id_hint: str = "", + payload: dict | None = None, +) -> str: + """Build a signed replicate-envelope POST package without opening Tor sockets.""" + if payload is not None: + payload_loader = ( + f"payload = enrich_connect_release_payload(dict({json.dumps(payload)}))\n" + ) + imports = ( + "import json, base64, hashlib, hmac\n" + "from services.mesh.mesh_dm_connect_delivery import enrich_connect_release_payload, relay_push_peer_urls_for_payload\n" + ) + else: + payload_loader = ( + f"outbox_id = {json.dumps(outbox_id)}\n" + "item = private_delivery_outbox._items.get(outbox_id, {})\n" + "payload = enrich_connect_release_payload(dict(item.get('payload') or {}))\n" + ) + imports = ( + "import json, base64, hashlib, hmac\n" + "from services.mesh.mesh_private_outbox import private_delivery_outbox\n" + "from services.mesh.mesh_dm_connect_delivery import enrich_connect_release_payload, relay_push_peer_urls_for_payload\n" + ) + return ( + imports + + "from services.mesh.mesh_dm_relay import dm_relay\n" + "from services.mesh.mesh_crypto import normalize_peer_url, resolve_peer_key_for_url\n" + f"msg_id_hint = {json.dumps(msg_id_hint)}\n" + + payload_loader + + "urls = [\n" + " str(raw or '').strip().rstrip('/')\n" + " for raw in list(payload.get('relay_push_peer_urls') or [])\n" + " if str(raw or '').strip()\n" + "]\n" + "if not urls:\n" + " urls = relay_push_peer_urls_for_payload(payload)\n" + "if not urls:\n" + " print(json.dumps({'ok': False, 'detail': 'no relay push urls in outbox payload'}))\n" + "else:\n" + " recipient_id = str(payload.get('recipient_id') or '')\n" + " envelope_obj = dict(payload.get('envelope') or {})\n" + " msg_id = str(payload.get('msg_id') or envelope_obj.get('msg_id') or msg_id_hint or '')\n" + " delivery_class = str(payload.get('delivery_class') or 'request').strip().lower()\n" + " recipient_token = str(payload.get('recipient_token') or '')\n" + " if not msg_id and recipient_id:\n" + " epoch = dm_relay._epoch_bucket()\n" + " for offset in (0, -1, -2):\n" + " key = dm_relay._mailbox_key('requests', recipient_id, epoch + offset)\n" + " for message in reversed(list(dm_relay._mailboxes.get(key, []))):\n" + " candidate = str(message.msg_id or '')\n" + " if candidate:\n" + " msg_id = candidate\n" + " break\n" + " if msg_id:\n" + " break\n" + " if delivery_class == 'shared':\n" + " mailbox_key = dm_relay._hashed_mailbox_token(recipient_token)\n" + " else:\n" + " mailbox_key = dm_relay.mailbox_key_for_delivery(\n" + " recipient_id=recipient_id,\n" + " delivery_class=delivery_class or 'request',\n" + " recipient_token=recipient_token or None,\n" + " )\n" + " envelope = dm_relay.envelope_for_replication(\n" + " mailbox_key=mailbox_key,\n" + " msg_id=msg_id,\n" + " recipient_id=recipient_id,\n" + " recipient_token=recipient_token or None,\n" + " )\n" + " if not envelope:\n" + " ciphertext = str(payload.get('ciphertext') or envelope_obj.get('ciphertext') or '')\n" + " sender_id = str(payload.get('sender_id') or envelope_obj.get('sender_id') or '')\n" + " sender_seal = str(payload.get('sender_seal') or envelope_obj.get('sender_seal') or '')\n" + " sender_token_hash = str(payload.get('sender_token_hash') or envelope_obj.get('sender_token_hash') or '')\n" + " payload_format = str(payload.get('format') or envelope_obj.get('payload_format') or 'mls1')\n" + " session_welcome = str(payload.get('session_welcome') or envelope_obj.get('session_welcome') or '')\n" + " if ciphertext and msg_id and mailbox_key:\n" + " sender_block_ref = dm_relay._sender_block_ref(\n" + " sender_id,\n" + " scope=dm_relay._sender_block_scope(\n" + " recipient_id=recipient_id,\n" + " recipient_token=str(recipient_token or ''),\n" + " delivery_class=delivery_class,\n" + " ),\n" + " )\n" + " envelope = {\n" + " 'msg_id': msg_id,\n" + " 'mailbox_key': mailbox_key,\n" + " 'recipient_id': recipient_id,\n" + " 'recipient_token': recipient_token,\n" + " 'sender_id': sender_id,\n" + " 'sender_block_ref': sender_block_ref,\n" + " 'sender_seal': sender_seal,\n" + " 'ciphertext': ciphertext,\n" + " 'delivery_class': delivery_class or 'request',\n" + " 'payload_format': payload_format,\n" + " 'session_welcome': session_welcome,\n" + " 'timestamp': float(payload.get('timestamp') or envelope_obj.get('timestamp') or 0) or __import__('time').time(),\n" + " }\n" + " if not envelope:\n" + " print(json.dumps({'ok': False, 'detail': 'envelope missing for scoped replicate', 'msg_id': msg_id}))\n" + " else:\n" + " if not str(envelope.get('delivery_class') or '').strip():\n" + " envelope['delivery_class'] = delivery_class or 'request'\n" + " if not str(envelope.get('recipient_id') or '').strip():\n" + " envelope['recipient_id'] = recipient_id\n" + " target = normalize_peer_url(str(urls[0]))\n" + " peer_key = resolve_peer_key_for_url(target)\n" + " if not peer_key:\n" + " print(json.dumps({'ok': False, 'detail': 'no peer key for replicate target', 'target': target}))\n" + " else:\n" + " body_bytes = json.dumps({'envelope': envelope}, separators=(',', ':'), sort_keys=True).encode('utf-8')\n" + " host = target.replace('http://', '').replace('https://', '').rstrip('/')\n" + " sig = hmac.new(peer_key, body_bytes, hashlib.sha256).hexdigest()\n" + " print(json.dumps({\n" + " 'ok': True,\n" + " 'target_host': host,\n" + " 'peer_url': target,\n" + " 'peer_hmac': sig,\n" + " 'body_b64': base64.b64encode(body_bytes).decode('ascii'),\n" + " 'msg_id': msg_id,\n" + " }))\n" + ) + + +def _local_api_health(*, timeout_s: int = 10) -> bool: proc = subprocess.run( [ "docker", @@ -274,21 +925,798 @@ def _warmup_tor() -> None: "-w", "%{http_code}", "--max-time", - "120", - "--socks5-hostname", - "127.0.0.1:9050", - f"http://{PETE_ONION}/api/health", + str(timeout_s), + "http://127.0.0.1:8000/api/health", + ], + capture_output=True, + text=True, + timeout=timeout_s + 15, + check=False, + ) + return (proc.stdout or "").strip() == "200" + + +def _ensure_local_api_responsive(*, reason: str = "") -> None: + if _local_api_health(timeout_s=10): + return + label = f" ({reason})" if reason else "" + print(f"local backend unresponsive{label} — restarting before replicate push") + subprocess.run( + _local_compose_cmd("restart", "backend"), + capture_output=True, + text=True, + timeout=180, + check=False, + ) + _wait_local_backend_healthy(timeout_s=300) + _prime_dm_wormhole() + + +def _pete_api_health(timeout_s: int = 10) -> bool: + proc = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + SSH_PETE, + f"curl -s -o /dev/null -w '%{{http_code}}' --max-time {int(timeout_s)} http://127.0.0.1:8000/api/health", + ], + capture_output=True, + text=True, + timeout=timeout_s + 20, + check=False, + ) + return (proc.stdout or "").strip() == "200" + + +def _ensure_pete_api_responsive(pete_admin: str = "", *, reason: str = "") -> None: + if _pete_api_health(timeout_s=10): + return + label = f" ({reason})" if reason else "" + print(f"Pete backend unresponsive{label} — restarting container") + subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + SSH_PETE, + "cd /home/ubuntu/Shadowbroker && docker compose -f docker-compose.yml -f docker-compose.participant.yml restart backend", ], capture_output=True, text=True, timeout=180, check=False, ) - code = (proc.stdout or "").strip() - print(f"Tor warmup Pete health: {code or proc.stderr.strip() or 'failed'}") + time.sleep(30) + for _ in range(20): + if _pete_api_health(timeout_s=10): + if pete_admin: + join = _prime_pete_wormhole_http(pete_admin) + print(json.dumps({"pete_reprime_after_restart": join}, indent=2)) + return + time.sleep(6) + raise RuntimeError("Pete backend did not become healthy after restart") -def _ssh_pete_python(code: str) -> dict: +def _push_replicate_package_direct_local(package: dict) -> dict: + """POST replicate-envelope to local uvicorn (no Tor) — lands in live mailbox.""" + if not package.get("ok"): + return package + py = ( + "import base64, json, subprocess\n" + f"body = base64.b64decode({json.dumps(package.get('body_b64', ''))})\n" + f"peer_url = {json.dumps(package.get('peer_url', ''))}\n" + f"peer_hmac = {json.dumps(package.get('peer_hmac', ''))}\n" + "proc = subprocess.run(\n" + " [\n" + " 'curl', '-s', '-w', '\\n%{http_code}', '--max-time', '60',\n" + " '-X', 'POST',\n" + " '-H', 'Content-Type: application/json',\n" + " '-H', f'X-Peer-Url: {peer_url}',\n" + " '-H', f'X-Peer-HMAC: {peer_hmac}',\n" + " '--data-binary', '@-',\n" + " 'http://127.0.0.1:8000/api/mesh/dm/replicate-envelope',\n" + " ],\n" + " input=body,\n" + " capture_output=True,\n" + ")\n" + "raw = (proc.stdout or b'').decode('utf-8', errors='replace').strip()\n" + "lines = raw.splitlines()\n" + "code = lines[-1] if lines else ''\n" + "text = '\\n'.join(lines[:-1]) if len(lines) > 1 else ''\n" + "replicate_ok = False\n" + "detail = (proc.stderr or b'').decode('utf-8', errors='replace').strip() or text\n" + "try:\n" + " payload = json.loads(text) if text else {}\n" + " if isinstance(payload, dict):\n" + " replicate_ok = bool(payload.get('ok'))\n" + " if not replicate_ok:\n" + " detail = str(payload.get('detail', '') or detail)\n" + "except Exception:\n" + " replicate_ok = code == '200'\n" + "print(json.dumps({\n" + " 'ok': bool(replicate_ok and code == '200'),\n" + " 'http_code': code,\n" + " 'detail': detail,\n" + " 'msg_id': " + f"{json.dumps(package.get('msg_id', ''))},\n" + "}))\n" + ) + return _docker_python(py) + + +def _local_accept_replica_direct(package: dict) -> dict: + """Ingest replicate envelope via one-off python (on-disk relay for poll/decrypt).""" + if not package.get("ok") or not package.get("body_b64"): + return {"ok": False, "detail": "missing replicate package"} + code = ( + "import json, base64\n" + "from services.mesh.mesh_dm_relay import dm_relay\n" + f"body = json.loads(base64.b64decode({json.dumps(package.get('body_b64', ''))}).decode('utf-8'))\n" + "envelope = dict(body.get('envelope') or {})\n" + f"result = dm_relay.accept_replica(envelope=envelope, originating_peer_url={json.dumps(str(package.get('peer_url') or ''))})\n" + "dm_relay._flush()\n" + "print(json.dumps(result))\n" + ) + try: + return _docker_python(code, timeout_s=60) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _push_replicate_package(package: dict, *, remote: str = "local") -> dict: + if not package.get("ok"): + return package + timeout_s = max(180, int(os.environ.get("MESH_RELAY_PUSH_TIMEOUT_S", "300") or 300) + 30) + py = ( + "import base64, json, subprocess\n" + f"body = base64.b64decode({json.dumps(package.get('body_b64', ''))})\n" + f"target = {json.dumps(package.get('target_host', ''))}\n" + f"peer_url = {json.dumps(package.get('peer_url', ''))}\n" + f"peer_hmac = {json.dumps(package.get('peer_hmac', ''))}\n" + "proc = subprocess.run(\n" + " [\n" + " 'curl', '-s', '-w', '\\n%{http_code}', '--max-time', " + f"{json.dumps(str(timeout_s))},\n" + " '--socks5-hostname', '127.0.0.1:9050',\n" + " '-X', 'POST',\n" + " '-H', 'Content-Type: application/json',\n" + " '-H', f'X-Peer-Url: {peer_url}',\n" + " '-H', f'X-Peer-HMAC: {peer_hmac}',\n" + " '--data-binary', '@-',\n" + " f'http://{target}/api/mesh/dm/replicate-envelope',\n" + " ],\n" + " input=body,\n" + " capture_output=True,\n" + ")\n" + "raw = (proc.stdout or b'').decode('utf-8', errors='replace').strip()\n" + "lines = raw.splitlines()\n" + "code = lines[-1] if lines else ''\n" + "text = '\\n'.join(lines[:-1]) if len(lines) > 1 else ''\n" + "replicate_ok = False\n" + "detail = (proc.stderr or b'').decode('utf-8', errors='replace').strip() or text\n" + "try:\n" + " payload = json.loads(text) if text else {}\n" + " if isinstance(payload, dict):\n" + " replicate_ok = bool(payload.get('ok'))\n" + " if not replicate_ok:\n" + " detail = str(payload.get('detail', '') or detail)\n" + "except Exception:\n" + " replicate_ok = code == '200'\n" + "print(json.dumps({\n" + " 'ok': bool(replicate_ok and code == '200'),\n" + " 'http_code': code,\n" + " 'detail': detail,\n" + " 'msg_id': " + f"{json.dumps(package.get('msg_id', ''))},\n" + "}))\n" + ) + if remote == "pete": + return _ssh_pete_python(py, timeout_s=timeout_s + 45) + return _docker_python(py) + + +def _nudge_scoped_replicate_to_pete( + outbox_id: str, + *, + msg_id: str = "", + pete_admin: str = "", +) -> dict: + """Push local sealed outbox envelope to Pete relay (scoped replicate).""" + outbox_id = str(outbox_id or "").strip() + if not outbox_id: + return {"ok": False, "detail": "missing outbox_id"} + try: + package: dict = {"ok": False} + payload = _fetch_local_outbox_payload(outbox_id) + if payload: + package = _docker_python( + _scoped_replicate_envelope_package_code("", msg_id_hint=msg_id, payload=payload) + ) + if package.get("ok"): + package["source"] = "local_disk_payload_local_sign" + if not package.get("ok"): + package = _docker_python(_scoped_replicate_envelope_package_code(outbox_id, msg_id_hint=msg_id)) + if package.get("ok"): + package["source"] = "local_outbox_exec" + if package.get("ok") and package.get("target_host"): + if pete_admin: + join = _prime_pete_wormhole_http(pete_admin) + if not join.get("ok"): + print(json.dumps({"pete_wormhole_prime_before_7c": join}, indent=2)) + pushed = _push_replicate_package(package, remote="pete") + result = { + "ok": bool(pushed.get("ok")), + "replicate": pushed, + "urls": [package.get("target_host", "")], + "msg_id": package.get("msg_id", msg_id), + "package_source": package.get("source", ""), + "push_via": "pete_tor", + "package": package, + } + if not result.get("ok"): + print(json.dumps({"pete_tor_push_failed": pushed}, indent=2)) + if result.get("ok"): + if not TOR_ONLY: + disk = _pete_accept_replica_direct(package) + result["disk_inject"] = disk + return result + if TOR_ONLY: + return result + pushed = _push_replicate_package_direct_pete(package) + result = { + "ok": bool(pushed.get("ok")), + "replicate": pushed, + "urls": [package.get("target_host", "")], + "msg_id": package.get("msg_id", msg_id), + "package_source": package.get("source", ""), + "push_via": "pete_http", + "package": package, + } + if not result.get("ok"): + print(json.dumps({"pete_http_push_failed": pushed}, indent=2)) + if result.get("ok") and not TOR_ONLY: + disk = _pete_accept_replica_direct(package) + result["disk_inject"] = disk + return result + if result.get("ok"): + return result + return _docker_python( + _scoped_replicate_outbox_nudge_code(outbox_id, msg_id_hint=msg_id, warm_arti=True) + ) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _nudge_scoped_replicate_from_pete( + outbox_id: str, + *, + msg_id: str = "", + pete_admin: str = "", +) -> dict: + """Tor-push Pete's sealed outbox envelope back to local onion (scoped replicate).""" + outbox_id = str(outbox_id or "").strip() + if not outbox_id: + return {"ok": False, "detail": "missing outbox_id"} + try: + package: dict = {"ok": False} + payload = _fetch_pete_outbox_payload(outbox_id) + if payload: + package = _docker_python( + _scoped_replicate_envelope_package_code("", msg_id_hint=msg_id, payload=payload) + ) + if package.get("ok"): + package["source"] = "pete_disk_payload_local_sign" + if not package.get("ok"): + package = _ssh_pete_python( + _scoped_replicate_envelope_package_code(outbox_id, msg_id_hint=msg_id), + timeout_s=90, + ) + if package.get("ok"): + package["source"] = "pete_outbox_exec" + if package.get("ok") and package.get("target_host"): + _ensure_local_api_responsive(reason="scoped replicate push") + pushed = _push_replicate_package(package, remote="local") + result = { + "ok": bool(pushed.get("ok")), + "replicate": pushed, + "urls": [package.get("target_host", "")], + "msg_id": package.get("msg_id", msg_id), + "package_source": package.get("source", ""), + "push_via": "local_tor", + "package": package, + } + if not result.get("ok"): + print(json.dumps({"local_tor_push_failed": pushed}, indent=2)) + if result.get("ok"): + if not TOR_ONLY: + disk = _local_accept_replica_direct(package) + result["disk_inject"] = disk + return result + if TOR_ONLY: + return result + pushed = _push_replicate_package_direct_local(package) + result = { + "ok": bool(pushed.get("ok")), + "replicate": pushed, + "urls": [package.get("target_host", "")], + "msg_id": package.get("msg_id", msg_id), + "package_source": package.get("source", ""), + "push_via": "local_http", + "package": package, + } + if not result.get("ok"): + print(json.dumps({"local_http_push_failed": pushed}, indent=2)) + if result.get("ok") and not TOR_ONLY: + disk = _local_accept_replica_direct(package) + result["disk_inject"] = disk + return result + if result.get("ok"): + return result + if pete_admin: + join = _prime_pete_wormhole_http(pete_admin) + if not join.get("ok"): + print(json.dumps({"pete_wormhole_http_prime": join}, indent=2)) + socks = _wait_pete_socks_port(timeout_s=90) + print(json.dumps({"pete_socks_before_push": socks}, indent=2)) + pushed = _push_replicate_package(package, remote="pete") + result = { + "ok": bool(pushed.get("ok")), + "replicate": pushed, + "urls": [package.get("target_host", "")], + "msg_id": package.get("msg_id", msg_id), + "package_source": package.get("source", ""), + "push_via": "pete", + } + if result.get("ok"): + return result + return _ssh_pete_python( + _scoped_replicate_outbox_nudge_code(outbox_id, msg_id_hint=msg_id, warm_arti=True), + timeout_s=240, + ) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _fetch_local_outbox_payload(outbox_id: str) -> dict | None: + """Read sealed outbox payload from local disk (reloads before lookup).""" + outbox_id = str(outbox_id or "").strip() + if not outbox_id: + return None + code = ( + "import json\n" + "from services.mesh.mesh_private_outbox import private_delivery_outbox\n" + f"outbox_id = {json.dumps(outbox_id)}\n" + "private_delivery_outbox._load()\n" + "item = private_delivery_outbox._items.get(outbox_id, {})\n" + "payload = dict(item.get('payload') or {})\n" + "print(json.dumps({'ok': bool(payload), 'payload': payload}))\n" + ) + try: + result = _docker_python(code, timeout_s=60) + if result.get("ok") and isinstance(result.get("payload"), dict): + return dict(result["payload"]) + except Exception as exc: + print(f"local outbox payload fetch skipped: {exc}") + return None + + +def _fetch_pete_outbox_payload(outbox_id: str) -> dict | None: + """Read sealed outbox payload from Pete disk (reloads before lookup).""" + outbox_id = str(outbox_id or "").strip() + if not outbox_id: + return None + code = ( + "import json\n" + "from services.mesh.mesh_private_outbox import private_delivery_outbox\n" + f"outbox_id = {json.dumps(outbox_id)}\n" + "private_delivery_outbox._load()\n" + "item = private_delivery_outbox._items.get(outbox_id, {})\n" + "payload = dict(item.get('payload') or {})\n" + "print(json.dumps({'ok': bool(payload), 'payload': payload}))\n" + ) + try: + result = _ssh_pete_python(code, timeout_s=60) + if result.get("ok") and isinstance(result.get("payload"), dict): + return dict(result["payload"]) + except Exception as exc: + print(f"Pete outbox payload fetch skipped: {exc}") + return None + + + return None + + +def _pete_accept_replica_direct(package: dict) -> dict: + """Ingest replicate envelope on Pete via one-off python (on-disk relay).""" + if not package.get("ok") or not package.get("body_b64"): + return {"ok": False, "detail": "missing replicate package"} + code = ( + "import json, base64\n" + "from services.mesh.mesh_dm_relay import dm_relay\n" + f"body = json.loads(base64.b64decode({json.dumps(package.get('body_b64', ''))}).decode('utf-8'))\n" + "envelope = dict(body.get('envelope') or {})\n" + f"result = dm_relay.accept_replica(envelope=envelope, originating_peer_url={json.dumps(str(package.get('peer_url') or ''))})\n" + "dm_relay._flush()\n" + "print(json.dumps(result))\n" + ) + try: + return _ssh_pete_python(code, timeout_s=60) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _push_replicate_package_direct_pete(package: dict) -> dict: + """POST replicate-envelope to Pete uvicorn (no Tor).""" + if not package.get("ok"): + return package + py = ( + "import base64, json, subprocess\n" + f"body = base64.b64decode({json.dumps(package.get('body_b64', ''))})\n" + f"peer_url = {json.dumps(package.get('peer_url', ''))}\n" + f"peer_hmac = {json.dumps(package.get('peer_hmac', ''))}\n" + "proc = subprocess.run(\n" + " [\n" + " 'curl', '-s', '-w', '\\n%{http_code}', '--max-time', '60',\n" + " '-X', 'POST',\n" + " '-H', 'Content-Type: application/json',\n" + " '-H', f'X-Peer-Url: {peer_url}',\n" + " '-H', f'X-Peer-HMAC: {peer_hmac}',\n" + " '--data-binary', '@-',\n" + " 'http://127.0.0.1:8000/api/mesh/dm/replicate-envelope',\n" + " ],\n" + " input=body,\n" + " capture_output=True,\n" + ")\n" + "raw = (proc.stdout or b'').decode('utf-8', errors='replace').strip()\n" + "lines = raw.splitlines()\n" + "code = lines[-1] if lines else ''\n" + "text = '\\n'.join(lines[:-1]) if len(lines) > 1 else ''\n" + "replicate_ok = False\n" + "detail = (proc.stderr or b'').decode('utf-8', errors='replace').strip() or text\n" + "try:\n" + " payload = json.loads(text) if text else {}\n" + " if isinstance(payload, dict):\n" + " replicate_ok = bool(payload.get('ok'))\n" + " if not replicate_ok:\n" + " detail = str(payload.get('detail', '') or detail)\n" + "except Exception:\n" + " replicate_ok = code == '200'\n" + "print(json.dumps({\n" + " 'ok': bool(replicate_ok and code == '200'),\n" + " 'http_code': code,\n" + " 'detail': detail,\n" + " 'msg_id': " + f"{json.dumps(package.get('msg_id', ''))},\n" + "}))\n" + ) + return _ssh_pete_python(py, timeout_s=90) + + +def _prime_pete_wormhole_http(pete_admin: str) -> dict: + """Prime Pete wormhole/Tor inside the running uvicorn process.""" + proc = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + SSH_PETE, + ( + "curl -s --max-time 120 -X POST " + f"-H 'X-Admin-Key: {pete_admin}' " + "-H 'Content-Type: application/json' " + "-d '{}' " + "'http://127.0.0.1:8000/api/wormhole/join'" + ), + ], + capture_output=True, + text=True, + timeout=150, + check=False, + ) + if proc.returncode != 0: + return {"ok": False, "detail": proc.stderr.strip() or proc.stdout.strip() or "pete join failed"} + try: + return json.loads(proc.stdout.strip() or "{}") + except json.JSONDecodeError: + return {"ok": False, "detail": proc.stdout.strip() or "pete join invalid json"} + + +def _wait_pete_socks_port(*, timeout_s: int = 120) -> dict: + code = ( + "import json, socket, time\n" + "from services.config import get_settings\n" + f"deadline = time.time() + {int(timeout_s)}\n" + "port = int(get_settings().MESH_ARTI_SOCKS_PORT or 9050)\n" + "ready = False\n" + "while time.time() < deadline:\n" + " try:\n" + " with socket.create_connection(('127.0.0.1', port), timeout=2.0) as sock:\n" + " sock.sendall(b'\\x05\\x01\\x00')\n" + " if sock.recv(2) == b'\\x05\\x00':\n" + " ready = True\n" + " break\n" + " except OSError:\n" + " pass\n" + " time.sleep(2)\n" + "print(json.dumps({'ok': ready, 'socks_port': port}))\n" + ) + try: + return _ssh_pete_python(code, timeout_s=timeout_s + 30) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _wait_pete_arti_ready(*, timeout_s: int = 120) -> dict: + code = ( + "import json, time\n" + "from routers.ai_intel import _write_env_value\n" + "from services.config import get_settings\n" + "from services.wormhole_supervisor import _check_arti_ready\n" + "_write_env_value('MESH_ARTI_ENABLED', 'true')\n" + "get_settings.cache_clear()\n" + f"deadline = time.time() + {int(timeout_s)}\n" + "ready = False\n" + "while time.time() < deadline:\n" + " if _check_arti_ready():\n" + " ready = True\n" + " break\n" + " time.sleep(2)\n" + "print(json.dumps({'ok': ready, 'arti_ready': ready}))\n" + ) + try: + return _ssh_pete_python(code, timeout_s=timeout_s + 30) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _scrub_local_dm_state() -> None: + """Drop persisted private outbox + local dm_relay spool between E2E runs.""" + proc = subprocess.run( + [ + "docker", + "exec", + "shadowbroker-backend", + "sh", + "-c", + "rm -f /app/data/private_outbox/sealed_private_outbox.json /app/data/dm_relay.json " + "/app/data/dm_alias/wormhole_dm_mls.json /app/data/dm_alias_rust/wormhole_dm_mls_rust.bin", + ], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + print(f"local DM state scrub skipped: {proc.stderr.strip() or proc.stdout.strip()}") + restart = subprocess.run( + _local_compose_cmd("restart", "backend"), + capture_output=True, + text=True, + timeout=120, + check=False, + ) + if restart.returncode != 0: + print(f"local backend restart after scrub skipped: {restart.stderr.strip() or restart.stdout.strip()}") + else: + try: + _wait_local_backend_healthy(timeout_s=120) + except Exception as exc: + print(f"local backend health wait after scrub skipped: {exc}") + + +def _drain_pete_request_mailbox(agent_id: str = "") -> None: + resolved_agent_id = str(agent_id or "").strip() + drain_code = f"""import json, secrets, time, urllib.request +from services.mesh.mesh_wormhole_persona import get_dm_identity, sign_dm_wormhole_event + +{_EMBED_SIGNED_MAILBOX_HELPERS} + +def _poll_once(): + agent_id = {json.dumps(resolved_agent_id)} or str((get_dm_identity() or {{}}).get("node_id") or "") + claims = [{{"type": "requests", "token": {json.dumps(_E2E_REQUESTS_MAILBOX_TOKEN)}}}] + body, data = _build_signed_mailbox_request( + agent_id=agent_id, + event_type="dm_poll", + kind="dm_poll", + endpoint="/api/mesh/dm/poll", + sequence_domain="dm_poll", + claims=claims, + ) + req = urllib.request.Request( + "http://127.0.0.1:8000/api/mesh/dm/poll", + data=data, + headers={{"Content-Type": "application/json"}}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=60) as resp: + return json.loads(resp.read().decode("utf-8")) + +drained = 0 +for _ in range(8): + payload = _poll_once() + count = int(payload.get("count", 0) or 0) + drained += count + if count <= 0 and not payload.get("has_more"): + break + time.sleep(1) +print(json.dumps({{"ok": True, "drained": drained}})) +""" + try: + result = _ssh_pete_python(drain_code) + print(f"Pete request mailbox drain: {result.get('drained', 0)} message(s)") + except Exception as exc: + print(f"Pete request mailbox drain skipped: {exc}") + + +def _restart_pete_backend() -> None: + repo_root = os.path.dirname(os.path.dirname(__file__)) + patch_files = [ + ("backend/services/mesh/mesh_dm_relay.py", "/tmp/mesh_dm_relay.py"), + ("backend/services/mesh/mesh_signed_events.py", "/tmp/mesh_signed_events.py"), + ("backend/services/openclaw_infonet.py", "/tmp/openclaw_infonet.py"), + ("backend/services/wormhole_supervisor.py", "/tmp/wormhole_supervisor.py"), + ("backend/services/tor_hidden_service.py", "/tmp/tor_hidden_service.py"), + ("backend/services/privacy_core_attestation.py", "/tmp/privacy_core_attestation.py"), + ("backend/routers/wormhole.py", "/tmp/wormhole_router.py"), + ("backend/main.py", "/tmp/main.py"), + ("docker-compose.participant.yml", "/tmp/docker-compose.participant.yml"), + ] + for rel_path, remote_tmp in patch_files: + local_path = os.path.join(repo_root, rel_path) + if os.path.isfile(local_path): + subprocess.run( + ["scp", "-o", "BatchMode=yes", local_path, f"{SSH_PETE}:{remote_tmp}"], + capture_output=True, + text=True, + check=False, + ) + remote_cmd = ( + "cd /home/ubuntu/Shadowbroker && " + "cp /tmp/docker-compose.participant.yml docker-compose.participant.yml 2>/dev/null || true && " + "docker compose -f docker-compose.yml -f docker-compose.participant.yml up -d backend && " + "sleep 8 && " + "docker exec shadowbroker-backend sh -c " + "'rm -f /app/data/dm_relay.json /app/data/private_outbox/sealed_private_outbox.json " + "/app/data/dm_alias/wormhole_dm_mls.json /app/data/dm_alias_rust/wormhole_dm_mls_rust.bin' && " + "docker cp /tmp/mesh_dm_relay.py shadowbroker-backend:/app/services/mesh/mesh_dm_relay.py 2>/dev/null || true; " + "docker cp /tmp/mesh_signed_events.py shadowbroker-backend:/app/services/mesh/mesh_signed_events.py 2>/dev/null || true; " + "docker cp /tmp/openclaw_infonet.py shadowbroker-backend:/app/services/openclaw_infonet.py 2>/dev/null || true; " + "docker cp /tmp/wormhole_supervisor.py shadowbroker-backend:/app/services/wormhole_supervisor.py 2>/dev/null || true; " + "docker cp /tmp/tor_hidden_service.py shadowbroker-backend:/app/services/tor_hidden_service.py 2>/dev/null || true; " + "docker cp /tmp/privacy_core_attestation.py shadowbroker-backend:/app/services/privacy_core_attestation.py 2>/dev/null || true; " + "docker cp /tmp/wormhole_router.py shadowbroker-backend:/app/routers/wormhole.py 2>/dev/null || true; " + "docker cp /tmp/main.py shadowbroker-backend:/app/main.py 2>/dev/null || true; " + "docker compose -f docker-compose.yml -f docker-compose.participant.yml restart backend" + ) + proc = subprocess.run( + ["ssh", "-o", "BatchMode=yes", SSH_PETE, remote_cmd], + capture_output=True, + text=True, + timeout=180, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError(proc.stderr.strip() or proc.stdout.strip() or "pete backend restart failed") + time.sleep(int(os.environ.get("E2E_DM_PETE_BOOTSTRAP_WAIT_S", "120"))) + + +def _prime_pete_dm_wormhole() -> dict: + code = ( + "import json\n" + "from routers.ai_intel import _write_env_value\n" + "from services.config import get_settings\n" + "from services.tor_hidden_service import tor_service\n" + "from services.wormhole_settings import write_wormhole_settings\n" + "from services.wormhole_supervisor import connect_wormhole\n" + "port = int(get_settings().MESH_ARTI_SOCKS_PORT or 9050)\n" + "write_wormhole_settings(enabled=True, transport='tor_arti', " + "socks_proxy=f'socks5h://127.0.0.1:{port}', socks_dns=True, anonymous_mode=True)\n" + "tor = tor_service.start(target_port=8000)\n" + "if tor.get('ok'):\n" + " _write_env_value('MESH_ARTI_ENABLED', 'true')\n" + " get_settings.cache_clear()\n" + "runtime = connect_wormhole(reason='e2e_dm_pete_warmup')\n" + "print(json.dumps({'ok': True, 'tor': tor, 'runtime': runtime}))\n" + ) + return _ssh_pete_python(code) + + +def _warmup_tor() -> None: + """Prime local Arti SOCKS before fleet lookups (cold Tor can exceed lookup budgets).""" + if not PETE_ONION: + return + for attempt in range(1, 7): + proc = subprocess.run( + [ + "docker", + "exec", + "shadowbroker-backend", + "curl", + "-s", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "120", + "--socks5-hostname", + "127.0.0.1:9050", + f"http://{PETE_ONION}/api/health", + ], + capture_output=True, + text=True, + timeout=150, + check=False, + ) + code = (proc.stdout or "").strip() + print(f"Tor warmup Pete health (attempt {attempt}): {code or proc.stderr.strip() or 'failed'}") + if code == "200": + return + time.sleep(30) + raise RuntimeError(f"Tor warmup to Pete onion failed after retries ({PETE_ONION})") + + +def _ensure_local_tor_hidden_service() -> dict: + """Start/refresh the local Tor hidden service inside the live uvicorn process.""" + join = _docker_json("POST", "/api/wormhole/join", body={}, timeout_s=120) + tor = dict(join.get("tor") or {}) + return { + "ok": bool(tor.get("ok")), + "tor": tor, + "onion_address": str(tor.get("onion_address") or ""), + } + + +def _warmup_tor_from_pete_to_local(local_onion: str, *, max_attempts: int = 0, raise_on_failure: bool = True) -> bool: + """Verify Pete can reach this node's inbound onion (accept replicate path).""" + host = str(local_onion or "").strip().replace("http://", "").replace("https://", "").rstrip("/") + if not host: + if raise_on_failure: + raise RuntimeError("missing local onion for Pete inbound Tor warmup") + return False + attempts = int(max_attempts or os.environ.get("E2E_DM_PETE_LOCAL_WARMUP_MAX", "12") or 12) + if attempts < 1: + attempts = 1 + if attempts == 12: + time.sleep(int(os.environ.get("E2E_DM_ONION_PROPAGATION_WAIT_S", "45"))) + else: + time.sleep(int(os.environ.get("E2E_DM_ONION_PROPAGATION_WAIT_SHORT_S", "10"))) + for attempt in range(1, attempts + 1): + proc = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + SSH_PETE, + ( + "docker exec shadowbroker-backend curl -s -o /dev/null -w '%{http_code}' " + f"--max-time 120 --socks5-hostname 127.0.0.1:9050 http://{host}/api/health" + ), + ], + capture_output=True, + text=True, + timeout=150, + check=False, + ) + code = (proc.stdout or "").strip() + print(f"Tor warmup Pete->local health (attempt {attempt}): {code or proc.stderr.strip() or 'failed'}") + if code == "200": + return True + time.sleep(30) + if raise_on_failure: + raise RuntimeError(f"Tor warmup from Pete to local onion failed ({host})") + return False + + +def _local_onion_from_join() -> str: + join = _docker_json("POST", "/api/wormhole/join", body={}, timeout_s=120) + onion = str((join.get("tor") or {}).get("onion_address") or "").strip().rstrip("/") + if not onion: + raise RuntimeError(f"could not resolve local onion from join: {join}") + return onion + + +def _ssh_pete_python(code: str, *, timeout_s: int = 120) -> dict: # Pipe script stdin to Pete's running backend container — avoids Windows # docker-exec base64 bugs and SSH command-line length limits on long polls. proc = subprocess.run( @@ -301,7 +1729,7 @@ def _ssh_pete_python(code: str) -> dict: ], input=code.encode("utf-8"), capture_output=True, - timeout=300, + timeout=timeout_s, check=False, ) if proc.returncode != 0: @@ -312,23 +1740,826 @@ def _ssh_pete_python(code: str) -> dict: return json.loads(lines[-1]) -def main() -> int: - if FRESH_BACKEND: - print("== prep: restart local backend for clean in-memory DM relay ==") - _restart_local_backend() +def _local_fetch_request_ciphertext( + agent_id: str, + *, + msg_id: str = "", + sender_id: str = "", +) -> dict: + code = f"""import json +from services.mesh.mesh_dm_relay import dm_relay +agent_id = {json.dumps(agent_id)} +msg_id = {json.dumps(msg_id)} +sender_id = {json.dumps(sender_id)} +token = {json.dumps(_E2E_REQUESTS_MAILBOX_TOKEN)} +ciphertext = "" +resolved_msg_id = "" +resolved_sender = "" +seen = [] +with dm_relay._lock: + dm_relay._refresh_from_shared_relay() + keys = [] + epoch = dm_relay._epoch_bucket() + for offset in range(-3, 2): + keys.append(dm_relay._mailbox_key("requests", agent_id, epoch + offset)) + bound = dm_relay._bound_mailbox_key(agent_id, "requests") + if bound: + keys.insert(0, bound) + dm_relay._remember_mailbox_binding(agent_id, "requests", token) + keys.extend(dm_relay._mailbox_keys_for_claim(agent_id, {{"type": "requests", "token": token}})) + for key in list(dict.fromkeys(keys)): + if msg_id: + envelope = dm_relay.envelope_for_replication( + mailbox_key=key, msg_id=msg_id, recipient_id=agent_id, + ) + if envelope and str(envelope.get("ciphertext") or ""): + ciphertext = str(envelope.get("ciphertext") or "") + resolved_msg_id = str(envelope.get("msg_id") or msg_id) + resolved_sender = str(envelope.get("sender_id") or "") + break + for message in list(dm_relay._mailboxes.get(key, [])): + seen.append(str(message.msg_id or "")) + if msg_id and str(message.msg_id) == msg_id: + ciphertext = str(message.ciphertext or "") + resolved_msg_id = str(message.msg_id or "") + resolved_sender = str(message.sender_id or "") + break + if sender_id and str(message.sender_id) in {{sender_id, f"sender_token:{{sender_id}}"}}: + ciphertext = str(message.ciphertext or "") + resolved_msg_id = str(message.msg_id or "") + resolved_sender = str(message.sender_id or "") + break + if ciphertext: + break +print(json.dumps({{ + "ok": bool(ciphertext), + "ciphertext": ciphertext, + "msg_id": resolved_msg_id, + "sender_id": resolved_sender, + "seen": seen, +}})) +""" + return _docker_python(code) - print("== prep: drain stale Pete request mailbox ==") - _drain_pete_request_mailbox() + +def _local_relay_requests_count(agent_id: str) -> dict: + """Count request-mailbox messages via persisted dm_relay (avoids wedged uvicorn HTTP).""" + code = f"""import json +from services.mesh.mesh_dm_relay import dm_relay +agent_id = {json.dumps(agent_id)} +token = {json.dumps(_E2E_REQUESTS_MAILBOX_TOKEN)} +claims = [{{"type": "requests", "token": token}}] +with dm_relay._lock: + dm_relay._refresh_from_shared_relay() + dm_relay._remember_mailbox_binding(agent_id, "requests", token) + count = int(dm_relay.count_claims(agent_id, claims)) +print(json.dumps({{ + "ok": True, + "count": count, +}})) +""" + return _docker_python(code) + + +def _local_http_dm_count(agent_id: str, *, timeout_s: int = 8) -> dict: + """Read mailbox count from the live uvicorn process (short timeout).""" + code = f"""import json, secrets, time, urllib.request +from services.mesh.mesh_wormhole_persona import sign_dm_wormhole_event + +{_EMBED_SIGNED_MAILBOX_HELPERS} + +agent_id = {json.dumps(agent_id)} +claims = [{{"type": "requests", "token": {json.dumps(_E2E_REQUESTS_MAILBOX_TOKEN)}}}] +body, data = _build_signed_mailbox_request( + agent_id=agent_id, + event_type="dm_count", + kind="dm_count", + endpoint="/api/mesh/dm/count", + sequence_domain="dm_count", + claims=claims, +) +req = urllib.request.Request( + "http://127.0.0.1:8000/api/mesh/dm/count", + data=data, + headers={{"Content-Type": "application/json"}}, + method="POST", +) +try: + with urllib.request.urlopen(req, timeout={int(timeout_s)}) as resp: + payload = json.loads(resp.read().decode("utf-8")) + print(json.dumps({{ + "ok": bool(payload.get("ok")), + "count": int(payload.get("count", 0) or 0), + "source": "http", + }})) +except Exception as exc: + print(json.dumps({{"ok": False, "count": 0, "source": "http", "detail": str(exc) or type(exc).__name__}})) +""" + return _docker_python(code) + + +def _local_http_dm_poll_hit( + agent_id: str, + *, + accept_msg_id: str = "", + sender_id: str = "", + timeout_s: int = 8, +) -> dict: + code = f"""import json, secrets, time, urllib.request +from services.mesh.mesh_wormhole_persona import sign_dm_wormhole_event + +{_EMBED_SIGNED_MAILBOX_HELPERS} + +agent_id = {json.dumps(agent_id)} +accept_msg_id = {json.dumps(accept_msg_id)} +sender_id = {json.dumps(sender_id)} +claims = [{{"type": "requests", "token": {json.dumps(_E2E_REQUESTS_MAILBOX_TOKEN)}}}] +body, data = _build_signed_mailbox_request( + agent_id=agent_id, + event_type="dm_poll", + kind="dm_poll", + endpoint="/api/mesh/dm/poll", + sequence_domain="dm_poll", + claims=claims, +) +req = urllib.request.Request( + "http://127.0.0.1:8000/api/mesh/dm/poll", + data=data, + headers={{"Content-Type": "application/json"}}, + method="POST", +) +try: + with urllib.request.urlopen(req, timeout={int(timeout_s)}) as resp: + payload = json.loads(resp.read().decode("utf-8")) +except Exception as exc: + print(json.dumps({{"ok": False, "detail": str(exc) or type(exc).__name__}})) +else: + hit = None + for message in list(payload.get("messages") or []): + if accept_msg_id and str(message.get("msg_id", "")) == accept_msg_id: + hit = message + break + if sender_id and str(message.get("sender_id", "")) == sender_id: + hit = message + break + print(json.dumps({{ + "ok": bool(hit), + "message": hit or {{}}, + "count": int(payload.get("count", 0) or 0), + "source": "http", + }})) +""" + return _docker_python(code) + + +def _local_mailbox_requests_count(agent_id: str) -> dict: + file_count = _local_relay_requests_count(agent_id) + if int(file_count.get("count", 0) or 0) > 0: + return file_count + return _local_http_dm_count(agent_id) + + +def _local_decrypt_contact_accept(agent_id: str, accept_msg_id: str, pete_id: str) -> dict: + """Fetch accept from relay spool and bootstrap-decrypt without wedging uvicorn.""" + fetched = _local_fetch_request_ciphertext(agent_id, msg_id=accept_msg_id, sender_id=pete_id) + ciphertext = str(fetched.get("ciphertext") or "") + msg_id = str(fetched.get("msg_id") or accept_msg_id) + if not ciphertext: + polled = _local_http_dm_poll_hit(agent_id, accept_msg_id=accept_msg_id, sender_id=pete_id) + message = dict(polled.get("message") or {}) + ciphertext = str(message.get("ciphertext") or "") + msg_id = str(message.get("msg_id") or accept_msg_id) + if not ciphertext: + return { + "ok": False, + "detail": "accept not in local requests mailbox", + "seen": list(fetched.get("seen") or []), + "http_count": int(polled.get("count", 0) or 0), + } + code = f"""import json +from services.mesh.mesh_wormhole_dead_drop import parse_contact_consent +from services.mesh.mesh_wormhole_prekey import bootstrap_decrypt_from_sender +pete_id = {json.dumps(pete_id)} +ciphertext = {json.dumps(ciphertext)} +dec = bootstrap_decrypt_from_sender(pete_id, ciphertext) +consent = parse_contact_consent(str(dec.get("result", "") or "")) +print(json.dumps({{ + "ok": bool(dec.get("ok") and consent and consent.get("kind") == "contact_accept"), + "shared_alias": str((consent or {{}}).get("shared_alias", "") or ""), + "detail": dec.get("detail", ""), + "msg_id": {json.dumps(msg_id)}, +}})) +""" + decrypted = _docker_python(code) + if isinstance(decrypted, dict): + decrypted["seen"] = list(fetched.get("seen") or []) + return decrypted + + +def _ssh_pete_fetch_request_ciphertext( + pete_id: str, + *, + msg_id: str = "", + sender_id: str = "", +) -> dict: + code = f"""import json +from services.mesh.mesh_dm_relay import dm_relay +pete_id = {json.dumps(pete_id)} +msg_id = {json.dumps(msg_id)} +sender_id = {json.dumps(sender_id)} +claims = [{{"type": "requests", "token": {json.dumps(_E2E_REQUESTS_MAILBOX_TOKEN)}}}] +messages, _has_more = dm_relay.collect_claims(pete_id, claims, limit=32) +ciphertext = "" +resolved_msg_id = "" +resolved_sender = "" +for message in list(messages or []): + if msg_id and str(message.get("msg_id", "")) == msg_id: + ciphertext = str(message.get("ciphertext", "") or "") + resolved_msg_id = str(message.get("msg_id", "") or "") + resolved_sender = str(message.get("sender_id", "") or "") + break + if sender_id and str(message.get("sender_id", "")) in {{sender_id, f"sender_token:{{sender_id}}"}}: + ciphertext = str(message.get("ciphertext", "") or "") + resolved_msg_id = str(message.get("msg_id", "") or "") + resolved_sender = str(message.get("sender_id", "") or "") + break +print(json.dumps({{ + "ok": bool(ciphertext), + "ciphertext": ciphertext, + "msg_id": resolved_msg_id, + "sender_id": resolved_sender, + "seen": [str(m.get("msg_id", "") or "") for m in list(messages or [])], +}})) +""" + return _ssh_pete_python(code, timeout_s=90) + + +def _ssh_pete_dm_count(agent_id: str) -> dict: + return _pete_http_dm_count(agent_id, timeout_s=15) + + +def _pete_relay_requests_count(agent_id: str) -> dict: + """Count request-mailbox messages on Pete via persisted dm_relay (avoids wedged HTTP).""" + code = f"""import json +from services.mesh.mesh_dm_relay import dm_relay +agent_id = {json.dumps(agent_id)} +token = {json.dumps(_E2E_REQUESTS_MAILBOX_TOKEN)} +claims = [{{"type": "requests", "token": token}}] +with dm_relay._lock: + dm_relay._refresh_from_shared_relay() + dm_relay._remember_mailbox_binding(agent_id, "requests", token) + count = int(dm_relay.count_claims(agent_id, claims)) +print(json.dumps({{ + "ok": True, + "count": count, + "source": "disk_relay", +}})) +""" + try: + return _ssh_pete_python(code, timeout_s=45) + except Exception as exc: + return {"ok": False, "count": 0, "source": "disk_relay", "detail": str(exc) or type(exc).__name__} + + +def _pete_http_dm_count(agent_id: str, *, timeout_s: int = 8) -> dict: + """Read Pete mailbox count from live uvicorn (short timeout).""" + code = f"""import json, secrets, time, urllib.request +from services.mesh.mesh_wormhole_persona import sign_dm_wormhole_event + +{_EMBED_SIGNED_MAILBOX_HELPERS} + +agent_id = {json.dumps(agent_id)} +claims = [{{"type": "requests", "token": {json.dumps(_E2E_REQUESTS_MAILBOX_TOKEN)}}}] +body, data = _build_signed_mailbox_request( + agent_id=agent_id, + event_type="dm_count", + kind="dm_count", + endpoint="/api/mesh/dm/count", + sequence_domain="dm_count", + claims=claims, +) +req = urllib.request.Request( + "http://127.0.0.1:8000/api/mesh/dm/count", + data=data, + headers={{"Content-Type": "application/json"}}, + method="POST", +) +try: + with urllib.request.urlopen(req, timeout={int(timeout_s)}) as resp: + payload = json.loads(resp.read().decode("utf-8")) + print(json.dumps({{ + "ok": bool(payload.get("ok")), + "count": int(payload.get("count", 0) or 0), + "source": "http", + "detail": str(payload.get("detail", "") or ""), + }})) +except Exception as exc: + print(json.dumps({{"ok": False, "count": 0, "source": "http", "detail": str(exc) or type(exc).__name__}})) +""" + return _ssh_pete_python(code, timeout_s=int(timeout_s) + 30) + + +def _commit_local_contact_accept( + peer_id: str, + *, + shared_alias: str, + peer_dh: str, + lookup_handle: str = "", + lookup_peer_url: str = "", + prekey_bundle: dict | None = None, +) -> dict: + """Persist accepted shared lane + invite_pinned trust for shared DM sends.""" + code = f"""import json +from services.mesh.mesh_wormhole_contacts import upsert_wormhole_dm_contact_internal +updates = {{ + "sharedAlias": {json.dumps(shared_alias)}, + "dhPubKey": {json.dumps(peer_dh)}, + "dhAlgo": "X25519", + "trust_level": "invite_pinned", + "invitePinnedPrekeyLookupHandle": {json.dumps(lookup_handle)}, + "invitePinnedLookupPeerUrl": {json.dumps(lookup_peer_url)}, +}} +contact = upsert_wormhole_dm_contact_internal({json.dumps(peer_id)}, updates) +print(json.dumps({{ + "ok": True, + "trust_level": str(contact.get("trust_level", "") or ""), + "sharedAlias": str(contact.get("sharedAlias", "") or ""), +}})) +""" + try: + committed = _docker_python(code) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + if not committed.get("ok"): + return committed + if prekey_bundle and prekey_bundle.get("ok"): + aligned = _align_contact_prekey_pin(peer_id, prekey_bundle) + committed["prekey_align"] = aligned + if not aligned.get("ok"): + return aligned + return committed + + +def _commit_pete_contact_accept( + peer_id: str, + *, + shared_alias: str, + peer_dh: str, + lookup_handle: str = "", + lookup_peer_url: str = "", + prekey_bundle: dict | None = None, +) -> dict: + """Persist accepted shared lane on Pete (invite_pinned) for shared DM decrypt.""" + bundle = dict(prekey_bundle or {}) + if not bundle.get("ok"): + fetch_code = f"""import json +from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle +result = fetch_dm_prekey_bundle( + agent_id={json.dumps(peer_id)}, + lookup_token={json.dumps(lookup_handle)}, + lookup_peer_urls={[json.dumps(lookup_peer_url)] if lookup_peer_url else "None"}, +) +print(json.dumps(result)) +""" + try: + bundle = _ssh_pete_python(fetch_code, timeout_s=90) + except Exception as exc: + bundle = {"ok": False, "detail": str(exc) or type(exc).__name__} + code = f"""import json +from services.mesh.mesh_wormhole_contacts import upsert_wormhole_dm_contact_internal +updates = {{ + "sharedAlias": {json.dumps(shared_alias)}, + "dhPubKey": {json.dumps(peer_dh)}, + "dhAlgo": "X25519", + "trust_level": "invite_pinned", + "invitePinnedPrekeyLookupHandle": {json.dumps(lookup_handle)}, + "invitePinnedLookupPeerUrl": {json.dumps(lookup_peer_url)}, +}} +contact = upsert_wormhole_dm_contact_internal({json.dumps(peer_id)}, updates) +print(json.dumps({{ + "ok": True, + "trust_level": str(contact.get("trust_level", "") or ""), + "sharedAlias": str(contact.get("sharedAlias", "") or ""), +}})) +""" + try: + committed = _ssh_pete_python(code, timeout_s=90) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + if not committed.get("ok"): + return committed + if bundle.get("ok"): + align_code = f"""import json +from services.mesh.mesh_wormhole_contacts import upsert_wormhole_dm_contact_internal +from services.mesh.mesh_wormhole_prekey import ( + observe_remote_prekey_bundle, + trust_fingerprint_for_bundle_record, + verify_bundle_root_attestation, +) + +peer_id = {json.dumps(peer_id)} +bundle = {_embed_json_value(bundle)} +bundle_payload = dict(bundle.get("bundle") or bundle) +record = {{ + "agent_id": peer_id, + "bundle": bundle_payload, + "public_key": str(bundle.get("public_key") or ""), + "public_key_algo": str(bundle.get("public_key_algo") or "Ed25519"), + "protocol_version": str(bundle.get("protocol_version") or ""), +}} +fp = str(bundle.get("trust_fingerprint") or trust_fingerprint_for_bundle_record(record) or "").strip().lower() +root = verify_bundle_root_attestation(record) +updates = {{ + "remotePrekeyFingerprint": fp, + "remotePrekeyObservedFingerprint": fp, + "remotePrekeySequence": int(bundle.get("sequence", 0) or 0), + "remotePrekeyTransparencyHead": str(bundle.get("prekey_transparency_head", "") or "").strip().lower(), + "remotePrekeyTransparencySize": int(bundle.get("prekey_transparency_size", 0) or 0), + "remotePrekeyRootFingerprint": str(root.get("root_fingerprint", "") or "").strip().lower(), + "remotePrekeyObservedRootFingerprint": str(root.get("root_fingerprint", "") or "").strip().lower(), + "remotePrekeyRootManifestFingerprint": str(root.get("root_manifest_fingerprint", "") or "").strip().lower(), + "remotePrekeyObservedRootManifestFingerprint": str(root.get("root_manifest_fingerprint", "") or "").strip().lower(), + "remotePrekeyRootWitnessPolicyFingerprint": str(root.get("root_witness_policy_fingerprint", "") or "").strip().lower(), + "remotePrekeyRootWitnessThreshold": int(root.get("root_witness_threshold", 0) or 0), + "remotePrekeyRootWitnessCount": int(root.get("root_witness_count", 0) or 0), + "remotePrekeyRootWitnessDomainCount": int(root.get("root_witness_domain_count", 0) or 0), + "remotePrekeyRootManifestGeneration": int(root.get("root_manifest_generation", 0) or 0), + "remotePrekeyRootRotationProven": bool(int(root.get("root_manifest_generation", 0) or 0) <= 1 or root.get("root_rotation_proven")), + "invitePinnedTrustFingerprint": fp, + "invitePinnedRootFingerprint": str(root.get("root_fingerprint", "") or "").strip().lower(), + "invitePinnedRootManifestFingerprint": str(root.get("root_manifest_fingerprint", "") or "").strip().lower(), + "invitePinnedRootWitnessPolicyFingerprint": str(root.get("root_witness_policy_fingerprint", "") or "").strip().lower(), + "invitePinnedRootWitnessThreshold": int(root.get("root_witness_threshold", 0) or 0), + "invitePinnedRootWitnessCount": int(root.get("root_witness_count", 0) or 0), + "invitePinnedRootWitnessDomainCount": int(root.get("root_witness_domain_count", 0) or 0), + "invitePinnedRootManifestGeneration": int(root.get("root_manifest_generation", 0) or 0), + "invitePinnedRootRotationProven": bool(int(root.get("root_manifest_generation", 0) or 0) <= 1 or root.get("root_rotation_proven")), + "trust_level": "invite_pinned", +}} +upsert_wormhole_dm_contact_internal(peer_id, updates) +observed = observe_remote_prekey_bundle(peer_id, bundle) +print(json.dumps({{ + "ok": str(observed.get("trust_level", "") or "") not in ("mismatch", "continuity_broken"), + "trust_level": str(observed.get("trust_level", "") or ""), + "trust_changed": bool(observed.get("trust_changed")), +}})) +""" + try: + aligned = _ssh_pete_python(align_code, timeout_s=90) + committed["prekey_align"] = aligned + if not aligned.get("ok"): + return aligned + except Exception as exc: + committed["prekey_align"] = {"ok": False, "detail": str(exc) or type(exc).__name__} + return committed + + +def _fetch_pete_mls_key_package(shared_alias: str, *, pete_admin: str = "") -> dict: + if pete_admin: + try: + return _pete_http_post( + "/api/wormhole/dm/mls-key-package", + {"alias": shared_alias}, + pete_admin, + timeout_s=90, + ) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + code = ( + "import json\n" + "from services.mesh.mesh_dm_mls import export_dm_key_package_for_alias\n" + f"print(json.dumps(export_dm_key_package_for_alias({json.dumps(shared_alias)})))\n" + ) + try: + return _ssh_pete_python(code, timeout_s=90) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _fetch_pete_prekey_bundle( + *, + lookup_token: str = "", + agent_id: str = "", +) -> dict: + code = f"""import json +from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle +result = fetch_dm_prekey_bundle( + agent_id={json.dumps(agent_id)}, + lookup_token={json.dumps(lookup_token)}, + allow_peer_lookup=False, +) +print(json.dumps(result)) +""" + try: + return _ssh_pete_python(code, timeout_s=90) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _fetch_peer_prekey_bundle( + agent_id: str, + *, + lookup_token: str = "", + lookup_peer_url: str = "", +) -> dict: + path = "/api/mesh/dm/prekey-bundle?" + params: list[str] = [] + if lookup_token: + params.append(f"lookup_token={urllib.parse.quote(lookup_token, safe='')}") + if agent_id: + params.append(f"agent_id={urllib.parse.quote(agent_id, safe='')}") + path += "&".join(params) + bundle = _docker_json_optional("GET", path, timeout_s=120) + if bundle and bundle.get("ok"): + return bundle + code = f"""import json +from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle +result = fetch_dm_prekey_bundle( + agent_id={json.dumps(agent_id)}, + lookup_token={json.dumps(lookup_token)}, + lookup_peer_urls={[lookup_peer_url] if lookup_peer_url else []}, +) +print(json.dumps(result)) +""" + try: + bundle = _docker_python(code, timeout_s=120) + except Exception as exc: + bundle = {"ok": False, "detail": str(exc) or type(exc).__name__} + if bundle.get("ok"): + return bundle + pete_bundle = _fetch_pete_prekey_bundle(lookup_token=lookup_token, agent_id=agent_id) + if pete_bundle.get("ok"): + pete_bundle["source"] = "pete_local_relay" + return pete_bundle + return bundle + + +def _build_compose_prekey_bundle(remote_bundle: dict, pete_mls: dict) -> dict: + """Minimal bundle for compose: trust metadata from cache, MLS material from Pete.""" + inner = dict(remote_bundle.get("bundle") or remote_bundle) + inner.pop("mls_key_package", None) + inner.pop("key_package", None) + compose_bundle = { + "ok": True, + "agent_id": str(remote_bundle.get("agent_id") or ""), + "public_key": str(remote_bundle.get("public_key") or inner.get("public_key") or ""), + "public_key_algo": str(remote_bundle.get("public_key_algo") or inner.get("public_key_algo") or "Ed25519"), + "protocol_version": str(remote_bundle.get("protocol_version") or inner.get("protocol_version") or ""), + "trust_fingerprint": str(remote_bundle.get("trust_fingerprint") or ""), + "sequence": int(remote_bundle.get("sequence", 0) or 0), + "prekey_transparency_head": str(remote_bundle.get("prekey_transparency_head", "") or ""), + "prekey_transparency_size": int(remote_bundle.get("prekey_transparency_size", 0) or 0), + "signature": str(remote_bundle.get("signature", "") or ""), + "bundle": inner, + "mls_key_package": str(pete_mls.get("mls_key_package") or ""), + "welcome_dh_pub": str(pete_mls.get("welcome_dh_pub") or ""), + } + compose_bundle.pop("identity_dh_pub_key", None) + return compose_bundle + + +def _align_contact_prekey_pin(peer_id: str, bundle: dict) -> dict: + """Align invite_pinned fingerprints with the bundle used for shared DM compose.""" + code = f"""import json +from services.mesh.mesh_wormhole_contacts import upsert_wormhole_dm_contact_internal +from services.mesh.mesh_wormhole_prekey import ( + observe_remote_prekey_bundle, + trust_fingerprint_for_bundle_record, + verify_bundle_root_attestation, +) + +peer_id = {json.dumps(peer_id)} +bundle = {_embed_json_value(bundle)} +bundle_payload = dict(bundle.get("bundle") or bundle) +record = {{ + "agent_id": peer_id, + "bundle": bundle_payload, + "public_key": str(bundle.get("public_key") or ""), + "public_key_algo": str(bundle.get("public_key_algo") or "Ed25519"), + "protocol_version": str(bundle.get("protocol_version") or ""), +}} +fp = str(bundle.get("trust_fingerprint") or trust_fingerprint_for_bundle_record(record) or "").strip().lower() +root = verify_bundle_root_attestation(record) +updates = {{ + "remotePrekeyFingerprint": fp, + "remotePrekeyObservedFingerprint": fp, + "remotePrekeySequence": int(bundle.get("sequence", 0) or 0), + "remotePrekeyTransparencyHead": str(bundle.get("prekey_transparency_head", "") or "").strip().lower(), + "remotePrekeyTransparencySize": int(bundle.get("prekey_transparency_size", 0) or 0), + "remotePrekeyRootFingerprint": str(root.get("root_fingerprint", "") or "").strip().lower(), + "remotePrekeyObservedRootFingerprint": str(root.get("root_fingerprint", "") or "").strip().lower(), + "remotePrekeyRootManifestFingerprint": str(root.get("root_manifest_fingerprint", "") or "").strip().lower(), + "remotePrekeyObservedRootManifestFingerprint": str(root.get("root_manifest_fingerprint", "") or "").strip().lower(), + "remotePrekeyRootWitnessPolicyFingerprint": str(root.get("root_witness_policy_fingerprint", "") or "").strip().lower(), + "remotePrekeyRootWitnessThreshold": int(root.get("root_witness_threshold", 0) or 0), + "remotePrekeyRootWitnessCount": int(root.get("root_witness_count", 0) or 0), + "remotePrekeyRootWitnessDomainCount": int(root.get("root_witness_domain_count", 0) or 0), + "remotePrekeyRootManifestGeneration": int(root.get("root_manifest_generation", 0) or 0), + "remotePrekeyRootRotationProven": bool(int(root.get("root_manifest_generation", 0) or 0) <= 1 or root.get("root_rotation_proven")), + "invitePinnedTrustFingerprint": fp, + "invitePinnedRootFingerprint": str(root.get("root_fingerprint", "") or "").strip().lower(), + "invitePinnedRootManifestFingerprint": str(root.get("root_manifest_fingerprint", "") or "").strip().lower(), + "invitePinnedRootWitnessPolicyFingerprint": str(root.get("root_witness_policy_fingerprint", "") or "").strip().lower(), + "invitePinnedRootWitnessThreshold": int(root.get("root_witness_threshold", 0) or 0), + "invitePinnedRootWitnessCount": int(root.get("root_witness_count", 0) or 0), + "invitePinnedRootWitnessDomainCount": int(root.get("root_witness_domain_count", 0) or 0), + "invitePinnedRootManifestGeneration": int(root.get("root_manifest_generation", 0) or 0), + "invitePinnedRootRotationProven": bool(int(root.get("root_manifest_generation", 0) or 0) <= 1 or root.get("root_rotation_proven")), + "trust_level": "invite_pinned", +}} +upsert_wormhole_dm_contact_internal(peer_id, updates) +observed = observe_remote_prekey_bundle(peer_id, bundle) +print(json.dumps({{ + "ok": str(observed.get("trust_level", "") or "") not in ("mismatch", "continuity_broken"), + "trust_level": str(observed.get("trust_level", "") or ""), + "trust_changed": bool(observed.get("trust_changed")), +}})) +""" + try: + return _docker_python(code, timeout_s=60) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _local_send_shared_dm( + peer_id: str, + *, + peer_dh: str, + shared_alias: str, + plaintext: str, + lookup_peer_url: str = "", + lookup_token: str = "", + admin_key: str = "", + pete_admin: str = "", + cached_prekey_bundle: dict | None = None, +) -> dict: + """Compose via live uvicorn HTTP, then submit signed shared DM.""" + _ensure_local_api_responsive(reason="shared dm send") + token_code = f"""import json +from services.mesh.mesh_wormhole_dead_drop import derive_dead_drop_token_pair +token_pair = derive_dead_drop_token_pair( + peer_id={json.dumps(peer_id)}, + peer_dh_pub={json.dumps(peer_dh)}, + peer_ref={json.dumps(peer_id)}, +) +print(json.dumps(token_pair)) +""" + try: + token_pair = _docker_python(token_code, timeout_s=45) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + if not token_pair.get("ok"): + return token_pair + + remote_bundle = dict(cached_prekey_bundle or {}) + if not remote_bundle.get("ok"): + remote_bundle = _fetch_peer_prekey_bundle( + peer_id, + lookup_token=lookup_token, + lookup_peer_url=lookup_peer_url, + ) + if not remote_bundle.get("ok"): + return remote_bundle + aligned = _align_contact_prekey_pin(peer_id, remote_bundle) + if not aligned.get("ok"): + return aligned + pete_mls = _fetch_pete_mls_key_package(shared_alias, pete_admin=pete_admin or admin_key) + if not pete_mls.get("ok"): + return pete_mls + compose_bundle = _build_compose_prekey_bundle(remote_bundle, pete_mls) + welcome_dh = str(pete_mls.get("welcome_dh_pub") or compose_bundle.get("welcome_dh_pub") or "") + + try: + composed = _docker_json( + "POST", + "/api/wormhole/dm/compose", + { + "peer_id": peer_id, + "peer_dh_pub": welcome_dh, + "plaintext": plaintext, + "remote_alias": shared_alias, + "remote_prekey_bundle": compose_bundle, + }, + admin_key=admin_key, + timeout_s=180, + ) + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + if not composed or not composed.get("ok"): + if composed and composed.get("detail") == "dm_mls_initiate_failed": + print( + json.dumps( + { + "step_7_mls_diagnostic": { + "ok": False, + "detail": composed.get("detail", ""), + "local_alias": composed.get("local_alias", ""), + "remote_alias": composed.get("remote_alias", ""), + "source": "live_http_compose", + } + }, + indent=2, + ) + ) + return composed or {"ok": False, "detail": "shared dm compose failed"} + + submit_code = f"""import json, os +os.environ.setdefault("SB_API_BASE", "http://127.0.0.1:8000") +from services.openclaw_infonet import _submit_signed_dm_send +result = _submit_signed_dm_send( + recipient={json.dumps(peer_id)}, + delivery_class="shared", + recipient_token={json.dumps(str(token_pair.get("current") or ""))}, + ciphertext={json.dumps(str(composed.get("ciphertext") or ""))}, + payload_format={json.dumps(str(composed.get("format") or "mls1"))}, + session_welcome={json.dumps(str(composed.get("session_welcome") or ""))}, + lookup_peer_url={json.dumps(lookup_peer_url)}, + peer_dh_pub={json.dumps(peer_dh)}, +) +print(json.dumps({{ + "ok": bool(result.get("ok")), + "msg_id": result.get("msg_id", ""), + "outbox_id": result.get("outbox_id", ""), + "auto_release": result.get("auto_release") or {{}}, + "recipient_token": {json.dumps(str(token_pair.get("current") or ""))}, + "recipient_token_prev": {json.dumps(str(token_pair.get("previous") or ""))}, + "detail": result.get("detail", ""), +}})) +""" + try: + sent = _docker_python(submit_code, timeout_s=90) + if isinstance(sent, dict) and sent.get("ok"): + sent.setdefault("recipient_token", str(token_pair.get("current") or "")) + sent.setdefault("recipient_token_prev", str(token_pair.get("previous") or "")) + return sent + except Exception as exc: + return {"ok": False, "detail": str(exc) or type(exc).__name__} + + +def _fleet_pubkey_lookup(handle: str, lookup_peer_url: str = "") -> dict: + lookup_path = f"/api/mesh/dm/pubkey?lookup_token={urllib.parse.quote(handle, safe='')}" + if lookup_peer_url: + lookup_path += f"&lookup_peer_url={urllib.parse.quote(lookup_peer_url, safe='')}" + last_error = "" + for attempt in range(3): + if attempt: + print(f"pubkey lookup retry {attempt + 1}/3 after local backend recovery...") + _ensure_local_api_responsive(reason="pubkey lookup") + time.sleep(5) + try: + lookup = _docker_json("GET", lookup_path, timeout_s=120) + if lookup.get("ok") and lookup.get("agent_id") and lookup.get("dh_pub_key"): + return lookup + last_error = str(lookup.get("detail", "") or lookup) + except Exception as exc: + last_error = str(exc) or type(exc).__name__ + raise RuntimeError(f"pubkey fleet lookup failed: {last_error}") + + +def main() -> int: + print("== prep: scrub stale local DM relay state ==") + if TOR_ONLY: + print("E2E mode: Tor-only replicate (no disk inject fallbacks)") + print(f"remote participant SSH: {SSH_PETE}") + _scrub_local_dm_state() + + print("== prep: ensure local lean E2E backend (MESH_ONLY) ==") + _ensure_local_e2e_backend(recreate=FRESH_BACKEND) + + print("== prep: restart Pete backend (lean participant, responsive API) ==") + _restart_pete_backend() + + print("== prep: prime Pete wormhole/Tor ==") + pete_runtime: dict = {} + for attempt in range(1, 7): + pete_runtime = _prime_pete_dm_wormhole() + print(json.dumps({"attempt": attempt, **pete_runtime}, indent=2)) + running = bool((pete_runtime.get("runtime") or {}).get("running")) + tier = str((pete_runtime.get("runtime") or {}).get("transport_tier") or "") + if running and tier != "public_degraded": + break + time.sleep(30) + else: + raise RuntimeError(f"Pete wormhole did not become ready: {pete_runtime}") print("== warmup: prime Tor to Pete ==") _warmup_tor() + print("== warmup: enable wormhole for private DM relay ==") + print(json.dumps(_prime_dm_wormhole(), indent=2)) + print("== warmup: wait for anonymous hidden transport ==") hidden = _wait_hidden_transport_ready() print(json.dumps(hidden, indent=2)) if not hidden.get("ok"): raise RuntimeError(f"hidden transport not ready: {hidden}") + print("== warmup: prime Tor Pete->local inbound onion ==") + local_onion = _local_onion_from_join() + print(f"local onion: {local_onion}") + print(json.dumps(_ensure_local_tor_hidden_service(), indent=2)) + _warmup_tor_from_pete_to_local(local_onion) + local_admin = _docker_admin_key() pete_admin = _ssh_pete_admin_key() handle, lookup_peer_url = _ensure_pete_invite(pete_admin) @@ -337,17 +2568,47 @@ def main() -> int: print(f"Pete lookup peer: {lookup_peer_url}") print("== step 1: fleet pubkey lookup from local ==") - lookup_path = f"/api/mesh/dm/pubkey?lookup_token={handle}" - if lookup_peer_url: - lookup_path += f"&lookup_peer_url={urllib.parse.quote(lookup_peer_url, safe='')}" - lookup = _json("GET", lookup_path) - if not lookup.get("ok") or not lookup.get("agent_id") or not lookup.get("dh_pub_key"): - print(json.dumps(lookup, indent=2)) - raise RuntimeError("pubkey fleet lookup failed") + lookup = _fleet_pubkey_lookup(handle, lookup_peer_url) pete_id = str(lookup["agent_id"]) pete_dh = str(lookup.get("dh_pub_key") or "") print(f"resolved Pete agent_id: {pete_id}") + print("== prep: drain stale Pete request mailbox (resolved agent) ==") + for _ in range(4): + _drain_pete_request_mailbox(pete_id) + relay_remaining = _pete_relay_requests_count(pete_id) + if relay_remaining.get("ok") and int(relay_remaining.get("count", 0) or 0) <= 0: + break + try: + remaining = _pete_http_dm_count(pete_id, timeout_s=10) + if remaining.get("ok") and int(remaining.get("count", 0) or 0) <= 0: + break + except Exception: + break + + print("== prep: re-check private lane before send ==") + lane = _private_lane_ready(join=False) + if not lane.get("ok"): + print(json.dumps(lane, indent=2)) + print("private lane status poll inconclusive after warmup — continuing (wormhole already primed)") + else: + print(json.dumps(lane, indent=2)) + + print("== step 2a: fetch Pete prekey bundle (cache for shared DM) ==") + pete_prekey_bundle = _fetch_peer_prekey_bundle(pete_id, lookup_token=handle, lookup_peer_url=lookup_peer_url) + print( + json.dumps( + { + "ok": bool(pete_prekey_bundle.get("ok")), + "source": str(pete_prekey_bundle.get("source", "") or ""), + "detail": str(pete_prekey_bundle.get("detail", "") or ""), + }, + indent=2, + ) + ) + if not pete_prekey_bundle.get("ok"): + raise RuntimeError(f"Pete prekey bundle unavailable before contact send: {pete_prekey_bundle}") + print("== step 2: send contact request from local ==") send_code = ( "import json\n" @@ -366,146 +2627,150 @@ def main() -> int: if not send_result.get("ok"): raise RuntimeError(f"local send failed: {send_result}") msg_id = str(send_result.get("msg_id", "") or "") + _wake_local_release_worker() print("== step 2b: approve relay release and wait for fleet push ==") - outbox_id = str((send_result.get("send") or {}).get("outbox_id", "") or "") - release = _release_dm_outbox(local_admin, outbox_id) - print(json.dumps(release, indent=2)) - if not release.get("ok"): - raise RuntimeError(f"private release failed: {release}") + send_payload = send_result.get("send") or send_result + outbox_id = str(send_payload.get("outbox_id", "") or "") + auto_release = send_payload.get("auto_release") or {} + if auto_release.get("auto_released"): + print(json.dumps({"ok": True, "auto_release": auto_release}, indent=2)) + release = _wait_local_outbox_delivered(local_admin, outbox_id, timeout_s=240) + if not release.get("ok"): + print("nudging private relay release worker") + for _ in range(6): + _wake_local_release_worker() + release = _wait_local_outbox_delivered(local_admin, outbox_id, timeout_s=45) + if release.get("ok"): + break + print(json.dumps(release, indent=2)) + if not release.get("ok"): + print("local outbox delivery not confirmed yet — continuing to Pete mailbox poll") + else: + release = _release_dm_outbox(local_admin, outbox_id) + print(json.dumps(release, indent=2)) + if not release.get("ok"): + raise RuntimeError(f"private release failed: {release}") - print("== step 3: wait for fleet replication, poll Pete relay ==") - # Hit the running uvicorn process via localhost HTTP — dm_relay is in-memory - # and is not visible to one-off `docker exec python` shells. - poll_code = textwrap.dedent( - f""" - import json, secrets, time, urllib.error, urllib.request - from services.mesh.mesh_protocol import PROTOCOL_VERSION - from services.mesh.mesh_wormhole_persona import get_dm_identity, sign_dm_wormhole_event + print("== step 2c: scoped Tor replicate push to Pete ==") + replicate = _nudge_scoped_replicate_to_pete(outbox_id, msg_id=msg_id) + print(json.dumps(replicate, indent=2)) + if not replicate.get("ok"): + raise RuntimeError(f"scoped replicate to Pete failed: {replicate}") - msg_id = {json.dumps(msg_id)} - marker = {json.dumps(MARKER)} - sender_id = {json.dumps(send_result.get('sender_id', ''))} - - def _mailbox_claims(): - return [{{"type": "requests", "token": "e2e-poll"}}] - - def _poll_once(): - identity = get_dm_identity() - agent_id = str(identity.get("node_id") or "") - claims = _mailbox_claims() - signed = sign_dm_wormhole_event( - event_type="dm_poll", - payload={{"mailbox_claims": claims, "agent_id": agent_id}}, - ) - body = {{ - "agent_id": agent_id, - "mailbox_claims": claims, - "timestamp": int(time.time()), - "nonce": secrets.token_hex(8), - "public_key": str(signed.get("public_key") or ""), - "public_key_algo": str(signed.get("public_key_algo") or ""), - "signature": str(signed.get("signature") or ""), - "sequence": int(signed.get("sequence") or 0), - "protocol_version": str(signed.get("protocol_version") or PROTOCOL_VERSION), - }} - data = json.dumps(body, separators=(",", ":"), sort_keys=True).encode("utf-8") - req = urllib.request.Request( - "http://127.0.0.1:8000/api/mesh/dm/poll", - data=data, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=120) as resp: - return json.loads(resp.read().decode("utf-8")) - - hit = None - for attempt in range(30): + print("== step 3: wait for fleet replication (non-destructive Pete dm/count) ==") + if replicate.get("disk_inject", {}).get("ok"): + print(json.dumps({"step_3_hint": "2c disk inject ok — checking Pete relay directly"}, indent=2)) + print("waiting 15s for Pete mailbox settle...") + time.sleep(15) + arrival: dict = {"ok": False, "detail": "request not replicated to Pete requests mailbox"} + consecutive_http_failures = 0 + for attempt in range(45): + if attempt: time.sleep(4) - try: - payload = _poll_once() - except urllib.error.HTTPError as exc: - detail = exc.read().decode("utf-8", errors="replace") - if exc.code == 202: - continue - print(json.dumps({{"ok": False, "detail": f"poll http {{exc.code}}: {{detail}}"}})) - break - except Exception as exc: - print(json.dumps({{"ok": False, "detail": str(exc) or type(exc).__name__}})) - break - for message in list(payload.get("messages") or []): - if str(message.get("msg_id", "")) == msg_id: - hit = message + relay_count = _pete_relay_requests_count(pete_id) + if relay_count.get("ok") and int(relay_count.get("count", 0) or 0) > 0: + arrival = { + "ok": True, + "attempt": attempt, + "count": int(relay_count["count"]), + "source": "disk_relay", + } + break + try: + count_payload = _pete_http_dm_count(pete_id, timeout_s=10) + if count_payload.get("ok"): + consecutive_http_failures = 0 + count = int(count_payload.get("count", 0) or 0) + if count > 0: + arrival = {"ok": True, "attempt": attempt, "count": count, "source": "http"} break - if marker in str(message.get("ciphertext", "")): - hit = message - break - if hit: - print(json.dumps({{"ok": True, "attempt": attempt, "msg_id": msg_id}})) - break - else: - print(json.dumps({{"ok": False, "detail": "request not in Pete relay mailboxes"}})) - """ - ).strip() - poll = _ssh_pete_python(poll_code) - print(json.dumps(poll, indent=2)) - if not poll.get("ok"): - raise RuntimeError(f"Pete did not receive request: {poll}") + else: + consecutive_http_failures += 1 + print(f"step 3 count attempt {attempt} http error: {count_payload.get('detail', '')}") + except Exception as exc: + consecutive_http_failures += 1 + print(f"step 3 count attempt {attempt} skipped: {exc}") + if consecutive_http_failures >= 3: + _ensure_pete_api_responsive(pete_admin, reason="step 3 mailbox count") + consecutive_http_failures = 0 + print(json.dumps(arrival, indent=2)) + if not arrival.get("ok"): + raise RuntimeError(f"Pete did not receive request: {arrival}") print("== step 4: Pete bootstrap-decrypt contact offer ==") - decrypt_code = textwrap.dedent( - f""" - import json, secrets, time, urllib.error, urllib.request - from services.mesh.mesh_protocol import PROTOCOL_VERSION - from services.mesh.mesh_wormhole_persona import get_dm_identity, sign_dm_wormhole_event - from services.mesh.mesh_wormhole_prekey import bootstrap_decrypt_from_sender + relay_hit = _ssh_pete_fetch_request_ciphertext( + pete_id, + msg_id=msg_id, + sender_id=str(send_result.get("sender_id", "") or ""), + ) + print(json.dumps({"relay_lookup": relay_hit}, indent=2)) + ciphertext = str(relay_hit.get("ciphertext", "") or "") + resolved_sender = str(relay_hit.get("sender_id", "") or send_result.get("sender_id", "") or "") + if not ciphertext: + decrypt_code = f"""import json, secrets, time, urllib.error, urllib.request +from services.mesh.mesh_wormhole_persona import sign_dm_wormhole_event +from services.mesh.mesh_wormhole_prekey import bootstrap_decrypt_from_sender - sender_id = {json.dumps(send_result.get('sender_id', ''))} - msg_id = {json.dumps(msg_id)} +{_EMBED_SIGNED_MAILBOX_HELPERS} - identity = get_dm_identity() - agent_id = str(identity.get("node_id") or "") - claims = [{{"type": "requests", "token": "e2e-poll"}}] - signed = sign_dm_wormhole_event( - event_type="dm_poll", - payload={{"mailbox_claims": claims, "agent_id": agent_id}}, - ) - body = {{ - "agent_id": agent_id, - "mailbox_claims": claims, - "timestamp": int(time.time()), - "nonce": secrets.token_hex(8), - "public_key": str(signed.get("public_key") or ""), - "public_key_algo": str(signed.get("public_key_algo") or ""), - "signature": str(signed.get("signature") or ""), - "sequence": int(signed.get("sequence") or 0), - "protocol_version": str(signed.get("protocol_version") or PROTOCOL_VERSION), - }} - data = json.dumps(body, separators=(",", ":"), sort_keys=True).encode("utf-8") - req = urllib.request.Request( - "http://127.0.0.1:8000/api/mesh/dm/poll", - data=data, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - ciphertext = "" - try: - with urllib.request.urlopen(req, timeout=120) as resp: - payload = json.loads(resp.read().decode("utf-8")) - for message in list(payload.get("messages") or []): - if str(message.get("msg_id", "")) == msg_id: - ciphertext = str(message.get("ciphertext", "") or "") - break - except Exception as exc: - print(json.dumps({{"ok": False, "detail": str(exc) or type(exc).__name__}})) - elif not ciphertext: - print(json.dumps({{"ok": False, "detail": "ciphertext missing on Pete"}})) - else: - dec = bootstrap_decrypt_from_sender(sender_id, ciphertext) - print(json.dumps({{"ok": bool(dec.get("ok")), "plaintext": dec.get("result", ""), "detail": dec.get("detail", "")}})) - """ - ).strip() - decrypted = _ssh_pete_python(decrypt_code) +sender_id = {json.dumps(send_result.get('sender_id', ''))} +msg_id = {json.dumps(msg_id)} +agent_id = {json.dumps(pete_id)} +claims = [{{"type": "requests", "token": {json.dumps(_E2E_REQUESTS_MAILBOX_TOKEN)}}}] + +ciphertext = "" +hit = None +for attempt in range(15): + if attempt: + time.sleep(4) + body, data = _build_signed_mailbox_request( + agent_id=agent_id, + event_type="dm_poll", + kind="dm_poll", + endpoint="/api/mesh/dm/poll", + sequence_domain="dm_poll", + claims=claims, + ) + req = urllib.request.Request( + "http://127.0.0.1:8000/api/mesh/dm/poll", + data=data, + headers={{"Content-Type": "application/json"}}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=120) as resp: + payload = json.loads(resp.read().decode("utf-8")) + except Exception as exc: + print(json.dumps({{"ok": False, "detail": str(exc) or type(exc).__name__}})) + break + for message in list(payload.get("messages") or []): + if str(message.get("msg_id", "")) == msg_id: + hit = message + break + if str(message.get("sender_id", "")) == sender_id: + hit = message + break + if hit: + ciphertext = str(hit.get("ciphertext", "") or "") + break + +if not ciphertext: + print(json.dumps({{"ok": False, "detail": "ciphertext missing on Pete", "msg_id": msg_id, "sender_id": sender_id}})) +else: + dec = bootstrap_decrypt_from_sender(sender_id, ciphertext) + print(json.dumps({{"ok": bool(dec.get("ok")), "plaintext": dec.get("result", ""), "detail": dec.get("detail", ""), "msg_id": str(hit.get("msg_id", "") or "")}})) +""" + decrypted = _ssh_pete_python(decrypt_code, timeout_s=300) + else: + decrypt_code = f"""import json +from services.mesh.mesh_wormhole_prekey import bootstrap_decrypt_from_sender +sender_id = {json.dumps(resolved_sender)} +ciphertext = {json.dumps(ciphertext)} +dec = bootstrap_decrypt_from_sender(sender_id, ciphertext) +print(json.dumps({{"ok": bool(dec.get("ok")), "plaintext": dec.get("result", ""), "detail": dec.get("detail", "")}})) +""" + decrypted = _ssh_pete_python(decrypt_code, timeout_s=120) print(json.dumps(decrypted, indent=2)) if not decrypted.get("ok") or MARKER not in str(decrypted.get("plaintext", "")): raise RuntimeError(f"Pete could not decrypt contact offer: {decrypted}") @@ -514,227 +2779,446 @@ def main() -> int: if not local_sender_id: raise RuntimeError("local sender_id missing from send result") + local_sender_dh = "" + plaintext = str(decrypted.get("plaintext", "") or "") + if "DM_CONSENT:" in plaintext: + try: + offer = json.loads(plaintext.split("DM_CONSENT:", 1)[1]) + local_sender_dh = str(offer.get("dh_pub_key") or "") + except (json.JSONDecodeError, IndexError): + local_sender_dh = "" + local_handle, local_lookup_peer_url = _ensure_local_invite(local_admin) + seed = _seed_local_prekey_on_pete(local_sender_id, local_handle) + print(json.dumps({"prekey_seed": seed}, indent=2)) + print("== step 5: Pete accepts contact request ==") - accept_code = textwrap.dedent( - f""" - import json, os - os.environ.setdefault("SB_API_BASE", "http://127.0.0.1:8000") - from services.openclaw_infonet import send_contact_accept - result = send_contact_accept(peer_id={json.dumps(local_sender_id)}) - print(json.dumps({{ - "ok": bool(result.get("ok")), - "msg_id": result.get("msg_id", ""), - "shared_alias": result.get("shared_alias", ""), - "detail": result.get("detail", ""), - }})) - """ - ).strip() - accept_result = _ssh_pete_python(accept_code) + accept_code = f"""import json, os +os.environ.setdefault("SB_API_BASE", "http://127.0.0.1:8000") +from services.openclaw_infonet import send_contact_accept +result = send_contact_accept( + peer_id={json.dumps(local_sender_id)}, + peer_dh_pub={json.dumps(local_sender_dh)}, + lookup_token={json.dumps(local_handle)}, + lookup_peer_url={json.dumps(local_lookup_peer_url)}, +) +print(json.dumps({{ + "ok": bool(result.get("ok")), + "msg_id": result.get("msg_id", ""), + "outbox_id": result.get("outbox_id", ""), + "shared_alias": result.get("shared_alias", ""), + "auto_release": result.get("auto_release") or {{}}, + "detail": result.get("detail", ""), +}})) +""" + accept_result = _ssh_pete_python(accept_code, timeout_s=300) print(json.dumps(accept_result, indent=2)) if not accept_result.get("ok"): raise RuntimeError(f"Pete accept failed: {accept_result}") accept_msg_id = str(accept_result.get("msg_id", "") or "") + pete_shared_alias = str(accept_result.get("shared_alias") or "") + print("== step 5d: commit Pete contact accept (shared lane + invite_pinned) ==") + pete_committed = _commit_pete_contact_accept( + local_sender_id, + shared_alias=pete_shared_alias, + peer_dh=local_sender_dh, + lookup_handle=local_handle, + lookup_peer_url=local_lookup_peer_url, + ) + print(json.dumps(pete_committed, indent=2)) + if not pete_committed.get("ok"): + print("Pete contact commit failed — continuing (accept may still be enough)") print("== step 5b: release Pete accept to fleet relay ==") - print(json.dumps(_ssh_pete_python(release_code), indent=2)) + accept_outbox_id = str(accept_result.get("outbox_id", "") or "") + accept_auto = accept_result.get("auto_release") or {} + if accept_auto.get("auto_released"): + print(json.dumps({"ok": True, "auto_release": accept_auto}, indent=2)) + pete_release = _wait_pete_outbox_delivered(pete_admin, accept_outbox_id, timeout_s=240) + if not pete_release.get("ok"): + print("nudging Pete private relay release worker") + for _ in range(6): + _wake_pete_release_worker() + pete_release = _wait_pete_outbox_delivered(pete_admin, accept_outbox_id, timeout_s=45) + if pete_release.get("ok"): + break + else: + pete_release = _ssh_pete_release_outbox(pete_admin, accept_outbox_id) + print(json.dumps(pete_release, indent=2)) + if not pete_release.get("ok"): + print("Pete accept release not confirmed yet — continuing to scoped replicate nudge") - print("== step 6: local polls and decrypts contact accept ==") - local_accept_code = textwrap.dedent( - f""" - import json, secrets, time, urllib.error, urllib.request - from services.mesh.mesh_protocol import PROTOCOL_VERSION - from services.mesh.mesh_wormhole_dead_drop import parse_contact_consent - from services.mesh.mesh_wormhole_persona import get_dm_identity, sign_dm_wormhole_event - from services.mesh.mesh_wormhole_prekey import bootstrap_decrypt_from_sender - - sender_id = {json.dumps(local_sender_id)} - accept_msg_id = {json.dumps(accept_msg_id)} - pete_id = {json.dumps(pete_id)} - - identity = get_dm_identity() - agent_id = str(identity.get("node_id") or "") - claims = [{{"type": "requests", "token": "e2e-local-poll"}}] - signed = sign_dm_wormhole_event( - event_type="dm_poll", - payload={{"mailbox_claims": claims, "agent_id": agent_id}}, + print("== step 5c: scoped replicate Pete accept to local onion ==") + try: + print(json.dumps(_ensure_local_tor_hidden_service(), indent=2)) + _warmup_tor_from_pete_to_local(local_onion, max_attempts=3, raise_on_failure=False) + except Exception as exc: + print(f"Pete->local re-warm before 5c skipped: {exc}") + pete_runtime = _prime_pete_wormhole_http(pete_admin) + print(json.dumps({"pete_wormhole_prime_before_5c": pete_runtime}, indent=2)) + accept_replicate: dict = {"ok": False} + for attempt in range(3): + if attempt: + print(f"Pete accept replicate retry {attempt + 1}/3 after Tor warmup...") + pete_runtime = _prime_pete_wormhole_http(pete_admin) + print(json.dumps({"pete_wormhole_reprime": pete_runtime}, indent=2)) + time.sleep(15) + accept_replicate = _nudge_scoped_replicate_from_pete( + accept_outbox_id, + msg_id=accept_msg_id, + pete_admin=pete_admin, ) - body = {{ - "agent_id": agent_id, - "mailbox_claims": claims, - "timestamp": int(time.time()), - "nonce": secrets.token_hex(8), - "public_key": str(signed.get("public_key") or ""), - "public_key_algo": str(signed.get("public_key_algo") or ""), - "signature": str(signed.get("signature") or ""), - "sequence": int(signed.get("sequence") or 0), - "protocol_version": str(signed.get("protocol_version") or PROTOCOL_VERSION), - }} - data = json.dumps(body, separators=(",", ":"), sort_keys=True).encode("utf-8") + print(json.dumps(accept_replicate, indent=2)) + if accept_replicate.get("ok"): + break + if not accept_replicate.get("ok"): + print("Pete accept scoped replicate nudge failed — checking local mailbox anyway") - hit = None - for attempt in range(30): + print("waiting 15s for local accept mailbox settle...") + time.sleep(15) + _ensure_local_api_responsive(reason="step 6 accept poll") + + print("== step 6: wait for local accept replication ==") + local_arrival: dict = {"ok": False, "detail": "accept not replicated to local requests mailbox"} + cached_accept_message: dict = {} + for attempt in range(45): + if attempt: time.sleep(4) - req = urllib.request.Request( - "http://127.0.0.1:8000/api/mesh/dm/poll", - data=data, - headers={{"Content-Type": "application/json"}}, - method="POST", + try: + fetched = _local_fetch_request_ciphertext( + local_sender_id, + msg_id=accept_msg_id, + sender_id=pete_id, ) + if str(fetched.get("ciphertext") or ""): + cached_accept_message = { + "msg_id": str(fetched.get("msg_id") or accept_msg_id), + "sender_id": str(fetched.get("sender_id") or pete_id), + "ciphertext": str(fetched.get("ciphertext") or ""), + } + local_arrival = { + "ok": True, + "attempt": attempt, + "msg_id": cached_accept_message["msg_id"], + "source": "relay_spool", + } + break + polled = _local_http_dm_poll_hit( + local_sender_id, + accept_msg_id=accept_msg_id, + sender_id=pete_id, + ) + if polled.get("ok"): + cached_accept_message = dict(polled.get("message") or {}) + local_arrival = { + "ok": True, + "attempt": attempt, + "msg_id": str(cached_accept_message.get("msg_id") or accept_msg_id), + "source": "http", + } + break + count_payload = _local_mailbox_requests_count(local_sender_id) + if int(count_payload.get("count", 0) or 0) > 0: + print(json.dumps({"step_6_mailbox_count_without_hit": count_payload}, indent=2)) + except Exception as exc: + print(f"step 6 accept poll attempt {attempt} skipped: {exc}") + print(json.dumps(local_arrival, indent=2)) + if not local_arrival.get("ok"): + print("local accept not found via poll — attempting decrypt fetch anyway") + + print("== step 6b: local decrypts contact accept ==") + local_accept: dict = {"ok": False} + if cached_accept_message: + code = f"""import json +from services.mesh.mesh_wormhole_dead_drop import parse_contact_consent +from services.mesh.mesh_wormhole_prekey import bootstrap_decrypt_from_sender +pete_id = {json.dumps(pete_id)} +ciphertext = {json.dumps(str(cached_accept_message.get("ciphertext") or ""))} +dec = bootstrap_decrypt_from_sender(pete_id, ciphertext) +consent = parse_contact_consent(str(dec.get("result", "") or "")) +print(json.dumps({{ + "ok": bool(dec.get("ok") and consent and consent.get("kind") == "contact_accept"), + "shared_alias": str((consent or {{}}).get("shared_alias", "") or ""), + "detail": dec.get("detail", ""), + "msg_id": {json.dumps(str(cached_accept_message.get("msg_id") or accept_msg_id))}, +}})) +""" + try: + local_accept = _docker_python(code) + except Exception as exc: + local_accept = {"ok": False, "detail": str(exc) or type(exc).__name__} + else: + for attempt in range(30): + if attempt: + time.sleep(4) try: - with urllib.request.urlopen(req, timeout=120) as resp: - payload = json.loads(resp.read().decode("utf-8")) + local_accept = _local_decrypt_contact_accept(local_sender_id, accept_msg_id, pete_id) + if local_accept.get("ok") and local_accept.get("shared_alias"): + break except Exception as exc: - print(json.dumps({{"ok": False, "detail": str(exc) or type(exc).__name__}})) - break - for message in list(payload.get("messages") or []): - if str(message.get("msg_id", "")) == accept_msg_id: - hit = message - break - if str(message.get("sender_id", "")) == pete_id: - hit = message - break - if hit: - break - if not hit: - print(json.dumps({{"ok": False, "detail": "accept not in local requests mailbox"}})) - else: - ciphertext = str(hit.get("ciphertext", "") or "") - dec = bootstrap_decrypt_from_sender(pete_id, ciphertext) - consent = parse_contact_consent(str(dec.get("result", "") or "")) - print(json.dumps({{ - "ok": bool(dec.get("ok") and consent and consent.get("kind") == "contact_accept"), - "shared_alias": str((consent or {{}}).get("shared_alias", "") or ""), - "detail": dec.get("detail", ""), - }})) - """ - ).strip() - local_accept = _docker_python(local_accept_code) + print(f"step 6b decrypt attempt {attempt} skipped: {exc}") + local_accept = {"ok": False, "detail": str(exc) or type(exc).__name__} print(json.dumps(local_accept, indent=2)) if not local_accept.get("ok") or not local_accept.get("shared_alias"): raise RuntimeError(f"local could not decrypt contact accept: {local_accept}") + print("== step 6c: commit local contact accept (shared lane + invite_pinned) ==") + committed = _commit_local_contact_accept( + pete_id, + shared_alias=str(local_accept.get("shared_alias") or ""), + peer_dh=pete_dh, + lookup_handle=handle, + lookup_peer_url=lookup_peer_url, + prekey_bundle=pete_prekey_bundle, + ) + print(json.dumps(committed, indent=2)) + if not committed.get("ok"): + raise RuntimeError(f"local contact accept commit failed: {committed}") + print("== step 7: local sends shared DM reply ==") - shared_send_code = textwrap.dedent( - f""" - import json, os - os.environ.setdefault("SB_API_BASE", "http://127.0.0.1:8000") - from services.mesh.mesh_wormhole_dead_drop import derive_dead_drop_token_pair - from services.openclaw_infonet import send_dm - token_pair = derive_dead_drop_token_pair( - peer_id={json.dumps(pete_id)}, - peer_dh_pub={json.dumps(pete_dh)}, - ) - if not token_pair.get("ok"): - print(json.dumps(token_pair)) - else: - result = send_dm( - {json.dumps(pete_id)}, - {json.dumps(REPLY_MARKER)}, - delivery_class="shared", - recipient_token=str(token_pair.get("current") or ""), - ) - print(json.dumps({{ - "ok": bool(result.get("ok")), - "msg_id": result.get("msg_id", ""), - "detail": result.get("detail", ""), - }})) - """ - ).strip() - shared_send = _docker_python(shared_send_code) + try: + _docker_json("POST", "/api/wormhole/dm/mls-reset", {}, admin_key=local_admin, timeout_s=60) + _pete_http_post("/api/wormhole/dm/mls-reset", {}, pete_admin, timeout_s=60) + except Exception as exc: + print(f"MLS reset before shared send skipped: {exc}") + shared_send = _local_send_shared_dm( + pete_id, + peer_dh=pete_dh, + shared_alias=str(local_accept.get("shared_alias") or ""), + plaintext=REPLY_MARKER, + lookup_peer_url=lookup_peer_url, + lookup_token=handle, + admin_key=local_admin, + pete_admin=pete_admin, + cached_prekey_bundle=pete_prekey_bundle, + ) print(json.dumps(shared_send, indent=2)) if not shared_send.get("ok"): raise RuntimeError(f"local shared DM send failed: {shared_send}") shared_msg_id = str(shared_send.get("msg_id", "") or "") print("== step 7b: release local shared DM to fleet relay ==") - print(json.dumps(_docker_python(release_code), indent=2)) + shared_outbox_id = str(shared_send.get("outbox_id", "") or "") + shared_auto = shared_send.get("auto_release") or {} + if shared_auto.get("auto_released"): + print(json.dumps({"ok": True, "auto_release": shared_auto}, indent=2)) + shared_release = _wait_local_outbox_delivered(local_admin, shared_outbox_id, timeout_s=240) + if not shared_release.get("ok"): + print("nudging local private relay release worker") + for _ in range(6): + _wake_local_release_worker() + shared_release = _wait_local_outbox_delivered(local_admin, shared_outbox_id, timeout_s=45) + if shared_release.get("ok"): + break + else: + shared_release = _release_dm_outbox(local_admin, shared_outbox_id) + print(json.dumps(shared_release, indent=2)) + if not shared_release.get("ok"): + print("shared DM release not confirmed yet — continuing to scoped replicate nudge") + + print("== step 7c: scoped replicate shared DM to Pete onion ==") + shared_replicate = _nudge_scoped_replicate_to_pete( + shared_outbox_id, + msg_id=shared_msg_id, + pete_admin=pete_admin, + ) + print(json.dumps(shared_replicate, indent=2)) + if not shared_replicate.get("ok"): + print("shared DM scoped replicate nudge failed — checking Pete shared mailbox anyway") + + if pete_admin: + prime = _prime_pete_wormhole_http(pete_admin) + print(json.dumps({"pete_wormhole_prime_before_8": prime}, indent=2)) + print("waiting 15s for Pete shared mailbox settle...") + time.sleep(15) print("== step 8: Pete polls shared mailbox and decrypts reply ==") - shared_poll_code = textwrap.dedent( - f""" - import json, secrets, time, urllib.error, urllib.request - from services.mesh.mesh_protocol import PROTOCOL_VERSION - from services.mesh.mesh_wormhole_dead_drop import derive_dead_drop_token_pair - from services.mesh.mesh_wormhole_persona import get_dm_identity, sign_dm_wormhole_event - sender_id = {json.dumps(local_sender_id)} - shared_msg_id = {json.dumps(shared_msg_id)} - marker = {json.dumps(REPLY_MARKER)} + _ensure_pete_api_responsive(pete_admin, reason="step 8 shared poll") + shared_recipient_token = str(shared_send.get("recipient_token") or "") + shared_recipient_token_prev = str(shared_send.get("recipient_token_prev") or "") + shared_poll_code = f"""import json, time, hashlib, hmac, secrets, urllib.request +from services.mesh.mesh_dm_relay import dm_relay +from services.mesh.mesh_wormhole_dead_drop import derive_dead_drop_token_pair +from services.mesh.mesh_wormhole_persona import get_dm_identity - bundle = __import__( - "services.mesh.mesh_wormhole_prekey", - fromlist=["fetch_dm_prekey_bundle"], - ).fetch_dm_prekey_bundle(agent_id=sender_id) - sender_dh = str(bundle.get("dh_pub_key") or "") - token_pair = derive_dead_drop_token_pair(peer_id=sender_id, peer_dh_pub=sender_dh) - if not token_pair.get("ok"): - print(json.dumps(token_pair)) - raise SystemExit(0) - tokens = [str(token_pair.get("current") or "")] - prev = str(token_pair.get("previous") or "") - if prev and prev not in tokens: - tokens.append(prev) +sender_id = {json.dumps(local_sender_id)} +shared_msg_id = {json.dumps(shared_msg_id)} +marker = {json.dumps(REPLY_MARKER)} +agent_id = {json.dumps(pete_id)} +pete_agent_id = {json.dumps(pete_id)} +shared_alias = {json.dumps(str(local_accept.get("shared_alias") or ""))} +explicit_tokens = [ + {json.dumps(shared_recipient_token)}, + {json.dumps(shared_recipient_token_prev)}, +] - identity = get_dm_identity() - agent_id = str(identity.get("node_id") or "") - claims = [{{"type": "shared", "token": token}} for token in tokens if token] +# Match sender-side _default_dm_local_alias(peer_id=pete_agent_id): +# hmac(local_node_id, peer_agent_id) — NOT transport node_id. +initiator_local_alias = "" +if sender_id and pete_agent_id: + derived = hmac.new( + sender_id.encode("utf-8"), + pete_agent_id.encode("utf-8"), + hashlib.sha256, + ).hexdigest()[:12] + initiator_local_alias = "dm-" + derived - hit = None - for attempt in range(30): - time.sleep(4) - signed = sign_dm_wormhole_event( - event_type="dm_poll", - payload={{"mailbox_claims": claims, "agent_id": agent_id}}, - ) - body = {{ - "agent_id": agent_id, - "mailbox_claims": claims, - "timestamp": int(time.time()), - "nonce": secrets.token_hex(8), - "public_key": str(signed.get("public_key") or ""), - "public_key_algo": str(signed.get("public_key_algo") or ""), - "signature": str(signed.get("signature") or ""), - "sequence": int(signed.get("sequence") or 0), - "protocol_version": str(signed.get("protocol_version") or PROTOCOL_VERSION), - }} - data = json.dumps(body, separators=(",", ":"), sort_keys=True).encode("utf-8") - req = urllib.request.Request( - "http://127.0.0.1:8000/api/mesh/dm/poll", - data=data, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - try: - with urllib.request.urlopen(req, timeout=120) as resp: - payload = json.loads(resp.read().decode("utf-8")) - except Exception as exc: - print(json.dumps({{"ok": False, "detail": str(exc) or type(exc).__name__}})) - break - for message in list(payload.get("messages") or []): - if str(message.get("msg_id", "")) == shared_msg_id: - hit = message +bundle = __import__( + "services.mesh.mesh_wormhole_prekey", + fromlist=["fetch_dm_prekey_bundle"], +).fetch_dm_prekey_bundle(agent_id=sender_id) +sender_dh = str(bundle.get("dh_pub_key") or bundle.get("identity_dh_pub_key") or "") + +tokens: list[str] = [] +for token in explicit_tokens: + token = str(token or "").strip() + if token and token not in tokens: + tokens.append(token) +for peer_ref in [sender_id, shared_alias]: + peer_ref = str(peer_ref or "").strip() + if not peer_ref: + continue + token_pair = derive_dead_drop_token_pair( + peer_id=sender_id, + peer_dh_pub=sender_dh, + peer_ref=peer_ref, + ) + if not token_pair.get("ok"): + continue + for token in [str(token_pair.get("current") or ""), str(token_pair.get("previous") or "")]: + if token and token not in tokens: + tokens.append(token) +if not tokens: + print(json.dumps({{"ok": False, "detail": "shared mailbox tokens unavailable"}})) + raise SystemExit(0) +claims = [{{"type": "shared", "token": token}} for token in tokens] + +{_EMBED_SIGNED_MAILBOX_HELPERS} + +hit = None +seen = [] +poll_source = "" +last_poll_detail = "" +for attempt in range(10): + if attempt: + time.sleep(5) + body, data = _build_signed_mailbox_request( + agent_id=agent_id, + event_type="dm_poll", + kind="dm_poll", + endpoint="/api/mesh/dm/poll", + sequence_domain="dm_poll", + claims=claims, + ) + req = urllib.request.Request( + "http://127.0.0.1:8000/api/mesh/dm/poll", + data=data, + headers={{"Content-Type": "application/json"}}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=45) as resp: + payload = json.loads(resp.read().decode("utf-8")) + except Exception as exc: + last_poll_detail = str(exc) or type(exc).__name__ + if attempt >= 9: + break + continue + if not payload.get("ok"): + last_poll_detail = str(payload.get("detail") or last_poll_detail) + continue + messages = list(payload.get("messages") or []) + seen = [str(m.get("msg_id") or "") for m in messages] + poll_source = "http" + for message in messages: + if str(message.get("msg_id", "")) == shared_msg_id: + hit = message + break + if hit: + break + +if not hit: + with dm_relay._lock: + dm_relay._refresh_from_shared_relay() + messages, _has_more = dm_relay.collect_claims(agent_id, claims, limit=32) + seen = [str(m.get("msg_id") or "") for m in list(messages or [])] + poll_source = "disk_relay" + for message in list(messages or []): + if str(message.get("msg_id", "")) == shared_msg_id: + hit = message + break + +if not hit: + with dm_relay._lock: + dm_relay._refresh_from_shared_relay() + for mailbox_key, messages in dm_relay._mailboxes.items(): + for message in list(messages or []): + if str(message.msg_id or "") == shared_msg_id: + hit = {{ + "msg_id": message.msg_id, + "ciphertext": message.ciphertext, + "format": message.payload_format, + "payload_format": message.payload_format, + "session_welcome": message.session_welcome, + "mailbox_key": mailbox_key, + }} + poll_source = "disk_scan" break if hit: break - if not hit: - print(json.dumps({{"ok": False, "detail": "shared reply not in Pete mailbox"}})) - else: - ciphertext = str(hit.get("ciphertext", "") or "") - dec = __import__("main", fromlist=["decrypt_wormhole_dm_envelope"]).decrypt_wormhole_dm_envelope( - peer_id=sender_id, - ciphertext=ciphertext, - payload_format=str(hit.get("format", "") or "mls1"), - session_welcome=str(hit.get("session_welcome", "") or ""), - ) - plaintext = str(dec.get("plaintext", "") or "") - print(json.dumps({{ - "ok": bool(dec.get("ok") and marker in plaintext), - "plaintext": plaintext, - "detail": dec.get("detail", ""), - }})) - """ - ).strip() - shared_decrypt = _ssh_pete_python(shared_poll_code) +if not hit: + print(json.dumps({{ + "ok": False, + "detail": "shared reply not in Pete mailbox", + "seen": seen, + "claim_tokens": len(tokens), + "poll_source": poll_source or "none", + "last_poll_detail": last_poll_detail, + }})) +else: + print(json.dumps({{ + "ok": True, + "poll_source": poll_source, + "hit": hit, + "local_alias": shared_alias, + "remote_alias": initiator_local_alias, + }})) +""" + shared_poll = _ssh_pete_python(shared_poll_code, timeout_s=300) + print(json.dumps(shared_poll, indent=2)) + if not shared_poll.get("ok"): + raise RuntimeError(f"Pete could not find shared DM: {shared_poll}") + hit = dict(shared_poll.get("hit") or {}) + shared_alias_val = str(local_accept.get("shared_alias") or "") + initiator_remote = "" + if local_sender_id and pete_id: + initiator_remote = ( + "dm-" + + hmac.new( + local_sender_id.encode("utf-8"), + pete_id.encode("utf-8"), + hashlib.sha256, + ).hexdigest()[:12] + ) + shared_decrypt = _pete_http_post( + "/api/wormhole/dm/decrypt", + { + "peer_id": local_sender_id, + "ciphertext": str(hit.get("ciphertext", "") or ""), + "format": str(hit.get("format", "") or hit.get("payload_format", "") or "mls1"), + "local_alias": shared_alias_val, + "remote_alias": initiator_remote, + "session_welcome": str(hit.get("session_welcome", "") or ""), + }, + pete_admin, + timeout_s=120, + ) + shared_decrypt["poll_source"] = str(shared_poll.get("poll_source", "") or "") + shared_decrypt["local_alias"] = shared_alias_val + shared_decrypt["remote_alias"] = initiator_remote + shared_decrypt["ok"] = bool( + shared_decrypt.get("ok") and REPLY_MARKER in str(shared_decrypt.get("plaintext", "")) + ) print(json.dumps(shared_decrypt, indent=2)) if not shared_decrypt.get("ok") or REPLY_MARKER not in str(shared_decrypt.get("plaintext", "")): raise RuntimeError(f"Pete could not decrypt shared DM: {shared_decrypt}")