Shadowbroker/backend/services/fetchers/financial.py

import logging
import math
import random
import time
import os
import urllib.request
import json
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
from services.fetchers.retry import with_retry

logger = logging.getLogger(__name__)

_YFINANCE_REQUEST_DELAY_SECONDS = 0.5
_YFINANCE_REQUEST_JITTER_SECONDS = 0.2

TICKERS_DEFENSE = ["RTX", "LMT", "NOC", "GD", "BA", "PLTR"]
TICKERS_TECH = ["NVDA", "AMD", "TSM", "INTC", "GOOGL", "AMZN", "MSFT", "AAPL", "TSLA", "META", "NFLX", "SMCI", "ARM", "ASML"]
TICKERS_CRYPTO = [
    ("BTC", "BINANCE:BTCUSDT", "BTC-USD"),
    ("ETH", "BINANCE:ETHUSDT", "ETH-USD"),
    ("SOL", "BINANCE:SOLUSDT", "SOL-USD"),
    ("XRP", "BINANCE:XRPUSDT", "XRP-USD"),
    ("ADA", "BINANCE:ADAUSDT", "ADA-USD"),
]

# Ticker priority for high-frequency updates (we update these every tick)
PRIORITY_SYMBOLS = ["BTC", "ETH", "NVDA", "PLTR"]

# Persistence for state between short-lived scheduler ticks
_last_fetch_results = {}
_last_fetch_time = 0.0
_rotating_index = 0
_executor = ThreadPoolExecutor(max_workers=10)


def _fetch_finnhub_quote(symbol: str, api_key: str):
    """Fetch from Finnhub. Returns (symbol, data) or (symbol, None)."""
    url = f"https://finnhub.io/api/v1/quote?symbol={symbol}&token={api_key}"
    try:
        req = urllib.request.Request(url)
        with urllib.request.urlopen(req, timeout=5) as response:
            data = json.loads(response.read().decode())
            if "c" not in data or data["c"] == 0:
                return symbol, None
            current = float(data["c"])
            change_p = float(data.get("dp", 0.0) or 0.0)
            return symbol, {
                "price": round(current, 2),
                "change_percent": round(change_p, 2),
                "up": bool(change_p >= 0),
            }
    except Exception as e:
        logger.debug(f"Finnhub error for {symbol}: {e}")
        return symbol, None


def _fetch_yfinance_single(symbol: str, period: str = "2d"):
    """Fetch from yfinance. Returns (symbol, data) or (symbol, None)."""
    try:
        import yfinance as yf
        ticker = yf.Ticker(symbol)
        hist = ticker.history(period=period)
        if len(hist) >= 1:
            current_price = hist["Close"].iloc[-1]
            prev_close = hist["Close"].iloc[0] if len(hist) > 1 else current_price
            change_percent = ((current_price - prev_close) / prev_close) * 100 if prev_close else 0
            current_price_f = float(current_price)
            change_percent_f = float(change_percent)
            if not math.isfinite(current_price_f) or not math.isfinite(change_percent_f):
                return symbol, None
            return symbol, {
                "price": round(current_price_f, 2),
                "change_percent": round(change_percent_f, 2),
                "up": bool(change_percent_f >= 0),
            }
    except Exception as e:
        logger.debug(f"Yfinance error for {symbol}: {e}")
    return symbol, None


@with_retry(max_retries=1, base_delay=1)
def financial_fetch_enabled() -> bool:
    """Return True only when the operator explicitly opts into financial pulls.

    Either ``FINANCIAL_ENABLED=true`` or the presence of ``FINNHUB_API_KEY``
    counts as an explicit opt-in. Without either, the default yfinance path
    is disabled to avoid silent outbound calls to finance.yahoo.com.
    """
    if os.getenv("FINNHUB_API_KEY", "").strip():
        return True
    return str(os.environ.get("FINANCIAL_ENABLED", "")).strip().lower() in {
        "1",
        "true",
        "yes",
        "on",
    }


def fetch_financial_markets():
    """Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance)."""
    global _last_fetch_time, _last_fetch_results, _rotating_index

    if not financial_fetch_enabled():
        logger.debug(
            "Financial fetch skipped; set FINANCIAL_ENABLED=true or supply "
            "FINNHUB_API_KEY to opt in"
        )
        with _data_lock:
            latest_data["financial"] = {}
        _mark_fresh("financial")
        return

    finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip()
    use_finnhub = bool(finnhub_key)
    
    now = time.time()
    # Throttle logic: 3s for Finnhub, 60s for yfinance fallback
    throttle_s = 3.0 if use_finnhub else 60.0
    
    if now - _last_fetch_time < throttle_s and _last_fetch_results:
        return # Skip if too frequent

    _last_fetch_time = now
    
    # Prepare symbol lists
    all_crypto = {label: (f_sym, y_sym) for label, f_sym, y_sym in TICKERS_CRYPTO}
    all_stocks = TICKERS_TECH + TICKERS_DEFENSE
    
    subset_to_fetch = []
    
    if use_finnhub:
        # Finnhub Free Limit: 60/min. 
        # Ticking every 3s = 20 ticks/min. 
        # To stay safe, we fetch only ~3 items per tick.
        # Priority items (BTC, ETH) + 1 rotating item.
        subset_to_fetch = ["BINANCE:BTCUSDT", "BINANCE:ETHUSDT"]
        
        # Determine rotating ticker
        all_other_symbols = []
        for sym in all_stocks:
            all_other_symbols.append(sym)
        for label, (f_sym, y_sym) in all_crypto.items():
            if label not in ["BTC", "ETH"]:
                all_other_symbols.append(f_sym)
        
        if all_other_symbols:
            rotated = all_other_symbols[_rotating_index % len(all_other_symbols)]
            subset_to_fetch.append(rotated)
            _rotating_index += 1
            
        # Concurrently fetch
        futures = [_executor.submit(_fetch_finnhub_quote, s, finnhub_key) for s in subset_to_fetch]
        for f in futures:
            sym, data = f.result()
            if data:
                # Map back to readable label if it was crypto
                label = sym
                for l, (fs, ys) in all_crypto.items():
                    if fs == sym:
                        label = l
                        break
                _last_fetch_results[label] = data
    else:
        # Yahoo Finance Fallback - fetch all (once per minute)
        logger.info("Finnhub key missing, using Yahoo Finance 60s update cycle.")
        to_fetch = all_stocks + [y_sym for l, (fs, y_sym) in all_crypto.items()]
        futures = [_executor.submit(_fetch_yfinance_single, s) for s in to_fetch]
        for f in futures:
            sym, data = f.result()
            if data:
                # Map back to readable label if it was crypto
                label = sym
                for l, (fs, ys) in all_crypto.items():
                    if ys == sym:
                        label = l
                        break
                _last_fetch_results[label] = data

    if not _last_fetch_results:
        return
        
    with _data_lock:
        latest_data["stocks"] = dict(_last_fetch_results)
        latest_data["financial_source"] = "finnhub" if use_finnhub else "yfinance"
    _mark_fresh("stocks")