"""Indicateurs externes gratuits -> tables `ext` et `stwits` de la meme base, evalues ensuite avec les memes
outils (analyze.py, field_map.py) que les champs Polymarket.

  Binance Futures : ratio comptes long/short, ratio positions des gros traders, volume preneur achat/vente,
                    interet ouvert (pas de 5 min, 30 j d'historique), funding (8 h), prime perpetuel/indice (1 min)
  Coinbase        : bougies 1 min -> prime Coinbase vs Binance (pression acheteuse americaine)
  Deribit         : DVOL, volatilite implicite des options BTC et ETH (1 min)
  alternative.me  : Fear & Greed Index (quotidien)
  Stocktwits      : messages avec etiquette Bullish / Bearish choisie par l'auteur (BTC.X, ETH.X, ...)

Chaque valeur est horodatee a l'instant ou elle devient DISPONIBLE : l'analyse ne peut pas regarder le futur.

Usage :
  python external.py --backfill-days 30
  python external.py --assets btc,eth
"""
import argparse
import logging
import os
import threading
import time

from common import SYMBOLS, connect, default_db_path, http_json, iso_ts, single_instance, ts_iso

FAPI = "https://fapi.binance.com"
COINBASE = "https://api.exchange.coinbase.com"
DERIBIT = "https://www.deribit.com/api/v2/public"
FNG = "https://api.alternative.me/fng/"
STWITS = "https://api.stocktwits.com/api/2/streams/symbol"
CB_PRODUCT = {"btc": "BTC-USD", "eth": "ETH-USD", "sol": "SOL-USD", "xrp": "XRP-USD"}
DVOL_CCY = {"btc": "BTC", "eth": "ETH"}
ST_SYMBOL = {"btc": "BTC.X", "eth": "ETH.X", "sol": "SOL.X", "xrp": "XRP.X"}
FUT = 300  # pas des series Binance Futures (s)

log = logging.getLogger("external")


def put(con, rows):
    if rows:
        con.executemany("INSERT OR REPLACE INTO ext VALUES(?,?,?,?,?)", rows)
        con.commit()
    return len(rows)


# ---------------------------------------------------------------- sources

def futures_data(asset, t0, t1):
    specs = [("globalLongShortAccountRatio", (("ls_acc", "longShortRatio"),)),
             ("topLongShortPositionRatio", (("ls_top", "longShortRatio"),)),
             ("takerlongshortRatio", (("taker_buy", "buyVol"), ("taker_sell", "sellVol"))),
             ("openInterestHist", (("oi", "sumOpenInterestValue"),))]
    t0 = max(t0, int(time.time()) - 29 * 86400)  # Binance ne garde que 30 j
    rows = []
    for ep, fields in specs:
        a = t0
        while a < t1:
            b = min(a + 500 * FUT, t1)
            data = http_json(f"{FAPI}/futures/data/{ep}", {"symbol": SYMBOLS[asset], "period": "5m", "limit": 500,
                                                           "startTime": a * 1000, "endTime": b * 1000})
            for d in data if isinstance(data, list) else []:
                ts = int(d["timestamp"]) // 1000 + FUT  # disponible a la fin de la periode (prudent)
                for key, f in fields:
                    if d.get(f) is not None:
                        rows.append(("binance", asset, key, ts, float(d[f])))
            a = b
    return rows


def funding(asset, t0, t1):
    rows, a = [], t0
    while a < t1:
        data = http_json(f"{FAPI}/fapi/v1/fundingRate", {"symbol": SYMBOLS[asset], "startTime": a * 1000,
                                                         "endTime": t1 * 1000, "limit": 1000})
        if not data:
            break
        rows += [("binance", asset, "funding", int(d["fundingTime"]) // 1000, float(d["fundingRate"])) for d in data]
        if len(data) < 1000:
            break
        a = int(data[-1]["fundingTime"]) // 1000 + 1
    return rows


def premium(asset, t0, t1):
    """Prime du perpetuel sur l'indice spot (bougies 1 min) : positionnement a effet de levier."""
    rows, a, now = [], max(t0, int(time.time()) - 29 * 86400), time.time()
    while a < t1:
        data = http_json(f"{FAPI}/fapi/v1/premiumIndexKlines", {"symbol": SYMBOLS[asset], "interval": "1m",
                                                                "startTime": a * 1000, "endTime": t1 * 1000,
                                                                "limit": 1500})
        if not data:
            break
        rows += [("binance", asset, "basis", int(k[0]) // 1000 + 60, float(k[4])) for k in data
                 if int(k[0]) // 1000 + 60 <= now]
        if len(data) < 1500:
            break
        a = int(data[-1][0]) // 1000 + 60
    return rows


def coinbase(asset, t0, t1):
    rows, a, now = [], t0, time.time()
    while a < t1:
        b = min(a + 300 * 60, t1)
        data = http_json(f"{COINBASE}/products/{CB_PRODUCT[asset]}/candles",
                         {"granularity": 60, "start": ts_iso(a), "end": ts_iso(b)})
        rows += [("coinbase", asset, "close", int(c[0]) + 60, float(c[4])) for c in data or [] if c[0] + 60 <= now]
        a = b
        time.sleep(0.15)  # limite publique Coinbase : 10 requetes/s
    return rows


def dvol(asset, t0, t1):
    if asset not in DVOL_CCY:
        return []
    rows, end, now = [], t1, time.time()
    while end > t0:
        r = http_json(f"{DERIBIT}/get_volatility_index_data", {"currency": DVOL_CCY[asset], "resolution": "60",
                                                               "start_timestamp": t0 * 1000, "end_timestamp": end * 1000})
        res = r.get("result") or {}
        data = res.get("data") or []
        rows += [("deribit", asset, "dvol", int(d[0]) // 1000 + 60, float(d[4])) for d in data
                 if int(d[0]) // 1000 + 60 <= now]
        cont = res.get("continuation")
        if not data or not cont or int(cont) // 1000 >= end:
            break
        end = int(cont) // 1000
    return rows


def fear_greed(days):
    data = http_json(FNG, {"limit": int(days) + 2})
    # valeur du jour publiee en debut de journee UTC ; marge d'une heure
    return [("alternative", "all", "fng", int(d["timestamp"]) + 3600, float(d["value"])) for d in data.get("data", [])]


def parse_stwits(asset, m):
    s = ((m.get("entities") or {}).get("sentiment") or {}).get("basic")
    return (int(m["id"]), asset, iso_ts(m["created_at"]), 1 if s == "Bullish" else (-1 if s == "Bearish" else 0))


def stocktwits_new(con, asset, max_pages=100):
    """Messages recents. Apres une interruption (PC eteint, coupure), remonte page par page jusqu'au dernier
    message deja connu : le trou est comble."""
    last = con.execute("SELECT MAX(id) FROM stwits WHERE symbol = ?", (asset,)).fetchone()[0]
    params, n = None, 0
    for _ in range(max_pages):
        d = http_json(f"{STWITS}/{ST_SYMBOL[asset]}.json", params)
        rows = [parse_stwits(asset, m) for m in d.get("messages", [])]
        con.executemany("INSERT OR IGNORE INTO stwits VALUES(?,?,?,?)", rows)
        con.commit()
        n += len(rows)
        oldest = min((r[0] for r in rows), default=None)
        if oldest is None or last is None or oldest <= last:
            break
        params = {"max": oldest - 1}
        time.sleep(2)
    return n


def stocktwits_back(con, asset, t0, max_pages, stop):
    oldest = con.execute("SELECT MIN(id), MIN(ts) FROM stwits WHERE symbol = ?", (asset,)).fetchone()
    max_id, n = oldest[0], 0
    if oldest[1] is not None and oldest[1] <= t0:
        return 0
    for _ in range(max_pages):
        if stop.is_set():
            break
        d = http_json(f"{STWITS}/{ST_SYMBOL[asset]}.json", {"max": max_id - 1} if max_id else None)
        ms = d.get("messages", [])
        if not ms:
            break
        rows = [parse_stwits(asset, m) for m in ms]
        con.executemany("INSERT OR IGNORE INTO stwits VALUES(?,?,?,?)", rows)
        con.commit()
        n += len(rows)
        max_id = min(r[0] for r in rows)
        if min(r[2] for r in rows) <= t0:
            break
        time.sleep(2)  # API publique sans cle : rester discret
    return n


# ---------------------------------------------------------------- historique + boucle

def backfill(db, assets, days, stop):
    con = connect(db)
    now = int(time.time())
    t0 = now - int(days * 86400)
    for name, fn in (("binance futures", futures_data), ("funding", funding), ("prime perp", premium),
                     ("coinbase", coinbase), ("deribit dvol", dvol)):
        for a in assets:
            if stop.is_set():
                return
            try:
                log.info("historique %s %s : %d valeurs", name, a, put(con, fn(a, t0, now)))
            except Exception as e:
                log.warning("historique %s %s : %s", name, a, e)
    try:
        log.info("historique fear & greed : %d valeurs", put(con, fear_greed(days)))
    except Exception as e:
        log.warning("historique fear & greed : %s", e)
    for a in assets:
        try:
            log.info("historique stocktwits %s : %d messages", a, stocktwits_back(con, a, t0, 400, stop))
        except Exception as e:
            log.warning("historique stocktwits %s : %s", a, e)
    log.info("historique termine")


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--db", default=default_db_path())
    ap.add_argument("--assets", default="btc,eth,sol,xrp")
    ap.add_argument("--backfill-days", type=float, default=0)
    args = ap.parse_args()
    assets = [a.strip() for a in args.assets.split(",") if a.strip()]

    os.makedirs(os.path.dirname(os.path.abspath(args.db)), exist_ok=True)
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s",
                        handlers=[logging.StreamHandler(),
                                  logging.FileHandler(os.path.join(os.path.dirname(os.path.abspath(args.db)),
                                                                   "external.log"), encoding="utf-8")])
    lock = single_instance("external", args.db)  # noqa: F841 (garde le verrou ouvert)
    con = connect(args.db)
    stop = threading.Event()
    if args.backfill_days > 0:
        threading.Thread(target=backfill, args=(args.db, assets, args.backfill_days, stop), daemon=True).start()

    # (nom, periode s, fonction) : chaque tache relit une petite fenetre recente, les doublons sont ecrases
    def since(src, a, key, now, default):
        """Reprendre a la derniere valeur connue (moins une marge) : un arret ou un ralentissement ne laisse
        pas de trou, dans la limite de l'historique conserve par la source (~30 j)."""
        last = con.execute("SELECT MAX(ts) FROM ext WHERE source = ? AND symbol = ? AND key = ?",
                           (src, a, key)).fetchone()[0]
        return max(int(last) - 900, now - 29 * 86400) if last else now - default

    tasks = [("binance futures", 300, lambda a, now: put(con, futures_data(a, since("binance", a, "oi", now, 1800), now))),
             ("funding", 3600, lambda a, now: put(con, funding(a, since("binance", a, "funding", now, 86400), now))),
             ("prime perp", 60, lambda a, now: put(con, premium(a, since("binance", a, "basis", now, 600), now))),
             ("coinbase", 60, lambda a, now: put(con, coinbase(a, since("coinbase", a, "close", now, 900), now))),
             ("deribit dvol", 60, lambda a, now: put(con, dvol(a, since("deribit", a, "dvol", now, 900), now))),
             ("stocktwits", 120, lambda a, now: stocktwits_new(con, a))]
    last = {}
    last_fng = last_status = 0
    count = 0
    log.info("indicateurs externes | base %s | actifs %s", args.db, assets)
    try:
        while True:
            now = int(time.time())
            for name, period, fn in tasks:
                if now - last.get(name, 0) < period:
                    continue
                last[name] = now
                for a in assets:
                    try:
                        count += fn(a, now)
                    except Exception as e:
                        log.warning("%s %s : %s", name, a, e)
            if now - last_fng >= 3600:
                try:
                    count += put(con, fear_greed(2))
                except Exception as e:
                    log.warning("fear & greed : %s", e)
                last_fng = now
            if now - last_status >= 600:
                log.info("+%d valeurs en 10 min", count)
                count, last_status = 0, now
            time.sleep(max(1, 60 - (time.time() - now)))
    except KeyboardInterrupt:
        log.info("arret demande")
    finally:
        stop.set()


if __name__ == "__main__":
    main()