"""Carte des champs Polymarket : pour chaque couche (5m, 15m, 1h, 4h), chaque champ et chaque tranche de TEMPS
RESTANT dans la fenetre (y compris avant son ouverture), lien avec le mouvement reel du prix a plusieurs horizons
futurs. Les indicateurs techniques et externes sont evalues aux memes horizons, pour comparaison.

Lecture : IC partiel = correlation de rang entre le champ et le rendement futur, une fois retire le momentum
recent du prix (Polymarket suit le prix : seul ce qui reste est une information propre). Positif = une valeur
elevee du champ annonce une hausse. Ecart Q5-Q1 = rendement futur moyen du quintile haut moins celui du bas (pb).
Une case n'est credible que si |t| > 3 ET qu'elle reste stable d'une analyse a l'autre.

Sorties : tableau console, field_map.csv et field_map.html (carte de chaleur interactive) a cote de la base.

Usage :
  python field_map.py
  python field_map.py --horizons 5m,15m,30m,1h,2h,4h --assets btc,eth --step 30 --window-days 30
"""
import argparse
import csv
import json
import math
import os
import sys
import time

from analyze import CONTROLS, X_FEATS, build_dataset, mean, pearson, ranks, residualize
import technical
from common import TF_SECONDS, connect, default_db_path, parse_duration

FIELDS = (("p", "Cote Up"), ("div", "Ecart a la juste valeur"), ("dp60", "Variation de cote 60 s"),
          ("flow60", "Flux net 60 s"), ("flowcum", "Flux net cumule"), ("imb5", "Desequilibre du carnet"))
NX_FIELDS = ("p", "dp60", "flowcum", "imb5")  # avant l'ouverture (l'ecart a la juste valeur y est la cote elle-meme)
BUCKETS = ((-1.0, 0.0), (0.0, 0.2), (0.2, 0.4), (0.4, 0.6), (0.6, 0.8), (0.8, 1.0))
OTHER = (("ta_r1z", "Rendement 1 min (z)"), ("ta_r15z", "Rendement 15 min (z)"), ("ta_r60z", "Rendement 60 min (z)"),
         ("ta_rsi", "RSI 14 (1 min)"), ("ta_macd", "MACD (1 min)"), ("ta_bb", "Bollinger %b"),
         ("ta_rv60", "Volatilite realisee 60 min"), ("ta_tv_15m", "Note facon TradingView 15 min"),
         ("ta_tv_1h", "Note facon TradingView 1 h"), ("ta_tv_4h", "Note facon TradingView 4 h"),
         ("x_ls_acc", "Binance : comptes long/short"), ("x_ls_acc_chg1h", "Binance : comptes long/short, var. 1 h"),
         ("x_ls_top", "Binance : gros traders long/short"), ("x_taker_5m", "Binance : achat/vente preneur 5 min"),
         ("x_taker_1h", "Binance : achat/vente preneur 1 h"), ("x_oi_chg15m", "Interet ouvert, var. 15 min"),
         ("x_oi_chg1h", "Interet ouvert, var. 1 h"), ("x_funding", "Funding"), ("x_basis", "Prime perpetuel/indice"),
         ("x_basis_chg15m", "Prime perpetuel, var. 15 min"), ("x_cb_prem", "Prime Coinbase"),
         ("x_cb_prem_chg15m", "Prime Coinbase, var. 15 min"), ("x_dvol", "Volatilite implicite (DVOL)"),
         ("x_dvol_chg1h", "DVOL, var. 1 h"), ("x_fng", "Fear & Greed"), ("x_fng_chg1d", "Fear & Greed, var. 1 j"),
         ("x_st_attn", "Stocktwits : pic d'attention"), ("x_st_bull", "Stocktwits : part haussiere vs 7 j"))
OTHER += tuple((f"tp_{k}_{tf}", technical.label(f"tp_{k}_{tf}")) for tf in technical.TF_NAMES for k in technical.PACK)
OTHER += (("xa_btc_r15z", "BTC, rendement 15 min (pour les autres cryptos)"),
          ("xa_btc_r60z", "BTC, rendement 60 min (pour les autres cryptos)"),
          ("xa_rel60", "Force relative vs BTC (60 min)"))
assert {k for k, _ in OTHER} >= set(X_FEATS)


def fmt_dur(sec):
    sec = int(round(sec))
    if sec >= 3600:
        return f"{sec // 3600}h{(sec % 3600) // 60:02d}"
    return f"{sec // 60} min"


def bucket_label(tf, lo, hi):
    d = TF_SECONDS[tf]
    if lo < 0:
        return "avant ouverture"
    return f"reste {fmt_dur(d * (1 - lo))} a {fmt_dur(d * (1 - hi))}"


def stats(sub, col, ycol, n_eff_windows, spacing, X, cache):
    """IC brut / partiel, t (n effectif), ecart de rendement Q5-Q1 (pb). `cache` (dict ou None) garde les rangs
    et residus du rendement futur quand `sub` est le sous-ensemble commun a tous les champs."""
    xs = [r[col] for r in sub]
    ys = [r[ycol] for r in sub]
    if len(set(xs)) < 5:
        return None
    fx = ranks(xs)
    if cache is None or "base" not in cache:
        fy = ranks(ys)
        ctrl = list(zip(*[ranks([r[c] for r in sub]) for c in CONTROLS]))
        base = (fy, ctrl, residualize(fy, ctrl))
        if cache is not None:
            cache["base"] = base
    fy, ctrl, ry = cache["base"] if cache is not None else base
    raw = pearson(fx, fy)
    part = pearson(residualize(fx, ctrl), ry) if col not in CONTROLS else raw
    n_eff = n_eff_windows * min(1.0, spacing / X)
    t = part * math.sqrt(max(n_eff - 2, 1)) / math.sqrt(max(1 - part * part, 1e-9)) if part == part else 0.0
    order = sorted(range(len(xs)), key=xs.__getitem__)
    q = max(len(order) // 5, 1)
    spread = mean([ys[i] for i in order[-q:]]) - mean([ys[i] for i in order[:q]])
    return {"n": len(sub), "n_eff": round(n_eff, 1), "ic_raw": raw, "ic": part, "t": t, "spread": spread}


def decorrelation_steps(rows, col):
    """Temps de decorrelation d'un indicateur (en pas de grille) : premier decalage ou son autocorrelation passe
    sous 1/e. Un indicateur quotidien (Fear & Greed) ne fournit qu'environ un episode independant par jour."""
    by_asset = {}
    for r in rows:
        if r.get(col) is not None:
            by_asset.setdefault(r["asset"], []).append((r["t"], r[col]))
    taus = []
    for seq in by_asset.values():
        seq.sort()
        x = [v for _, v in seq]
        n = len(x)
        m = sum(x) / n
        var = sum((v - m) ** 2 for v in x)
        if n < 50 or var <= 0:
            continue
        lag, tau = 1, None
        while lag < n // 2:
            rho = sum((x[i] - m) * (x[i + lag] - m) for i in range(n - lag)) / var
            if rho < 1 / math.e:
                tau = lag
                break
            lag *= 2
        taus.append(tau or n // 2)
    return max(taus) if taus else 1


def compute(rows, tfs, horizons, step):
    cells = []
    base_ok = [r for r in rows if all(r.get(c) is not None for c in CONTROLS)]
    for tf in tfs:
        d = TF_SECONDS[tf]
        for lo, hi in BUCKETS:
            pre = lo < 0
            el = f"{tf}_nx_el" if pre else f"{tf}_el"
            in_b = [r for r in base_ok if r.get(el) is not None and lo <= r[el] < hi]
            for h, X in horizons.items():
                ycol = f"y_{h}"
                base = [r for r in in_b if r.get(ycol) is not None]
                if len(base) < 60:
                    continue
                cache = {}
                for fk, _ in FIELDS:
                    if pre and fk not in NX_FIELDS:
                        continue
                    col = f"{tf}_nx_{fk}" if pre else f"{tf}_{fk}"
                    sub = base if all(r.get(col) is not None for r in base) else \
                        [r for r in base if r.get(col) is not None]
                    if len(sub) < 60:
                        continue
                    windows = len({round(r["t"] - r[el] * d) for r in sub})
                    s = stats(sub, col, ycol, windows, d, X, cache if sub is base else None)
                    if s:
                        cells.append({"section": "polymarket", "layer": tf, "field": fk, "lo": lo, "hi": hi,
                                      "bucket": bucket_label(tf, lo, hi), "horizon": h, **s})
    tau = {fk: decorrelation_steps(base_ok, fk) for fk, _ in OTHER}
    for h, X in horizons.items():
        ycol = f"y_{h}"
        base = [r for r in base_ok if r.get(ycol) is not None]
        cache = {}
        for fk, _ in OTHER:
            sub = base if all(r.get(fk) is not None for r in base) else [r for r in base if r.get(fk) is not None]
            if len(sub) < 60:
                continue
            # episodes independants : limites par le chevauchement des rendements ET par la lenteur de l'indicateur
            times = len({r["t"] for r in sub})
            s = stats(sub, fk, ycol, min(times, times / tau[fk] * X / step), step, X,
                      cache if sub is base else None)
            if s:
                cells.append({"section": "autres", "layer": "", "field": fk, "lo": None, "hi": None, "bucket": "",
                              "horizon": h, **s})
    return cells


def print_console(cells, tfs, horizons):
    def mark(c):
        return "**" if abs(c["t"]) > 3 else ("* " if abs(c["t"]) > 2 else "  ")

    idx = {(c["section"], c["layer"], c["field"], c["bucket"], c["horizon"]): c for c in cells}
    print("\nIC partiel par champ x temps restant x horizon futur (* |t|>2, ** |t|>3)")
    for tf in tfs:
        print(f"\n=== couche {tf} " + "=" * 60)
        print(f"{'champ':<24}{'temps restant':<24}" + "".join(f"{h:>10}" for h in horizons))
        for fk, label in FIELDS:
            for lo, hi in BUCKETS:
                b = bucket_label(tf, lo, hi)
                line = f"{label[:23]:<24}{b[:23]:<24}"
                vals = [idx.get(("polymarket", tf, fk, b, h)) for h in horizons]
                if not any(vals):
                    continue
                line += "".join(f"{c['ic']:+8.3f}{mark(c)}" if c else f"{'-':>10}" for c in vals)
                print(line)
    print(f"\n=== indicateurs techniques et externes " + "=" * 38)
    for fk, label in OTHER:
        vals = [idx.get(("autres", "", fk, "", h)) for h in horizons]
        if any(vals):
            print(f"{label[:47]:<48}" + "".join(f"{c['ic']:+8.3f}{mark(c)}" if c else f"{'-':>10}" for c in vals))
    top = sorted(cells, key=lambda c: -abs(c["t"]))[:12]
    print("\nCases les plus significatives :")
    for c in top:
        where = f"{c['layer']} {c['field']} [{c['bucket']}]" if c["section"] == "polymarket" else c["field"]
        print(f"  {where:<46} -> {c['horizon']:<4} IC {c['ic']:+.3f}  t={c['t']:+.1f}  "
              f"ecart Q5-Q1 {c['spread']:+.1f} pb  (n={c['n']}, ~{c['n_eff']:.0f} indep.)")


def write_html(path, cells, meta):
    data = json.dumps({"cells": cells, "meta": meta,
                       "fields": [list(f) for f in FIELDS], "other": [list(o) for o in OTHER]},
                      separators=(",", ":"), allow_nan=False, default=lambda x: None)
    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "field_map_template.html"),
              encoding="utf-8") as f:
        html = f.read()
    with open(path, "w", encoding="utf-8") as f:
        f.write(html.replace("__DATA__", data.replace("</", "<\\/")))


def clean(v):
    return None if isinstance(v, float) and (math.isnan(v) or math.isinf(v)) else v


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--db", default=default_db_path())
    ap.add_argument("--assets", default="btc,eth,sol,xrp")
    ap.add_argument("--tfs", default="5m,15m,1h,4h")
    ap.add_argument("--horizons", default="5m,15m,30m,1h,2h,4h", help="horizons futurs du mouvement reel")
    ap.add_argument("--step", type=int, default=60, help="pas de la grille temporelle (s)")
    ap.add_argument("--window-days", type=float, default=0, help="n'utiliser que les N derniers jours (0 = tout)")
    ap.add_argument("--out", help="dossier des sorties (defaut : a cote de la base)")
    args = ap.parse_args()
    sys.stdout.reconfigure(errors="replace")

    tfs = [t.strip() for t in args.tfs.split(",") if t.strip() in TF_SECONDS]
    horizons = {h.strip(): parse_duration(h) for h in args.horizons.split(",") if h.strip()}
    assets = [a.strip() for a in args.assets.split(",") if a.strip()]
    since = int(time.time() - args.window_days * 86400) if args.window_days else 0
    out = args.out or os.path.dirname(os.path.abspath(args.db))
    con = connect(args.db)

    print(f"base : {args.db}\nconstruction du jeu de donnees ...")
    rows = build_dataset(con, tfs, horizons, [], args.step, assets, since)
    if len(rows) < 200:
        print("Trop peu de donnees : laisser tourner collector.py (ou --backfill-days) puis relancer.")
        return
    t0, t1 = min(r["t"] for r in rows), max(r["t"] for r in rows)
    days = len({r["t"] for r in rows}) * args.step / 86400  # jours reellement couverts (sans les trous)
    print(f"  {len(rows)} points, {days:.1f} jours ; calcul de la carte ...")
    if days < 14:
        print(f"  !!! {days:.1f} jour(s) seulement : carte indicative, PAS significative (viser 2 a 4 semaines)")
    cells = [{k: clean(v) for k, v in c.items()} for c in compute(rows, tfs, horizons, args.step)]
    print_console(cells, tfs, horizons)

    csv_path = os.path.join(out, "field_map.csv")
    with open(csv_path, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=list(cells[0]))
        w.writeheader()
        w.writerows(cells)
    meta = {"generated": time.strftime("%Y-%m-%d %H:%M"), "assets": sorted({r["asset"] for r in rows}),
            "from": time.strftime("%Y-%m-%d %H:%M", time.localtime(t0)),
            "to": time.strftime("%Y-%m-%d %H:%M", time.localtime(t1)), "days": round(days, 1),
            "points": len(rows), "tfs": tfs, "horizons": list(horizons), "step": args.step}
    html_path = os.path.join(out, "field_map.html")
    write_html(html_path, cells, meta)
    print(f"\nsorties : {csv_path}\n          {html_path}")


if __name__ == "__main__":
    main()