"""Analyse multi-couches et apprentissage : l'etat Polymarket des 4 horizons (5m, 15m, 1h, 4h) a l'instant t,
combine aux indicateurs techniques, predit-il le mouvement REEL du spot entre t et t+X ?

  1. grille temporelle (un point toutes les --step s par actif) ; a chaque point, pour chaque couche :
     cote Up, ecart a la "juste valeur" deduite du spot, variation sur 60 s, flux d'ordres, desequilibre du
     carnet ; + consensus entre couches ; + indicateurs techniques Binance (RSI, MACD, Bollinger, momentum).
  2. etiquettes : rendement spot a t+X, MFE / MAE, et "barriere" : +x % touche avant -x % (ou l'inverse).
  3. IC de chaque facteur, brut et partiel (momentum spot retire : Polymarket suit le spot, seule la
     partielle dit s'il apporte une information propre).
  4. arbres de decision : configurations ("agencements") de facteurs menant a une hausse / baisse de x %,
     trouvees sur l'entrainement et verifiees sur le test.
  5. signal combine (ridge) en walk-forward + simulation achat/vente avec objectif et stop-loss en multiples
     de volatilite, frais deduits.
  6. --save : enregistre l'historique des essais (table model_runs) et publie strategy.json si une strategie
     passe la validation hors echantillon (sinon strategie inactive). --every 6h : recommence en boucle sur
     les donnees accumulees, la strategie s'adapte donc aux observations.

Usage :
  python analyze.py
  python analyze.py --horizons 5m,15m,1h --moves 0.25,0.5,1 --cost-bps 20 --csv dataset.csv
  python analyze.py --save --every 6h --window-days 30
"""
import argparse
import bisect
import csv
import json
import math
import os
import sys
import time
from collections import defaultdict

from common import BINANCE, SYMBOLS, TF_SECONDS, connect, default_db_path, http_json, parse_duration, single_instance
from technical import RULES, TF_NAMES, TechPack, pack_names

CONTROLS = ("ta_r1z", "ta_r15z")
LAYER_FEATS = ("p", "div", "dp60", "flow60", "flowcum", "imb5", "el")  # el = avancement de la fenetre (<0 : avant)
STALE = 90  # s : age max d'un releve de cote pour etre utilise

RUNS_SCHEMA = """
CREATE TABLE IF NOT EXISTS model_runs(
  ts INTEGER, horizon TEXT, mode TEXT, oos_ic REAL, n_trades INTEGER, total_usd REAL, usd_per_day REAL,
  profit_factor REAL, exit_spec TEXT, lam REAL, n_train INTEGER, n_test INTEGER, chosen INTEGER
);
"""


# ---------------------------------------------------------------- utilitaires numeriques

def ncdf(x):
    return 0.5 * (1 + math.erf(x / math.sqrt(2)))


def logit(p, lo=0.02):
    p = min(max(p, lo), 1 - lo)
    return math.log(p / (1 - p))


def mean(a):
    return sum(a) / len(a) if a else float("nan")


def median(a):
    s = sorted(a)
    n = len(s)
    return float("nan") if not n else (s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2)


def ranks(a):
    order = sorted(range(len(a)), key=a.__getitem__)
    r = [0.0] * len(a)
    i = 0
    while i < len(order):
        j = i
        while j + 1 < len(order) and a[order[j + 1]] == a[order[i]]:
            j += 1
        for k in range(i, j + 1):
            r[order[k]] = (i + j) / 2
        i = j + 1
    return r


def pearson(a, b):
    n = len(a)
    if n < 3:
        return float("nan")
    ma, mb = sum(a) / n, sum(b) / n
    va = sum((x - ma) ** 2 for x in a)
    vb = sum((y - mb) ** 2 for y in b)
    if va <= 0 or vb <= 0:
        return float("nan")
    return sum((x - ma) * (y - mb) for x, y in zip(a, b)) / math.sqrt(va * vb)


def solve(A, b):
    n = len(b)
    M = [list(A[i]) + [b[i]] for i in range(n)]
    for c in range(n):
        p = max(range(c, n), key=lambda r: abs(M[r][c]))
        M[c], M[p] = M[p], M[c]
        if abs(M[c][c]) < 1e-12:
            continue
        for r in range(n):
            if r != c and M[r][c]:
                f = M[r][c] / M[c][c]
                M[r] = [x - f * y for x, y in zip(M[r], M[c])]
    return [M[i][n] / M[i][i] if abs(M[i][i]) > 1e-12 else 0.0 for i in range(n)]


def residualize(y, X):
    """Residus de y regresse (OLS + constante) sur les colonnes de X."""
    k = len(X[0]) + 1
    A = [[0.0] * k for _ in range(k)]
    b = [0.0] * k
    for xi, yi in zip(X, y):
        v = (1.0,) + tuple(xi)
        for i in range(k):
            b[i] += v[i] * yi
            for j in range(k):
                A[i][j] += v[i] * v[j]
    w = solve(A, b)
    return [yi - w[0] - sum(wj * xj for wj, xj in zip(w[1:], xi)) for xi, yi in zip(X, y)]


# ---------------------------------------------------------------- donnees de marche (Binance)

def ensure_klines(con, symbol, t0, t1):
    t0, t1 = int(t0) // 60 * 60, min(int(t1), int(time.time()) - 60) // 60 * 60
    lo, hi = con.execute("SELECT MIN(open_ts), MAX(open_ts) FROM klines WHERE symbol = ?", (symbol,)).fetchone()
    ranges = [(t0, t1)] if lo is None else [(t0, lo - 60), (hi + 60, t1)]  # jamais de trou entre deux appels
    for a, b in ranges:
        while a <= b:
            rows = http_json(f"{BINANCE}/api/v3/klines", {"symbol": symbol, "interval": "1m", "startTime": a * 1000,
                                                         "endTime": b * 1000 + 59999, "limit": 1000})
            if not rows:
                break
            con.executemany("INSERT OR IGNORE INTO klines VALUES(?,?,?,?,?,?,?)",
                            [(symbol, r[0] // 1000, float(r[1]), float(r[2]), float(r[3]), float(r[4]), float(r[5]))
                             for r in rows])
            con.commit()
            a = rows[-1][0] // 1000 + 60


def tv_rating(c, h, l):
    """Note technique facon TradingView ("Technical Ratings"), recalculee localement a partir des prix :
    moyenne de votes moyennes mobiles (SMA/EMA 10..200 sous/au-dessus du prix) et de votes oscillateurs
    (RSI, stochastique, CCI, momentum, MACD, Williams %R). -1 = vente forte, +1 = achat fort."""
    n = len(c)
    ps = [0.0]
    for x in c:
        ps.append(ps[-1] + x)

    def ema(x, span):
        a, out, e = 2 / (span + 1), [], x[0] if x else 0
        for v in x:
            e = a * v + (1 - a) * e
            out.append(e)
        return out

    lengths = (10, 20, 30, 50, 100, 200)
    emas = {L: ema(c, L) for L in lengths}
    macd = [a - b for a, b in zip(ema(c, 12), ema(c, 26))]
    sig = ema(macd, 9)
    rsi, g, ls = [None] * n, 0.0, 0.0
    for i in range(1, n):
        d = c[i] - c[i - 1]
        g = (g * 13 + max(d, 0)) / 14 if i > 14 else g + max(d, 0) / 14
        ls = (ls * 13 + max(-d, 0)) / 14 if i > 14 else ls + max(-d, 0) / 14
        if i >= 14:
            rsi[i] = 100.0 if ls == 0 else 100 - 100 / (1 + g / ls)
    k_raw = [None] * n
    for i in range(13, n):
        hh, ll = max(h[i - 13:i + 1]), min(l[i - 13:i + 1])
        k_raw[i] = 100 * (c[i] - ll) / (hh - ll) if hh > ll else 50.0
    k = [None] * n
    for i in range(15, n):
        k[i] = sum(k_raw[i - 2:i + 1]) / 3
    dline = [None] * n
    for i in range(17, n):
        dline[i] = sum(k[i - 2:i + 1]) / 3
    tp = [(a + b + x) / 3 for a, b, x in zip(h, l, c)]
    cci = [None] * n
    for i in range(19, n):
        w = tp[i - 19:i + 1]
        m = sum(w) / 20
        md = sum(abs(x - m) for x in w) / 20
        cci[i] = (tp[i] - m) / (0.015 * md) if md else 0.0
    out = [None] * n
    for i in range(1, n):
        votes = []
        for L in lengths:
            if i >= L - 1:
                sma = (ps[i + 1] - ps[i + 1 - L]) / L
                votes += [1 if sma < c[i] else -1 if sma > c[i] else 0, 1 if emas[L][i] < c[i] else -1 if emas[L][i] > c[i] else 0]
        if len(votes) < 4:
            continue
        osc = [1 if macd[i] > sig[i] else -1 if macd[i] < sig[i] else 0]
        if rsi[i] is not None and rsi[i - 1] is not None:
            osc.append(1 if rsi[i] < 30 and rsi[i] > rsi[i - 1] else -1 if rsi[i] > 70 and rsi[i] < rsi[i - 1] else 0)
        if k[i] is not None and dline[i] is not None:
            osc.append(1 if k[i] < 20 and k[i] > dline[i] else -1 if k[i] > 80 and k[i] < dline[i] else 0)
        if cci[i] is not None and cci[i - 1] is not None:
            osc.append(1 if cci[i] < -100 and cci[i] > cci[i - 1] else -1 if cci[i] > 100 and cci[i] < cci[i - 1] else 0)
        if i >= 11:
            mom, mom0 = c[i] - c[i - 10], c[i - 1] - c[i - 11]
            osc.append(1 if mom > mom0 else -1 if mom < mom0 else 0)
        if k_raw[i] is not None and k_raw[i - 1] is not None:
            wr, wr0 = k_raw[i] - 100, k_raw[i - 1] - 100  # Williams %R = %K brut - 100
            osc.append(1 if wr < -80 and wr > wr0 else -1 if wr > -20 and wr < wr0 else 0)
        out[i] = (sum(votes) / len(votes) + sum(osc) / len(osc)) / 2
    return out


class Series:
    """Prix spot (ticks du collecteur + bougies 1 min) et indicateurs techniques d'un symbole."""

    def __init__(self, con, symbol, since=0):
        k = con.execute("SELECT open_ts, o, h, l, c, v FROM klines WHERE symbol = ? AND open_ts >= ? ORDER BY open_ts",
                        (symbol, since)).fetchall()
        self.kt = [r[0] for r in k]
        self.idx = {t: i for i, t in enumerate(self.kt)}
        self.o, self.h, self.l, self.c, self.v = ([r[i] for r in k] for i in range(1, 6))
        s = con.execute("SELECT ts, mid FROM spot WHERE symbol = ? AND ts >= ? ORDER BY ts", (symbol, since)).fetchall()
        self.st = [r[0] for r in s]
        self.sm = [r[1] for r in s]
        self.end = max(self.kt[-1] + 60 if self.kt else 0, self.st[-1] if self.st else 0)
        self._indicators()

    def _indicators(self):
        c, n = self.c, len(self.c)
        r = [0.0] + [math.log(c[i] / c[i - 1]) for i in range(1, n)]
        ps, ps2 = [0.0], [0.0]
        for x in r:
            ps.append(ps[-1] + x)
            ps2.append(ps2[-1] + x * x)

        def rstd(i, w):
            if i < w:
                return None
            m = (ps[i + 1] - ps[i + 1 - w]) / w
            v = (ps2[i + 1] - ps2[i + 1 - w]) / w - m * m
            return math.sqrt(max(v, 0) * w / (w - 1)) or None

        self.rv60 = [rstd(i, 60) for i in range(n)]
        self.rv15 = [rstd(i, 15) for i in range(n)]

        def ema(x, span):
            a, out, e = 2 / (span + 1), [], x[0] if x else 0
            for v in x:
                e = a * v + (1 - a) * e
                out.append(e)
            return out

        macd = [a - b for a, b in zip(ema(c, 12), ema(c, 26))]
        sig = ema(macd, 9)
        self.macd = [m - s for m, s in zip(macd, sig)]
        self.rsi = [None] * n
        g = l_ = 0.0
        for i in range(1, n):
            d = c[i] - c[i - 1]
            if i <= 14:
                g += max(d, 0) / 14
                l_ += max(-d, 0) / 14
            else:
                g = (g * 13 + max(d, 0)) / 14
                l_ = (l_ * 13 + max(-d, 0)) / 14
            if i >= 14:
                self.rsi[i] = 100.0 if l_ == 0 else 100 - 100 / (1 + g / l_)
        self.bb = [None] * n
        for i in range(19, n):
            w = c[i - 19:i + 1]
            m = sum(w) / 20
            sd = math.sqrt(sum((x - m) ** 2 for x in w) / 20)
            self.bb[i] = (c[i] - (m - 2 * sd)) / (4 * sd) if sd else 0.5
        # barres 15 min / 1 h / 4 h closes : pack technique, declencheurs classiques, note facon TradingView
        self.pack = TechPack(self.kt, self.o, self.h, self.l, self.c, self.v)
        self.tv = {name: (b["ends"], tv_rating(b["c"], b["h"], b["l"])) for name, b in self.pack.tf.items()}

    def tv_at(self, name, t):
        ends, rating = self.tv[name]
        j = bisect.bisect_right(ends, t) - 1
        return rating[j] if j >= 0 else None

    def price(self, t):
        j = bisect.bisect_right(self.st, t) - 1
        if j >= 0 and t - self.st[j] <= 15:
            return self.sm[j]
        i = self.idx.get(int(t) // 60 * 60)
        return self.o[i] if i is not None else None

    def last_closed(self, t):
        """Indice de la derniere bougie 1 min entierement cloturee a l'instant t (pas de regard vers le futur)."""
        return self.idx.get(int(t) // 60 * 60 - 60)

    def returns_z(self, t):
        """Rendements 1 / 15 / 60 min normalises par la volatilite (None si historique insuffisant)."""
        i = self.last_closed(t)
        if i is None or i < 60 or not self.rv60[i]:
            return None
        rv, c = self.rv60[i], self.c
        return tuple(math.log(c[i] / c[i - k]) / (rv * math.sqrt(k)) for k in (1, 15, 60))

    def ta(self, t):
        i = self.last_closed(t)
        z = self.returns_z(t)
        if z is None:
            return None
        rv, c = self.rv60[i], self.c
        return {"ta_r1z": z[0], "ta_r15z": z[1], "ta_r60z": z[2],
                "ta_rsi": self.rsi[i], "ta_macd": self.macd[i] / (c[i] * rv),
                "ta_bb": self.bb[i], "ta_rv60": rv * 1e4,
                "ta_volratio": (self.rv15[i] / rv) if self.rv15[i] else None, "_rv": rv,
                "ta_tv_15m": self.tv_at("15m", t), "ta_tv_1h": self.tv_at("1h", t), "ta_tv_4h": self.tv_at("4h", t),
                **self.pack.features(t)}

    def path(self, t, X):
        """Bougies (high, low) entierement posterieures a t et anterieures a t+X."""
        m = (int(t) + 59) // 60 * 60
        out = []
        while m + 60 <= t + X:
            i = self.idx.get(m)
            if i is not None:
                out.append((self.h[i], self.l[i]))
            m += 60
        return out


# ---------------------------------------------------------------- donnees Polymarket

class Layer:
    """Fenetres successives d'un (actif, horizon) et releves de cote / carnet / flux de chacune."""

    def __init__(self):
        self.starts, self.mk = [], []

    def window(self, t):
        i = bisect.bisect_right(self.starts, t) - 1
        if i >= 0 and t < self.mk[i]["end"]:
            return self.mk[i]
        return None

    def next(self, t):
        """Fenetre suivante, pas encore ouverte (etape "avant ouverture")."""
        i = bisect.bisect_right(self.starts, t)
        if i < len(self.mk) and self.mk[i]["start"] - t <= self.mk[i]["end"] - self.mk[i]["start"]:
            return self.mk[i]
        return None


def snap(m, t):
    j = bisect.bisect_right(m["ts"], t) - 1
    if j >= 0 and t - m["ts"][j] <= STALE:
        return j
    return None


def load_polymarket(con, tfs, since=0):
    by_cid = {}
    for cid, asset, tf, start, end in con.execute(
            "SELECT id, asset, tf, start_ts, end_ts FROM markets WHERE end_ts > ? AND tf IN (%s) "
            "ORDER BY start_ts" % ",".join("?" * len(tfs)), (since, *tfs)):
        by_cid[cid] = {"asset": asset, "tf": tf, "start": start, "end": end,
                       "ts": [], "mid": [], "imb": [], "fb": [], "fu": [0.0], "fg": [0.0]}
    cids = list(by_cid)
    for i in range(0, len(cids), 500):
        chunk = cids[i:i + 500]
        q = ",".join("?" * len(chunk))
        hist = defaultdict(list)
        for cid, ts, src, mid, bd, ad in con.execute(
                f"SELECT mkt, ts, src, mid, bid_d5, ask_d5 FROM book WHERE mkt IN ({q}) "
                f"ORDER BY mkt, ts", chunk):
            if mid is None:
                continue
            if src != "live":
                hist[cid].append((ts, mid))
                continue
            m = by_cid[cid]
            m["ts"].append(ts)
            m["mid"].append(mid)
            m["imb"].append((bd - ad) / (bd + ad) if bd is not None and ad is not None and bd + ad > 0 else None)
        # releves en direct en priorite ; la cote a la minute (historique) comble seulement les trous de collecte
        for cid, hs in hist.items():
            m = by_cid[cid]
            lt = m["ts"]
            add = [(t, p) for t, p in hs if not lt or min(abs(t - lt[max(bisect.bisect_left(lt, t) - 1, 0)]),
                                                         abs(t - lt[min(bisect.bisect_left(lt, t), len(lt) - 1)])) > STALE]
            if add:
                merged = sorted([(t, p, i) for t, p, i in zip(lt, m["mid"], m["imb"])] + [(t, p, None) for t, p in add])
                m["ts"], m["mid"], m["imb"] = [x[0] for x in merged], [x[1] for x in merged], [x[2] for x in merged]
        for cid, b, up, gross in con.execute(f"SELECT mkt, bucket_ts, up_pressure, gross FROM flow "
                                             f"WHERE mkt IN ({q}) ORDER BY mkt, bucket_ts", chunk):
            m = by_cid[cid]
            m["fb"].append(b)
            m["fu"].append(m["fu"][-1] + up)
            m["fg"].append(m["fg"][-1] + gross)
    layers = defaultdict(dict)
    for m in by_cid.values():
        if m["ts"]:
            L = layers[m["asset"]].setdefault(m["tf"], Layer())
            L.starts.append(m["start"])
            L.mk.append(m)
    return layers


def flow_between(m, a, b):
    """Flux net normalise ([-1, 1]) des tranches de 10 s entierement comprises dans [a, b]."""
    if not m["fb"]:
        return None
    i = bisect.bisect_left(m["fb"], a)
    j = bisect.bisect_right(m["fb"], b - 10)
    g = m["fg"][j] - m["fg"][i]
    return (m["fu"][j] - m["fu"][i]) / g if g > 0 else 0.0


def layer_features(m, t, S, sp):
    j = snap(m, t)
    if j is None:
        return None
    p = m["mid"][j]
    d = m["end"] - m["start"]
    el = (t - m["start"]) / d
    if el >= 0:
        K = sp.price(m["start"])
        i = sp.last_closed(t)
        rv = sp.rv60[i] if i is not None else None
        if K is None or not rv:
            return None
        fair = ncdf(math.log(S / K) / (rv * math.sqrt(max((m["end"] - t) / 60, 0.5))))
    else:
        fair = 0.5
    j0 = snap(m, t - 60)
    return {"p": p, "div": logit(p) - logit(fair),
            "dp60": (logit(p) - logit(m["mid"][j0])) if j0 is not None else None,
            "flow60": flow_between(m, t - 60, t), "flowcum": flow_between(m, m["start"] - d, t),
            "imb5": m["imb"][j], "el": el}


class Ext:
    """Indicateurs externes (external.py), lus "a la date de disponibilite" : jamais de valeur future."""

    def __init__(self, con, assets, since=0):
        self.s, self.st = {}, {}
        q = ",".join("?" * (len(assets) + 1))
        for src, sym, key, ts, v in con.execute(
                f"SELECT source, symbol, key, ts, value FROM ext WHERE ts >= ? AND symbol IN ({q}) "
                f"ORDER BY source, symbol, key, ts", (since, *assets, "all")):
            a = self.s.setdefault((src, sym, key), ([], []))
            a[0].append(ts)
            a[1].append(v)
        for sym, ts, sent in con.execute(f"SELECT symbol, ts, sent FROM stwits WHERE ts >= ? AND symbol IN ({q}) "
                                         f"ORDER BY symbol, ts", (since, *assets, "all")):
            a = self.st.setdefault(sym, ([], [0], [0], [0]))
            a[0].append(ts)
            a[1].append(a[1][-1] + 1)
            a[2].append(a[2][-1] + (sent == 1))
            a[3].append(a[3][-1] + (sent == -1))

    def at(self, src, sym, key, t, stale):
        a = self.s.get((src, sym, key))
        if not a:
            return None
        j = bisect.bisect_right(a[0], t) - 1
        return a[1][j] if j >= 0 and t - a[0][j] <= stale else None

    def exact(self, src, sym, key, ts):
        a = self.s.get((src, sym, key))
        if not a:
            return None
        j = bisect.bisect_left(a[0], ts)
        return a[1][j] if j < len(a[0]) and a[0][j] == ts else None

    def total(self, src, sym, key, t0, t1):
        a = self.s.get((src, sym, key))
        if not a:
            return None
        i, j = bisect.bisect_right(a[0], t0), bisect.bisect_right(a[0], t1)
        return sum(a[1][i:j]) if j > i else None

    def stwits(self, sym, t0, t1):
        a = self.st.get(sym)
        if not a:
            return None
        i, j = bisect.bisect_right(a[0], t0), bisect.bisect_right(a[0], t1)
        return a[1][j] - a[1][i], a[2][j] - a[2][i], a[3][j] - a[3][i]


X_FEATS = ("x_ls_acc", "x_ls_acc_chg1h", "x_ls_top", "x_taker_5m", "x_taker_1h", "x_oi_chg15m", "x_oi_chg1h",
           "x_funding", "x_basis", "x_basis_chg15m", "x_cb_prem", "x_cb_prem_chg15m", "x_dvol", "x_dvol_chg1h",
           "x_fng", "x_fng_chg1d", "x_st_attn", "x_st_bull")


def ext_features(ext, asset, t, sp):
    """Positionnement (Binance Futures), primes (Coinbase, perpetuel), volatilite implicite (Deribit),
    sentiment (Fear & Greed, Stocktwits) a l'instant t."""
    f = {}

    def at(src, key, stale, dt=0, sym=asset):
        return ext.at(src, sym, key, t - dt, stale)

    ls, ls1 = at("binance", "ls_acc", 900), at("binance", "ls_acc", 900, 3600)
    if ls:
        f["x_ls_acc"] = math.log(ls)
        if ls1:
            f["x_ls_acc_chg1h"] = math.log(ls / ls1)
    top = at("binance", "ls_top", 900)
    if top:
        f["x_ls_top"] = math.log(top)
    b5, s5 = at("binance", "taker_buy", 900), at("binance", "taker_sell", 900)
    if b5 and s5:
        f["x_taker_5m"] = math.log(b5 / s5)
    b1, s1 = ext.total("binance", asset, "taker_buy", t - 3600, t), ext.total("binance", asset, "taker_sell", t - 3600, t)
    if b1 and s1:
        f["x_taker_1h"] = math.log(b1 / s1)
    oi, oi15, oi1 = at("binance", "oi", 900), at("binance", "oi", 900, 900), at("binance", "oi", 900, 3600)
    if oi and oi15:
        f["x_oi_chg15m"] = math.log(oi / oi15) * 1e4
    if oi and oi1:
        f["x_oi_chg1h"] = math.log(oi / oi1) * 1e4
    fr = at("binance", "funding", 9 * 3600)
    if fr is not None:
        f["x_funding"] = fr * 1e4
    bs, bs15 = at("binance", "basis", 180), at("binance", "basis", 180, 900)
    if bs is not None:
        f["x_basis"] = bs * 1e4
        if bs15 is not None:
            f["x_basis_chg15m"] = (bs - bs15) * 1e4

    def cb_prem(m):  # Coinbase (USD) vs Binance (USDT) sur la meme bougie 1 min cloturee
        cb, i = ext.exact("coinbase", asset, "close", m + 60), sp.idx.get(m)
        return math.log(cb / sp.c[i]) * 1e4 if cb and i is not None else None

    m = int(t) // 60 * 60 - 60
    p0, p15 = cb_prem(m), cb_prem(m - 900)
    if p0 is not None:
        f["x_cb_prem"] = p0
        if p15 is not None:
            f["x_cb_prem_chg15m"] = p0 - p15
    dv, dv1 = at("deribit", "dvol", 180), at("deribit", "dvol", 180, 3600)
    if dv is not None:
        f["x_dvol"] = dv
        if dv1 is not None:
            f["x_dvol_chg1h"] = dv - dv1
    fg, fg1 = at("alternative", "fng", 2 * 86400, sym="all"), at("alternative", "fng", 2 * 86400, 86400, sym="all")
    if fg is not None:
        f["x_fng"] = fg
        if fg1 is not None:
            f["x_fng_chg1d"] = fg - fg1
    h1, d1, w7 = ext.stwits(asset, t - 3600, t), ext.stwits(asset, t - 86400, t), ext.stwits(asset, t - 7 * 86400, t)
    if h1 and d1 and d1[0] >= 24:
        f["x_st_attn"] = math.log((h1[0] + 1) / (d1[0] / 24 + 1))  # pic d'attention vs moyenne horaire 24 h
    if h1 and w7 and h1[1] + h1[2] >= 5 and w7[1] + w7[2] >= 50:
        f["x_st_bull"] = h1[1] / (h1[1] + h1[2]) - w7[1] / (w7[1] + w7[2])  # part haussiere 1 h vs 7 j
    return f


def features_at(asset, t, layers, sp, tfs, ext=None, btc=None, ev_window=60):
    """Toutes les couches + indicateurs a l'instant t (n'utilise que des donnees anterieures a t).
    btc : serie BTC de reference pour les autres cryptos ; ev_window : fenetre des declencheurs classiques."""
    S = sp.price(t)
    ta = sp.ta(t) if S else None
    if ta is None:
        return None
    r = {"asset": asset, "t": t, "_S": S, "_sp": sp, **ta, "_events": sp.pack.events(t, ev_window)}
    bz = btc.returns_z(t) if btc is not None and btc is not sp else None
    if bz:  # BTC entraine souvent les autres cryptos : son mouvement recent et l'ecart de force relative
        r["xa_btc_r1z"], r["xa_btc_r15z"], r["xa_btc_r60z"] = bz
        r["xa_rel60"] = ta["ta_r60z"] - bz[2]
    n_layers = 0
    for tf in tfs:
        L = layers.get(asset, {}).get(tf)
        m = L.window(t) if L else None
        f = layer_features(m, t, S, sp) if m else None
        if f:
            n_layers += 1
            for k, v in f.items():
                r[f"{tf}_{k}"] = v
        nx = L.next(t) if L else None
        f = layer_features(nx, t, S, sp) if nx else None
        if f:
            for k in ("p", "div", "dp60", "flowcum", "imb5", "el"):
                r[f"{tf}_nx_{k}"] = f[k]
    if not n_layers:
        return None
    if ext is not None:
        r.update(ext_features(ext, asset, t, sp))
    divs = [r[f"{tf}_div"] for tf in tfs if f"{tf}_div" in r]
    flows = [r[f"{tf}_flow60"] for tf in tfs if r.get(f"{tf}_flow60") is not None]
    r["ml_div_mean"] = mean(divs)
    r["ml_div_agree"] = sum(1 if d > 0 else -1 for d in divs) / len(divs)
    r["ml_flow_mean"] = mean(flows) if flows else None
    r["ml_n_layers"] = n_layers
    return r


def first_touch(sp, t, S, X, x_bps):
    """+1 si +x touche avant -x dans [t, t+X], -1 si l'inverse, 0 sinon (ou ambigu dans la meme bougie)."""
    up, dn = S * math.exp(x_bps / 1e4), S * math.exp(-x_bps / 1e4)
    for h, l_ in sp.path(t, X):
        hit_up, hit_dn = h >= up, l_ <= dn
        if hit_up and hit_dn:
            return 0
        if hit_up:
            return 1
        if hit_dn:
            return -1
    return 0


def add_labels(r, sp, horizons, moves):
    t, S = r["t"], r["_S"]
    for name, X in horizons.items():
        if t + X > sp.end:
            continue
        P1 = sp.price(t + X)
        if P1 is None:
            continue
        r[f"y_{name}"] = math.log(P1 / S) * 1e4
        pth = sp.path(t, X)
        if pth:
            r[f"mfe_{name}"] = math.log(max(h for h, _ in pth) / S) * 1e4
            r[f"mae_{name}"] = math.log(min(l_ for _, l_ in pth) / S) * 1e4
        for x in moves:
            r[f"hit_{name}_{x:g}"] = first_touch(sp, t, S, X, x * 100)


HISTORY = 9 * 86400  # historique en amont : notes techniques 1 h / 4 h et references Stocktwits 7 j


def build_dataset(con, tfs, horizons, moves, step, assets, since=0):
    layers = load_polymarket(con, tfs, since)
    ext = Ext(con, assets, max(since - HISTORY, 0))
    spans = {}
    for asset in assets:
        mks = [m for L in layers.get(asset, {}).values() for m in L.mk]
        if mks:
            spans[asset] = (max(min(m["ts"][0] for m in mks), since), max(m["ts"][-1] for m in mks))
    if not spans:
        return []
    lo, hi = min(s[0] for s in spans.values()), max(s[1] for s in spans.values())
    print("  bougies Binance BTCUSDT (reference) ...", flush=True)
    ensure_klines(con, "BTCUSDT", lo - HISTORY, hi + max(horizons.values()) + 3600)
    btc = Series(con, "BTCUSDT", int(lo) - HISTORY)
    rows = []
    for asset, (t_lo, t_hi) in spans.items():
        sym = SYMBOLS[asset]
        if asset == "btc":
            sp = btc
        else:
            print(f"  {asset}: bougies Binance {sym} ...", flush=True)
            ensure_klines(con, sym, t_lo - HISTORY, t_hi + max(horizons.values()) + 3600)
            sp = Series(con, sym, int(t_lo) - HISTORY)
        t = (int(t_lo) // step + 1) * step
        while t <= t_hi:
            r = features_at(asset, t, layers, sp, tfs, ext, btc, step)
            if r:
                add_labels(r, sp, horizons, moves)
                rows.append(r)
            t += step
    return rows


def feature_names(rows, tfs):
    names = [f"{tf}_{k}" for tf in tfs for k in LAYER_FEATS]
    names += [f"{tf}_nx_{k}" for tf in tfs for k in ("p", "flowcum")]
    names += ["ml_div_mean", "ml_div_agree", "ml_flow_mean", "ta_r1z", "ta_r15z", "ta_r60z", "ta_rsi", "ta_macd",
              "ta_bb", "ta_rv60", "ta_volratio", "ta_tv_15m", "ta_tv_1h", "ta_tv_4h", *X_FEATS,
              *pack_names(), "xa_btc_r1z", "xa_btc_r15z", "xa_btc_r60z", "xa_rel60"]
    return [n for n in names if sum(1 for r in rows if r.get(n) is not None) >= 100]


def split_time(rows, train_frac):
    times = sorted({r["t"] for r in rows})
    return times[int(len(times) * train_frac)] if len(times) >= 200 else None


# ---------------------------------------------------------------- 1. IC

def ic_report(rows, feats, horizons, step):
    print("\n=== 1. Pouvoir predictif de chaque facteur (IC = correlation de rang avec le rendement a t+X) ===")
    print("    cellule = IC brut / IC partiel (momentum spot retire) ; * |t|>2, ** |t|>3 (n effectif corrige du")
    print("    chevauchement). Avec autant de tests, quelques * apparaissent par hasard : seuls les ** stables comptent.\n")
    print(f"{'facteur':<17}" + "".join(f"{h:>18}" for h in horizons))
    found = []
    for f in feats:
        line = f"{f:<17}"
        for h, X in horizons.items():
            sub = [r for r in rows if r.get(f) is not None and r.get(f"y_{h}") is not None
                   and all(r.get(c) is not None for c in CONTROLS)]
            if len(sub) < 100:
                line += f"{'-':>18}"
                continue
            fx = ranks([r[f] for r in sub])
            fy = ranks([r[f"y_{h}"] for r in sub])
            raw = pearson(fx, fy)
            part = float("nan")
            if f not in CONTROLS:
                ctrl = list(zip(*[ranks([r[c] for r in sub]) for c in CONTROLS]))
                part = pearson(residualize(fx, ctrl), residualize(fy, ctrl))
            n_eff = len({r["t"] for r in sub}) * min(1.0, step / X)
            ic = raw if f in CONTROLS else part
            tstat = ic * math.sqrt(max(n_eff - 2, 1)) / math.sqrt(max(1 - ic * ic, 1e-9)) if ic == ic else 0.0
            mark = "**" if abs(tstat) > 3 else ("*" if abs(tstat) > 2 else "")
            cell = f"{raw:+.3f}/" + (f"{part:+.3f}" if part == part else "  -   ") + f"{mark:<2}"
            line += f"{cell:>18}"
            found.append((abs(tstat), f, h, raw, part, tstat))
        print(line)
    found.sort(reverse=True)
    print("\n  Facteurs les plus significatifs :")
    for _, f, h, raw, part, ts in found[:8]:
        print(f"    {f:<17} horizon {h:<4} IC brut {raw:+.3f}  partiel {part:+.3f}  t={ts:+.1f}")


# ---------------------------------------------------------------- 2. arbres : agencements -> mouvement de x %

def grow_tree(X, y, idx, feats, depth, min_leaf, path=()):
    pos = sum(y[i] for i in idx)
    n = len(idx)
    node = {"n": n, "rate": pos / n if n else 0, "path": path}
    if depth == 0 or n < 2 * min_leaf or pos == 0 or pos == n:
        return [node]
    best = None
    parent_g = n * (1 - (pos / n) ** 2 - (1 - pos / n) ** 2)
    for f in feats:
        vals = sorted((X[i][f], y[i]) for i in idx)
        cum = [0]
        for _, yy in vals:
            cum.append(cum[-1] + yy)
        for q in range(1, 20):
            k = q * n // 20
            if k < min_leaf or n - k < min_leaf or vals[k - 1][0] == vals[k][0]:
                continue
            pl, pr = cum[k] / k, (pos - cum[k]) / (n - k)
            g = k * (1 - pl * pl - (1 - pl) ** 2) + (n - k) * (1 - pr * pr - (1 - pr) ** 2)
            if best is None or g < best[0]:
                best = (g, f, (vals[k - 1][0] + vals[k][0]) / 2)
    if best is None or best[0] >= parent_g * 0.995:
        return [node]
    _, f, thr = best
    left = [i for i in idx if X[i][f] <= thr]
    right = [i for i in idx if X[i][f] > thr]
    return (grow_tree(X, y, left, feats, depth - 1, min_leaf, path + ((f, "<=", thr),)) +
            grow_tree(X, y, right, feats, depth - 1, min_leaf, path + ((f, ">", thr),)))


def matches(x, path):
    return all((x[f] <= thr) if op == "<=" else (x[f] > thr) for f, op, thr in path)


def rules_report(rows, feats, horizons, moves, train_frac, step):
    print("\n=== 2. Agencements (arbres de decision) menant a +x % ou -x % avant le mouvement inverse ===")
    print("    regle trouvee sur l'entrainement ; 'test' = taux reel sur la periode suivante jamais vue.")
    print("    OK = taux test >= 1.3 x taux de base, >= 30 episodes independants en test (points espaces de moins")
    print("    que l'horizon = chevauchement), et mouvement annonce >= 1.3 x mouvement inverse")
    print("    (sinon la regle detecte seulement de la volatilite, pas une direction).\n")
    cut = split_time(rows, train_frac)
    if cut is None:
        print("  pas assez de donnees")
        return
    tfeats = [f for f in feats if sum(1 for r in rows if r.get(f) is not None) >= 0.9 * len(rows)]
    for h, X in horizons.items():
        for x in moves:
            key = f"hit_{h}_{x:g}"
            train = [r for r in rows if r.get(key) is not None and r["t"] + X <= cut]
            test = [r for r in rows if r.get(key) is not None and r["t"] >= cut]
            if len(train) < 500 or len(test) < 150:
                continue
            med = {f: median([r[f] for r in train if r.get(f) is not None]) for f in tfeats}
            Xtr = [{f: (r[f] if r.get(f) is not None else med[f]) for f in tfeats} for r in train]
            Xte = [{f: (r[f] if r.get(f) is not None else med[f]) for f in tfeats} for r in test]
            for direction, label in ((1, f"hausse {x:g}%"), (-1, f"baisse {x:g}%")):
                ytr = [1 if r[key] == direction else 0 for r in train]
                yte = [1 if r[key] == direction else 0 for r in test]
                base_tr, base_te = mean(ytr), mean(yte)
                if not 0.03 <= base_tr <= 0.7:
                    continue
                leaves = grow_tree(Xtr, ytr, list(range(len(train))), tfeats, 3, max(50, len(train) // 50))
                good = []
                for lf in leaves:
                    if not lf["path"] or lf["rate"] < 1.25 * base_tr:
                        continue
                    idx = [i for i, x_ in enumerate(Xte) if matches(x_, lf["path"])]
                    if len(idx) >= 20:
                        rte = mean([yte[i] for i in idx])
                        # taux du mouvement INVERSE dans la meme configuration : une forte volatilite fait toucher
                        # +x % comme -x % ; seule l'asymetrie entre les deux est un signal de direction
                        rinv = mean([1 if test[i][key] == -direction else 0 for i in idx])
                        good.append((rte / base_te if base_te else 0, lf, len(idx), rte, rinv))
                print(f"  {label:<13} en {h:<4} taux de base : train {base_tr:5.1%}  test {base_te:5.1%}")
                for lift, lf, nte, rte, rinv in sorted(good, key=lambda g: -g[0])[:3]:
                    rule = " ET ".join(f"{f} {op} {thr:+.3g}" for f, op, thr in lf["path"])
                    n_eff = nte * min(1.0, step / X)
                    ok = lift >= 1.3 and n_eff >= 30 and rte >= 1.3 * rinv
                    print(f"     {'OK ' if ok else '   '}train {lf['rate']:5.1%} (n={lf['n']:<5}) "
                          f"test {rte:5.1%} (n={nte:<5} ~{n_eff:4.0f} indep.) x{lift:4.2f} inverse {rinv:5.1%} | {rule}")


# ---------------------------------------------------------------- 3. signal combine + TP/SL

def standardize(train, others, feats):
    cols = []
    for f in feats:
        v = [r[f] for r in train if r.get(f) is not None]
        if len(v) < 0.8 * len(train) or len(v) < 50:
            continue
        m = mean(v)
        sd = math.sqrt(sum((x - m) ** 2 for x in v) / len(v))
        if sd > 0:
            cols.append((f, m, sd))
    for r in train + others:
        r["_z"] = [((r[f] - m) / sd if r.get(f) is not None else 0.0) for f, m, sd in cols]
    return cols


def normal_eq(rows, target):
    k = len(rows[0]["_z"])
    A = [[0.0] * k for _ in range(k)]
    b, s = [0.0] * k, [0.0] * k
    for r in rows:
        x, y = r["_z"], r[target]
        for i in range(k):
            xi = x[i]
            if xi:
                b[i] += xi * y
                s[i] += xi
                Ai = A[i]
                for j in range(i, k):
                    Ai[j] += xi * x[j]
    for i in range(k):
        for j in range(i):
            A[i][j] = A[j][i]
    return A, b, s


def ridge(A, b, s, n, ym, lam):
    k = len(b)
    return solve([[A[i][j] + (lam if i == j else 0) for j in range(k)] for i in range(k)],
                 [b[i] - ym * s[i] for i in range(k)])


def fit_ridge(train, target):
    cut = int(len(train) * 0.8)
    fit, val = train[:cut], train[cut:]
    A, b, s = normal_eq(fit, target)
    ym = mean([r[target] for r in fit])
    best = None
    for lam_k in (0.01, 0.1, 1.0, 10.0):
        w = ridge(A, b, s, len(fit), ym, lam_k * len(fit))
        ic = pearson(ranks([sum(a * z for a, z in zip(w, r["_z"])) for r in val]), ranks([r[target] for r in val]))
        if best is None or (ic == ic and ic > best[0]):
            best = (ic, lam_k)
    A, b, s = normal_eq(train, target)
    ym = mean([r[target] for r in train])
    return ridge(A, b, s, len(train), ym, best[1] * len(train)), ym, best[1]


def predict(rows, w, ym):
    return [ym + sum(a * z for a, z in zip(w, r["_z"])) for r in rows]


def exit_levels(S, rv, side, X, spec):
    """Prix d'objectif et de stop. spec = {"mode": "pct", "tp": %, "sl": %} | {"mode": "vol", "tp": k, "sl": k}
    (k x volatilite 60 min ramenee a l'horizon) | {"mode": "none"} (sortie a t+X seulement)."""
    if spec["mode"] == "pct":
        tp_b, sl_b = spec["tp"] * 100, spec["sl"] * 100
    elif spec["mode"] == "vol":
        sig = rv * math.sqrt(X / 60) * 1e4
        tp_b, sl_b = spec["tp"] * sig, spec["sl"] * sig
    else:
        return None, None
    return S * math.exp(side * tp_b / 1e4), S * math.exp(-side * sl_b / 1e4)


def run_trade(sp, S, t, rv, side, X, spec):
    """Rejoue une position sur les bougies 1 min : sortie a l'objectif, au stop, ou a t+X (duree max)."""
    tp, sl = exit_levels(S, rv, side, X, spec)
    m = (int(t) + 59) // 60 * 60
    while m + 60 <= t + X:
        i = sp.idx.get(m)
        if i is not None and tp is not None:
            hi, lo = sp.h[i], sp.l[i]
            # meme bougie touchant les deux : on suppose le stop touche en premier (prudent)
            if (side > 0 and lo <= sl) or (side < 0 and hi >= sl):
                return side * math.log(sl / S), m + 60, "stop"
            if (side > 0 and hi >= tp) or (side < 0 and lo <= tp):
                return side * math.log(tp / S), m + 60, "objectif"
        m += 60
    P = sp.price(t + X)
    return (side * math.log(P / S), t + X, "duree") if P else (None, t + X, None)


def usd_pnl(g, side, unit, cost_bps):
    """Gain en $ d'une mise `unit` (achat, ou vente a decouvert de la meme valeur), frais aller-retour deduits."""
    gross = unit * (math.exp(g) - 1) if side > 0 else unit * (1 - math.exp(-g))
    return gross - unit * cost_bps / 1e4


def simulate(rows, preds, hi, lo, X, spec, sim):
    sides = [1 if p >= hi else (-1 if sim["allow_short"] and p <= lo else 0) for p in preds]
    return simulate_sides(rows, sides, X, spec, sim)


def simulate_sides(rows, sides, X, spec, sim):
    """Une mise de sim['unit'] $ a chaque signal ; plusieurs positions peuvent courir en meme temps, avec un
    delai minimal (cooldown) entre deux entrees sur le meme actif et un plafond de positions ouvertes."""
    order = sorted(range(len(rows)), key=lambda i: rows[i]["t"])
    last_entry, open_exits, trades = {}, [], []
    for i in order:
        r, side = rows[i], int(sides[i])
        if not side or r["t"] - last_entry.get(r["asset"], -1e18) < sim["cooldown"]:
            continue
        open_exits = [e for e in open_exits if e > r["t"]]
        if len(open_exits) >= sim["max_open"]:
            continue
        g, t_exit, why = run_trade(r["_sp"], r["_S"], r["t"], r["_rv"], side, X, spec)
        if g is None:
            continue
        open_exits.append(t_exit)
        last_entry[r["asset"]] = r["t"]
        trades.append({"t": r["t"], "exit": t_exit, "usd": usd_pnl(g, side, sim["unit"], sim["cost"]),
                       "why": why, "open": len(open_exits)})
    return trades


def pnl_summary(trades, unit):
    if not trades:
        return {"n": 0, "win": 0, "usd": 0, "mean": 0, "pf": 0, "max_open": 0, "dd": 0, "per_day": 0,
                "tp": 0, "sl": 0}
    pnl = [t["usd"] for t in trades]
    wins = [p for p in pnl if p > 0]
    losses = [-p for p in pnl if p < 0]
    eq = peak = dd = 0.0
    for t in sorted(trades, key=lambda t: t["exit"]):
        eq += t["usd"]
        peak = max(peak, eq)
        dd = max(dd, peak - eq)
    days = max((max(t["exit"] for t in trades) - min(t["t"] for t in trades)) / 86400, 1 / 24)
    return {"n": len(pnl), "win": len(wins) / len(pnl), "usd": sum(pnl), "mean": mean(pnl),
            "pf": sum(wins) / sum(losses) if losses else 99.0, "max_open": max(t["open"] for t in trades),
            "dd": dd, "per_day": sum(pnl) / days,
            "tp": sum(t["why"] == "objectif" for t in trades) / len(pnl),
            "sl": sum(t["why"] == "stop" for t in trades) / len(pnl)}


def fmt(s, unit):
    if not s["n"]:
        return "aucun trade"
    return (f"{s['n']:>4} trades | gagnants {s['win']:5.1%} | objectif {s['tp']:4.0%} stop {s['sl']:4.0%} | "
            f"total {s['usd']:+8.2f} $ ({s['per_day']:+.2f} $/j) | moy {s['mean']:+.3f} $ | PF {s['pf']:4.2f} | "
            f"max {s['max_open']} ouvertes ({s['max_open'] * unit:.0f} $ engages) | repli max {s['dd']:.2f} $")


def spec_label(spec):
    if spec["mode"] == "pct":
        return f"objectif +{spec['tp']:g}% / stop -{spec['sl']:g}%"
    if spec["mode"] == "vol":
        return f"objectif {spec['tp']:g}x vol / stop {spec['sl']:g}x vol"
    return "sortie a t+X"


def spec_grid(args, moves):
    if args.tp_pct is not None and args.sl_pct is not None:
        return [{"mode": "pct", "tp": args.tp_pct, "sl": args.sl_pct}]
    grid = [{"mode": "none"}]
    grid += [{"mode": "pct", "tp": a, "sl": b} for a in moves for b in moves]
    grid += [{"mode": "vol", "tp": a, "sl": b} for a in (0.5, 1.0, 2.0) for b in (0.5, 1.0, 2.0)]
    return grid


def model_report(rows, feats, horizons, moves, args):
    print("\n=== 3. Signal multi-couches combine (ridge) : entraine sur le debut, teste sur la fin ===")
    print(f"    scenario : {args.unit_usd:g} $ par signal, cooldown {args.cooldown} par actif, "
          f"max {args.max_open} positions ouvertes, frais {args.cost_bps:g} bps aller-retour")
    cut = split_time(rows, args.train_frac)
    if cut is None:
        print("  pas assez de donnees pour un walk-forward (continuer la collecte)")
        return []
    grid = spec_grid(args, moves)
    candidates = []
    for h, X in horizons.items():
        tgt = f"y_{h}"
        data = [r for r in rows if r.get(tgt) is not None]
        train = [r for r in data if r["t"] + X <= cut]  # aucune etiquette d'entrainement ne deborde sur le test
        test = [r for r in data if r["t"] >= cut]
        if len(train) < 300 or len(test) < 100:
            print(f"\n  horizon {h}: donnees insuffisantes (train {len(train)}, test {len(test)})")
            continue
        cols = standardize(train, test, feats)
        if not cols:
            continue
        w, ym, lam_k = fit_ridge(train, tgt)
        ptr, pte = predict(train, w, ym), predict(test, w, ym)
        oos = pearson(ranks(pte), ranks([r[tgt] for r in test]))
        top = sorted(zip(w, (c[0] for c in cols)), key=lambda x: -abs(x[0]))[:6]
        print(f"\n  --- horizon {h} | train {len(train)} pts, test {len(test)} pts | IC hors echantillon {oos:+.3f}")
        print("  poids principaux : " + ", ".join(f"{c} {wi:+.2f}" for wi, c in top))

        order = sorted(range(len(test)), key=lambda i: pte[i])
        print(f"  {'decile':<7}{'rend. moy':>11}{'hausse %':>10}{'MFE med':>10}{'MAE med':>10}   (bps, test)")
        for d in range(10):
            idx = order[d * len(order) // 10:(d + 1) * len(order) // 10]
            ys = [test[i][tgt] for i in idx]
            mf = [test[i][f"mfe_{h}"] for i in idx if test[i].get(f"mfe_{h}") is not None]
            ma = [test[i][f"mae_{h}"] for i in idx if test[i].get(f"mae_{h}") is not None]
            print(f"  D{d + 1:<6}{mean(ys):>+11.1f}{sum(y > 0 for y in ys) / len(ys):>10.1%}"
                  f"{median(mf):>+10.1f}{median(ma):>+10.1f}")

        s = sorted(ptr)
        hi, lo = s[int((1 - args.quantile) * (len(s) - 1))], s[int(args.quantile * (len(s) - 1))]
        for allow_short, mode in ((False, "achat seul"), (True, "achat + vente")):
            sim = {"unit": args.unit_usd, "cost": args.cost_bps, "cooldown": args.cooldown_s,
                   "max_open": args.max_open, "allow_short": allow_short}
            # objectif / stop choisis sur l'entrainement uniquement
            spec = max(grid, key=lambda g: sum(t["usd"] for t in simulate(train, ptr, hi, lo, X, g, sim)) or -1e18)
            res = pnl_summary(simulate(test, pte, hi, lo, X, spec, sim), args.unit_usd)
            print(f"  {mode:<14} [{spec_label(spec):<26}] {fmt(res, args.unit_usd)}")
            candidates.append({"type": "ridge", "horizon": h, "X": X, "mode": mode, "allow_short": allow_short,
                               "oos_ic": oos,
                               "spec": spec, "lam_k": lam_k, "n_train": len(train), "n_test": len(test),
                               "test": res})
    print("\n  objectif/stop choisis sur l'entrainement ; resultats ci-dessus = periode de test jamais vue.")
    print("  si ni objectif ni stop n'est touche, la position est fermee a t+X (duree max = horizon).")
    return candidates


def neutral_of(rows, key, ycol):
    """Rendement moyen (pb) quand ni +x ni -x n'est touche dans l'horizon."""
    v = [r[ycol] for r in rows if r[key] == 0]
    return mean(v) if v else 0.0


def composite_report(rows, feats, args):
    print("\n=== 4. Indicateur composite (gradient boosting) : situation complete -> +x % avant -x % ? ===")
    try:
        import numpy as np
        import composite as cp
    except ImportError:
        print("  scikit-learn absent : python -m pip install scikit-learn")
        return []
    cut = split_time(rows, args.train_frac)
    if cut is None:
        print("  pas assez de donnees")
        return []
    comp_h = {h.strip(): parse_duration(h) for h in args.comp_horizons.split(",") if h.strip()}
    moves = [float(x) for x in args.moves.split(",") if x.strip()]
    print("    Le modele lit tous les facteurs a la fois et apprend leurs combinaisons. Signal = esperance de gain")
    print("    (objectif +x %, stop -x %, sortie a l'horizon, frais deduits) au-dessus d'une marge choisie en validation.")
    print("    annonce = gain moyen promis pour les signaux emis ; obtenu = gain reel de ces signaux (pb, test jamais vu).")
    print(f"\n  {'horizon':<8}{'mouv.':<7}{'mode':<15}{'base +/-':<12}{'marge':>6}{'signaux':>8}{'reussite':>9}"
          f"{'annonce':>9}{'obtenu':>8}   simulation {args.unit_usd:g} $ (cooldown, plafond)")
    cands, best_detail = [], None
    for h, X in comp_h.items():
        for x in moves:
            key, ycol, x_bps = f"hit_{h}_{x:g}", f"y_{h}", x * 100
            data = [r for r in rows if r.get(key) is not None and r.get(ycol) is not None]
            train = [r for r in data if r["t"] + X <= cut]  # aucune etiquette d'entrainement ne deborde sur le test
            test = [r for r in data if r["t"] >= cut]
            if len(train) < 1000 or len(test) < 300:
                continue
            up_b = mean([r[key] == 1 for r in train])
            dn_b = mean([r[key] == -1 for r in train])
            if min(up_b, dn_b) < 0.03:
                continue
            # 1) validation interne (fin de la periode d'entrainement) : choix de la marge
            times = sorted({r["t"] for r in train})
            cut2 = times[int(len(times) * 0.8)]
            fit_rows = [r for r in train if r["t"] + X <= cut2]
            val = [r for r in train if r["t"] >= cut2]
            m1, n_iter = cp.fit(fit_rows, feats, key, val=val) if len(fit_rows) >= 500 and len(val) >= 100 \
                else (None, 0)
            if m1 is None:
                continue
            u, d = cp.probs(m1, cp.matrix(val, feats))
            evl, evs = cp.expectancy(u, d, x_bps, neutral_of(fit_rows, key, ycol), args.cost_bps)
            margins = {}
            for allow_short in (False, True):
                best = None
                for mg in cp.MARGINS:
                    real = cp.realized(val, key, h, x_bps, cp.decide(evl, evs, mg, allow_short), args.cost_bps)
                    if len(real) >= 20 and (best is None or sum(real) > best[1]):
                        best = (mg, sum(real))
                margins[allow_short] = best
            # 2) modele entraine sur toute la periode d'entrainement, evalue sur le test
            m, _ = cp.fit(train, feats, key, n_iter=n_iter)
            Xte = cp.matrix(test, feats)
            u, d = cp.probs(m, Xte)
            evl, evs = cp.expectancy(u, d, x_bps, neutral_of(train, key, ycol), args.cost_bps)
            oos = pearson(ranks((evl - evs).tolist()), ranks([r[ycol] for r in test]))
            for allow_short, mode in ((False, "achat seul"), (True, "achat + vente")):
                mb = margins[allow_short]
                if mb is None:
                    print(f"  {h:<8}{x:<7g}{mode:<15}{up_b:4.0%}/{dn_b:<7.0%}  trop peu de signaux en validation")
                    continue
                side = cp.decide(evl, evs, mb[0], allow_short)
                fired = side != 0
                real = cp.realized(test, key, h, x_bps, side, args.cost_bps)
                promised = float(np.where(side == 1, evl, evs)[fired].mean()) if fired.any() else 0.0
                hits = sum(1 for r, sd in zip(test, side) if sd and r[key] == sd)
                sim = {"unit": args.unit_usd, "cost": args.cost_bps, "cooldown": args.cooldown_s,
                       "max_open": args.max_open, "allow_short": allow_short}
                res = pnl_summary(simulate_sides(test, side, X, {"mode": "pct", "tp": x, "sl": x}, sim), args.unit_usd)
                ok_val = mb[1] > 0
                print(f"  {h:<8}{x:<7g}{mode:<15}{up_b:4.0%}/{dn_b:<7.0%}{mb[0]:>6}{len(real):>8}"
                      f"{(hits / len(real) if real else 0):>9.0%}{promised:>+9.1f}{(mean(real) if real else 0):>+8.1f}"
                      f"   {res['n']} trades, {res['usd']:+.2f} $ ({res['per_day']:+.2f} $/j), PF {res['pf']:.2f}"
                      f"{'' if ok_val else '  [perdant en validation]'}")
                c = {"type": "composite", "horizon": h, "X": X, "move": x, "mode": mode, "allow_short": allow_short,
                     "oos_ic": oos, "spec": {"mode": "pct", "tp": x, "sl": x}, "lam_k": None, "margin": mb[0],
                     "n_train": len(train), "n_test": len(test), "test": res, "val_ok": ok_val, "n_iter": n_iter,
                     "promised": promised, "real": mean(real) if real else 0.0}
                cands.append(c)
                if res["n"] and (best_detail is None or res["per_day"] > best_detail[0]["test"]["per_day"]):
                    best_detail = (c, m, test, Xte, u, d, key)
    if best_detail is None:
        print("  aucune combinaison horizon x mouvement exploitable (continuer la collecte)")
        return cands
    c, m, test, Xte, u, d, key = best_detail
    print(f"\n  Detail de la meilleure combinaison (horizon {c['horizon']}, +/-{c['move']:g} %, {c['mode']}, "
          f"{c['n_iter']} arbres) :")
    print("  calibration P(hausse) sur le test : quand le modele annonce p, la hausse arrive-t-elle ~p du temps ?")
    y = np.array([r[key] for r in test])
    order = np.argsort(u)
    for q, idx in enumerate(np.array_split(order, 5)):
        print(f"    quintile {q + 1} : annonce {u[idx].mean():5.1%}  ->  reel {np.mean(y[idx] == 1):5.1%}"
              f"   |  baisse annoncee {d[idx].mean():5.1%} -> reelle {np.mean(y[idx] == -1):5.1%}")
    try:
        from sklearn.inspection import permutation_importance
        idx = np.random.default_rng(0).choice(len(test), size=min(3000, len(test)), replace=False)
        imp = permutation_importance(m, Xte[idx], y[idx], n_repeats=3, random_state=0, scoring="neg_log_loss")
        top = np.argsort(-imp.importances_mean)[:10]
        print("  facteurs qui comptent le plus pour le modele (perte de precision si on les brouille) :")
        for j in top:
            if imp.importances_mean[j] > 0:
                print(f"    {cp.label(feats[j]):<44} {imp.importances_mean[j] * 1000:+6.2f}")
    except Exception as e:
        print(f"  (importance des facteurs indisponible : {e})")
    return cands


# ---------------------------------------------------------------- 5. meta-etiquetage des signaux classiques

META_FEATS = ("m_rule", "m_tf", "m_dir")
META_THRESHOLDS = (0.0, 0.3, 0.4, 0.5, 0.6, 0.7)  # chance de reussite minimale ; 0 = sans filtre


def meta_rows(rows):
    """Une ligne par declencheur classique survenu (regle, echelle, sens), avec tout l'etat du marche a cet instant."""
    out = []
    for r in rows:
        for rule, tf, d, _ in r.get("_events") or ():
            m = dict(r)
            m.update({"m_rule": RULES.index(rule), "m_tf": TF_NAMES.index(tf), "m_dir": d, "_rule": f"{rule} {tf}"})
            out.append(m)
    return out


def meta_report(rows, feats, args):
    print("\n=== 5. Meta-etiquetage : les signaux techniques classiques, filtres par l'algorithme ===")
    print("    Chaque declencheur classique (Bollinger, RSI, Donchian, Supertrend, MACD) propose un trade dans son sens.")
    print("    seule = on les prend tous ; filtree = seulement ceux que l'algo juge bons au vu de TOUT l'etat du marche")
    print("    (Polymarket, positionnement, sentiment, autres indicateurs). Gains en pb, frais deduits, test jamais vu.")
    try:
        import numpy as np
        import composite as cp
        from technical import RULE_LABELS
    except ImportError:
        print("  scikit-learn absent : python -m pip install scikit-learn")
        return []
    events = meta_rows(rows)
    cut = split_time(rows, args.train_frac)
    if cut is None or len(events) < 300:
        print(f"  {len(events)} declencheurs seulement : continuer la collecte")
        return []
    mfeats = list(feats) + list(META_FEATS)
    comp_h = {h.strip(): parse_duration(h) for h in args.comp_horizons.split(",") if h.strip()}
    moves = [float(x) for x in args.moves.split(",") if x.strip()]
    print(f"\n  {'horizon':<8}{'mouv.':<7}{'mode':<15}{'seule : n / reussite / gain':>30}"
          f"{'filtree : seuil / n / reussite / gain':>40}   simulation {args.unit_usd:g} $")
    cands, best = [], None
    for h, X in comp_h.items():
        for x in moves:
            key, ycol, x_bps, lab = f"hit_{h}_{x:g}", f"y_{h}", x * 100, f"ms_{h}_{x:g}"
            data = [e for e in events if e.get(key) is not None and e.get(ycol) is not None]
            for e in data:
                e[lab] = 1 if e[key] == e["m_dir"] else 0
            train = [e for e in data if e["t"] + X <= cut]
            test = [e for e in data if e["t"] >= cut]
            if len(train) < 300 or len(test) < 60:
                continue
            times = sorted({e["t"] for e in train})
            cut2 = times[int(len(times) * 0.8)]
            fit_rows = [e for e in train if e["t"] + X <= cut2]
            val = [e for e in train if e["t"] >= cut2]
            if len(fit_rows) < 200 or len(val) < 40:
                continue
            m1, n_iter = cp.fit(fit_rows, mfeats, lab, val=val)
            if m1 is None:
                continue
            pv = cp.probs(m1, cp.matrix(val, mfeats))[0]
            m, _ = cp.fit(train, mfeats, lab, n_iter=n_iter)
            pt = cp.probs(m, cp.matrix(test, mfeats))[0]
            outcome = cp.realized(test, key, h, x_bps, [e["m_dir"] for e in test], args.cost_bps)
            oos = pearson(ranks(pt.tolist()), ranks(outcome))
            for allow_short, mode in ((False, "achat seul"), (True, "achat + vente")):
                def sides_for(evs, ps, thr):
                    return [e["m_dir"] if (allow_short or e["m_dir"] > 0) and pr >= thr else 0 for e, pr in zip(evs, ps)]
                choice = None
                for thr in META_THRESHOLDS:
                    real = cp.realized(val, key, h, x_bps, sides_for(val, pv, thr), args.cost_bps)
                    if len(real) >= 10 and (choice is None or sum(real) > choice[1]):
                        choice = (thr, sum(real))
                if choice is None:
                    continue
                raw_sides = sides_for(test, np.ones(len(test)), 0.0)
                raw = cp.realized(test, key, h, x_bps, raw_sides, args.cost_bps)
                sides = sides_for(test, pt, choice[0])
                real = cp.realized(test, key, h, x_bps, sides, args.cost_bps)
                raw_hit = sum(1 for e, sd in zip(test, raw_sides) if sd and e[key] == sd) / max(len(raw), 1)
                hit = sum(1 for e, sd in zip(test, sides) if sd and e[key] == sd) / max(len(real), 1)
                sim = {"unit": args.unit_usd, "cost": args.cost_bps, "cooldown": args.cooldown_s,
                       "max_open": args.max_open, "allow_short": allow_short}
                res = pnl_summary(simulate_sides(test, sides, X, {"mode": "pct", "tp": x, "sl": x}, sim), args.unit_usd)
                print(f"  {h:<8}{x:<7g}{mode:<15}{len(raw):>12} / {raw_hit:4.0%} / {(mean(raw) if raw else 0):+6.1f}"
                      f"{choice[0]:>14.0%} / {len(real):>4} / {hit:4.0%} / {(mean(real) if real else 0):+6.1f}"
                      f"   {res['n']} trades, {res['usd']:+.2f} $, PF {res['pf']:.2f}"
                      f"{'' if choice[1] > 0 else '  [perdant en validation]'}")
                c = {"type": "meta", "horizon": h, "X": X, "move": x, "mode": mode, "allow_short": allow_short,
                     "oos_ic": oos, "spec": {"mode": "pct", "tp": x, "sl": x}, "lam_k": None,
                     "threshold": choice[0], "n_iter": n_iter, "n_train": len(train), "n_test": len(test),
                     "test": res, "val_ok": choice[1] > 0, "promised": float(np.mean(pt)) * 100,
                     "real": mean(real) if real else 0.0}
                cands.append(c)
                if res["n"] and (best is None or res["per_day"] > best[0]["test"]["per_day"]):
                    best = (c, test, sides, raw_sides, key)
    if best is None:
        print("  pas encore assez de declencheurs exploitables (continuer la collecte)")
        return cands
    c, test, sides, raw_sides, key = best
    print(f"\n  Detail par regle (horizon {c['horizon']}, +/-{c['move']:g} %, {c['mode']}, test) :")
    print(f"    {'regle':<34}{'seule : n / reussite / gain':>30}{'filtree : n / gain':>22}")
    x_bps = c["move"] * 100
    for rule in sorted({e["_rule"] for e in test}):
        idx = [i for i, e in enumerate(test) if e["_rule"] == rule]
        rs = [raw_sides[i] if i in idx else 0 for i in range(len(test))]
        fs = [sides[i] if i in idx else 0 for i in range(len(test))]
        raw = cp.realized(test, key, c["horizon"], x_bps, rs, args.cost_bps)
        flt = cp.realized(test, key, c["horizon"], x_bps, fs, args.cost_bps)
        if not raw:
            continue
        hit = sum(1 for i in idx if rs[i] and test[i][key] == rs[i]) / len(raw)
        name, tf = rule.rsplit(" ", 1)
        print(f"    {RULE_LABELS.get(name, name) + ' ' + tf:<34}{len(raw):>12} / {hit:4.0%} / {mean(raw):+6.1f}"
              f"{len(flt):>14} / {(mean(flt) if flt else 0):+6.1f}")
    return cands


# ---------------------------------------------------------------- 4. publication de la strategie

def paper_report(con):
    if not con.execute("SELECT name FROM sqlite_master WHERE name = 'signals'").fetchone():
        return
    rows = con.execute("""SELECT strategy_id, COUNT(*), SUM(pnl_usd), AVG(pnl_usd > 0), MAX(unit_usd)
                          FROM signals WHERE status = 'closed' GROUP BY strategy_id ORDER BY strategy_id""").fetchall()
    if rows:
        print("\n=== Observations reelles du moteur de signal (paper trading, frais deduits) ===")
        for sid, n, tot, win, unit in rows:
            print(f"  strategie {time.strftime('%Y-%m-%d %H:%M', time.localtime(sid))} : {n:>4} signaux de {unit:g} $ | "
                  f"gagnants {win:5.1%} | total {tot:+8.2f} $")


def publish(con, rows, feats, tfs, cands, args, lab_specs=()):
    con.executescript(RUNS_SCHEMA)
    now = int(time.time())
    # duree REELLEMENT couverte : les trous de collecte ne comptent ni dans les jours ni dans la periode de test
    times = sorted({r["t"] for r in rows})
    days = len(times) * args.step / 86400
    cut = split_time(rows, args.train_frac) or times[-1]
    test_span = sum(1 for t in times if t >= cut) * args.step
    # la periode de test doit contenir au moins 30 horizons sans chevauchement : sinon quelques episodes
    # chanceux (ou un indicateur de regime lent) suffisent a "valider" une strategie
    ok = [c for c in cands if c["oos_ic"] > 0 and c["test"]["n"] >= args.min_trades and c["test"]["usd"] > 0
          and c["test"]["pf"] > 1.1 and test_span >= 30 * c["X"] and days >= args.min_days
          and c.get("val_ok", True) and c.get("real", 1) > 0]
    best = max(ok, key=lambda c: c["test"]["per_day"]) if ok else None
    for c in cands:
        t = c["test"]
        con.execute("INSERT INTO model_runs VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
                    (now, c["horizon"], f"{c['type']} {c['mode']}" + (f" +/-{c['move']:g}%" if "move" in c else ""),
                     c["oos_ic"], t["n"], t["usd"], t["per_day"], t["pf"],
                     json.dumps(c["spec"]), c["lam_k"], c["n_train"], c["n_test"], int(c is best)))
    con.commit()
    strat = {"created": now, "active": False, "tfs": tfs, "cost_bps": args.cost_bps, "unit_usd": args.unit_usd,
             "cooldown": args.cooldown_s, "max_open": args.max_open}
    if best is None:
        strat["reason"] = (f"aucune combinaison ne passe la validation hors echantillon (IC>0, gain>0, PF>1.1, "
                           f">= {args.min_days:g} j de donnees, test >= 30 horizons) ; donnees : {days:.1f} j")
    elif best["type"] == "meta":
        import numpy as np
        import composite as cp
        key, ycol, lab = f"hit_{best['horizon']}_{best['move']:g}", f"y_{best['horizon']}", "_success"
        data = [e for e in meta_rows(rows) if e.get(key) is not None and e.get(ycol) is not None]
        for e in data:
            e[lab] = 1 if e[key] == e["m_dir"] else 0
        mfeats = list(feats) + list(META_FEATS)
        model, _ = cp.fit(data, mfeats, lab, n_iter=best["n_iter"])  # memes reglages, toute la fenetre
        M = cp.matrix(data, mfeats)
        medians = {f: float(np.nanmedian(M[:, j])) for j, f in enumerate(mfeats) if not np.isnan(M[:, j]).all()}
        cp.save(model, os.path.join(os.path.dirname(os.path.abspath(args.db)), "meta_model.pkl"))
        strat.update({
            "type": "meta", "active": True, "reason": "valide hors echantillon",
            "horizon": best["horizon"], "X": best["X"], "move": best["move"], "allow_short": best["allow_short"],
            "exit": best["spec"], "threshold": best["threshold"], "n_iter": best["n_iter"],
            "features": mfeats, "medians": medians, "model_file": "meta_model.pkl",
            "validation": {"oos_ic": best["oos_ic"], "real": best["real"], **best["test"]},
            "data": {"rows": len(data), "from": min(e["t"] for e in data), "to": max(e["t"] for e in data)},
        })
    elif best["type"] == "composite":
        import numpy as np
        import composite as cp
        key, ycol = f"hit_{best['horizon']}_{best['move']:g}", f"y_{best['horizon']}"
        data = [r for r in rows if r.get(key) is not None and r.get(ycol) is not None]
        model, _ = cp.fit(data, feats, key, n_iter=best["n_iter"])  # memes reglages, toute la fenetre
        M = cp.matrix(data, feats)
        medians = {f: float(np.nanmedian(M[:, j])) for j, f in enumerate(feats) if not np.isnan(M[:, j]).all()}
        cp.save(model, os.path.join(os.path.dirname(os.path.abspath(args.db)), "composite_model.pkl"))
        strat.update({
            "type": "composite", "active": True, "reason": "valide hors echantillon",
            "horizon": best["horizon"], "X": best["X"], "move": best["move"], "allow_short": best["allow_short"],
            "exit": best["spec"], "margin": best["margin"], "neutral": neutral_of(data, key, ycol),
            "n_iter": best["n_iter"],
            "features": feats, "medians": medians, "model_file": "composite_model.pkl",
            "validation": {"oos_ic": best["oos_ic"], "promised": best["promised"], "real": best["real"], **best["test"]},
            "data": {"rows": len(data), "from": min(r["t"] for r in data), "to": max(r["t"] for r in data)},
        })
    else:
        # memes hyperparametres, reentraine sur toute la fenetre de donnees pour le deploiement
        tgt = f"y_{best['horizon']}"
        data = [r for r in rows if r.get(tgt) is not None]
        cols = standardize(data, [], feats)
        A, b, s = normal_eq(data, tgt)
        ym = mean([r[tgt] for r in data])
        w = ridge(A, b, s, len(data), ym, best["lam_k"] * len(data))
        p = sorted(predict(data, w, ym))
        strat.update({
            "type": "ridge", "active": True, "reason": "valide hors echantillon",
            "horizon": best["horizon"], "X": best["X"], "allow_short": best["allow_short"], "exit": best["spec"],
            "ym": ym, "hi": p[int((1 - args.quantile) * (len(p) - 1))], "lo": p[int(args.quantile * (len(p) - 1))],
            "features": [{"name": f, "mean": m, "sd": sd, "w": wi} for (f, m, sd), wi in zip(cols, w)],
            "validation": {"oos_ic": best["oos_ic"], **best["test"]},
            "data": {"rows": len(data), "from": min(r["t"] for r in data), "to": max(r["t"] for r in data)},
        })
    if strat.get("active"):  # indicateurs du laboratoire utilises par la strategie : le moteur doit les recalculer
        used = {f["name"] if isinstance(f, dict) else f for f in strat["features"]}
        strat["lab"] = [{"id": s["id"], "formula": s["formula"], "norms": s["norms"]} for s in lab_specs
                        if f"lab_{s['id']}" in used]
    path = args.strategy or os.path.join(os.path.dirname(os.path.abspath(args.db)), "strategy.json")
    tmp = path + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump(strat, f, indent=1)
    os.replace(tmp, path)
    print(f"\n=== Strategie publiee : {path} -> {'ACTIVE' if strat['active'] else 'inactive'} ({strat['reason']})")
    if strat["active"]:
        print(f"    {strat['type']} | horizon {strat['horizon']} | {best['mode']} | {spec_label(best['spec'])} | "
              f"{fmt(best['test'], args.unit_usd)}")


# ---------------------------------------------------------------- principal

def run_once(args):
    tfs = [t.strip() for t in args.tfs.split(",") if t.strip() in TF_SECONDS]
    horizons = {h.strip(): parse_duration(h) for h in args.horizons.split(",") if h.strip()}
    label_h = dict(horizons)
    if not args.no_composite:
        label_h.update({h.strip(): parse_duration(h) for h in args.comp_horizons.split(",") if h.strip()})
    if not args.no_lab:
        label_h.update({h.strip(): parse_duration(h) for h in args.lab_horizons.split(",") if h.strip()})
    moves = [float(x) for x in args.moves.split(",") if x.strip()]
    assets = [a.strip() for a in args.assets.split(",") if a.strip()]
    since = int(time.time() - args.window_days * 86400) if args.window_days else 0
    con = connect(args.db)

    print(f"\n##### analyse {time.strftime('%Y-%m-%d %H:%M')} | base : {args.db}")
    for asset, tf, n, n_res, lo, hi in con.execute(
            """SELECT asset, tf, COUNT(*), SUM(outcome IN (0, 1)), MIN(start_ts), MAX(end_ts)
               FROM markets GROUP BY asset, tf ORDER BY asset, tf"""):
        print(f"  {asset} {tf:<4} {n:>6} fenetres ({n_res or 0} resolues)  "
              f"{time.strftime('%Y-%m-%d %H:%M', time.localtime(lo))} -> {time.strftime('%Y-%m-%d %H:%M', time.localtime(hi))}")
    for src, n in con.execute("SELECT src, COUNT(*) FROM book GROUP BY src"):
        print(f"  releves de cote '{src}' : {n}")
    paper_report(con)

    print("\nconstruction du jeu de donnees ...")
    rows = build_dataset(con, tfs, label_h, moves, args.step, assets, since)
    print(f"  {len(rows)} points (actif x instant)")
    if len(rows) < 200:
        print("Trop peu de donnees : laisser tourner collector.py (ou --backfill-days) puis relancer.")
        return
    days = len({r["t"] for r in rows}) * args.step / 86400  # jours reellement couverts (sans les trous)
    if days < 14:
        print(f"\n  !!! {days:.1f} jour(s) de donnees seulement : les resultats ci-dessous ne sont PAS significatifs")
        print("  !!! (un seul regime de marche ; train et test trop courts). Viser au moins 2 a 4 semaines.")
    feats = feature_names(rows, tfs)
    lab_specs = []
    if not args.no_lab:
        try:
            import lab
            from composite import label
            lab_h = {h.strip(): parse_duration(h) for h in args.lab_horizons.split(",") if h.strip()}
            print("\n=== Laboratoire d'indicateurs : formules inventees, jugees sur les donnees posterieures a leur "
                  "invention ===")
            lab_specs, lines = lab.run(con, rows, feats, args.step, lab_h, args.save, label=label)
            for line in lines or ["  aucun indicateur invente pour l'instant"]:
                print(line)
            feats += lab.add_features(rows, lab_specs)
        except ImportError:
            print("  laboratoire indisponible : python -m pip install scikit-learn")
    ic_report(rows, feats, horizons, args.step)
    rules_report(rows, feats, horizons, moves, args.train_frac, args.step)
    cands = model_report(rows, feats, horizons, moves, args)
    if not args.no_composite:
        cands += composite_report(rows, feats, args)
        cands += meta_report(rows, feats, args)
    if args.save:
        publish(con, rows, feats, tfs, cands, args, lab_specs)

    if args.csv:
        keys = sorted({k for r in rows for k in r if not k.startswith("_")})
        with open(args.csv, "w", newline="", encoding="utf-8") as f:
            wr = csv.DictWriter(f, fieldnames=keys, extrasaction="ignore")
            wr.writeheader()
            for r in rows:
                wr.writerow(r)
        print(f"\njeu de donnees exporte : {args.csv}")
    con.close()


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--db", default=default_db_path())
    ap.add_argument("--assets", default="btc,eth,sol,xrp")
    ap.add_argument("--tfs", default="5m,15m,1h,4h", help="couches Polymarket combinees")
    ap.add_argument("--horizons", default="5m,15m,1h,4h", help="horizons X du mouvement reel a predire")
    ap.add_argument("--moves", default="0.25,0.5,1", help="seuils x (en %%) des mouvements recherches")
    ap.add_argument("--step", type=int, default=60, help="pas de la grille temporelle (s)")
    ap.add_argument("--cost-bps", type=float, default=20, help="frais + spread aller-retour sur le marche reel")
    ap.add_argument("--train-frac", type=float, default=0.7)
    ap.add_argument("--quantile", type=float, default=0.1, help="signal si score dans les 10 %% extremes")
    ap.add_argument("--unit-usd", type=float, default=10, help="mise par signal ($)")
    ap.add_argument("--cooldown", default="5m", help="delai minimal entre deux entrees sur un meme actif")
    ap.add_argument("--max-open", type=int, default=20, help="positions ouvertes simultanement au maximum")
    ap.add_argument("--tp-pct", type=float, help="objectif fixe en %% (sinon choisi sur l'entrainement)")
    ap.add_argument("--sl-pct", type=float, help="stop-loss fixe en %% (avec --tp-pct)")
    ap.add_argument("--window-days", type=float, default=0, help="n'utiliser que les N derniers jours (0 = tout)")
    ap.add_argument("--save", action="store_true", help="historiser les essais et publier strategy.json")
    ap.add_argument("--strategy", help="chemin de strategy.json (defaut : a cote de la base)")
    ap.add_argument("--min-trades", type=int, default=30)
    ap.add_argument("--comp-horizons", default="15m,30m,1h,2h,4h", help="horizons de l'indicateur composite")
    ap.add_argument("--no-composite", action="store_true", help="sans l'indicateur composite (scikit-learn)")
    ap.add_argument("--lab-horizons", default="15m,1h,4h", help="horizons du laboratoire d'indicateurs")
    ap.add_argument("--no-lab", action="store_true", help="sans le laboratoire d'indicateurs")
    ap.add_argument("--min-days", type=float, default=14, help="jours de donnees minimum avant toute publication")
    ap.add_argument("--every", help="recommencer en boucle, ex. 6h (reentrainement continu)")
    ap.add_argument("--csv", help="exporter le jeu de donnees (une ligne par actif et par instant)")
    ap.add_argument("--delay", help="avec --every : attendre avant la premiere analyse, ex. 1h")
    args = ap.parse_args()
    args.cooldown_s = parse_duration(args.cooldown)
    sys.stdout.reconfigure(errors="replace")

    if not args.every:
        run_once(args)
        return
    lock = single_instance("analyze", args.db)  # noqa: F841 (une seule boucle d'apprentissage)
    period = parse_duration(args.every)
    if args.delay:
        print(f"premiere analyse dans {args.delay} (le temps que l'historique se charge) ...", flush=True)
        time.sleep(parse_duration(args.delay))
    while True:
        t0 = time.time()
        try:
            run_once(args)
        except Exception as e:  # une analyse ratee ne doit pas arreter la boucle
            print(f"!! analyse en erreur : {e}")
        wait = max(60, period - (time.time() - t0))
        print(f"\nprochaine analyse dans {wait / 3600:.1f} h (Ctrl+C pour arreter)")
        time.sleep(wait)


if __name__ == "__main__":
    main()