"""Convert mid/small Tiingo caches, rebuild the PIT panels, report the
survivorship change (pre-Tiingo backup and the previous build vs rebuilt
member counts per year). Converts every cached ticker on the mid/small plan
(plus the batch list), so it is re-runnable after each monthly Tiingo batch.

    PYTHONPATH=. uv run python -m research.midsmall_rebuild
"""
from __future__ import annotations

import json
import os
import pickle

import pandas as pd


def convert_list() -> list[str]:
    plan = json.load(open("data/research/tiingo_plan.json"))
    txt = [l.strip() for l in open("data/research/tiingo_midsmall.txt") if l.strip()]
    want = set(plan["mid_small"]) | set(txt)
    cached = [f[:-4] for f in os.listdir("data/research/tiingo") if f.endswith(".csv")]
    return sorted(set(cached) & want)


def main() -> None:
    conv = convert_list()
    print(f"mid/small cached tickers to convert: {len(conv)}", flush=True)

    from research.lab.tiingo_merge import write_yahoo_mid
    write_yahoo_mid(conv)

    prev_counts = None
    pkl = "data/research/us_midsmall.panel.pkl"
    if os.path.exists(pkl):
        with open(pkl, "rb") as fh:
            prev = pickle.load(fh)
        prev_counts = prev.member.sum(axis=1).resample("YE").last()
        del prev

    for f in ("us_mid_bars.pkl", "us_mid.panel.pkl", "us_small.panel.pkl", "us_midsmall.panel.pkl"):
        p = os.path.join("data", "research", f)
        if os.path.exists(p):
            os.remove(p)
            print(f"removed {p}", flush=True)

    from research.lab.panel_mid import load_us_mid
    new = load_us_mid("midsmall")
    load_us_mid("mid")
    load_us_mid("small")
    with open("data/research/us_midsmall.panel.pre_tiingo.pkl", "rb") as fh:
        old = pickle.load(fh)

    n, o = new.member.sum(axis=1), old.member.sum(axis=1)
    comp = pd.DataFrame({"post": n.resample("YE").last(), "pre_tiingo": o.resample("YE").last(),
                         "prev_build": prev_counts}).dropna(subset=["pre_tiingo"])
    comp["delta_prev"] = comp["post"] - comp["prev_build"]
    comp.index = comp.index.year
    print("\nPIT members per year (pre_tiingo -> previous build -> post):")
    print(comp.to_string())
    from research.lab.fetch_us_mid import load_bars
    restored = [t for t in conv if t in new.close.columns
                and new.member[t].sum() > 0 and load_bars(t) is not None
                and t not in old.close.columns]
    print(f"\nnew PIT-covered names not in old panel: {len(restored)}")
    print(new.close.shape, flush=True)
    print("REBUILD DONE", flush=True)


if __name__ == "__main__":
    main()