"""Convert mid/small Tiingo caches, rebuild the PIT panels, report the
survivorship change (pre-Tiingo backup vs rebuilt member counts per year).

    PYTHONPATH=. uv run python -m research.midsmall_rebuild
"""
from __future__ import annotations

import os
import pickle
import shutil

import pandas as pd


def main() -> None:
    stage = [l.strip() for l in open("data/research/tiingo_midsmall.txt")][:230]
    cached = [n for n in stage if os.path.exists(f"data/research/tiingo/{n}.csv")]
    print(f"mid/small tickers: {len(stage)} listed, {len(cached)} cached", flush=True)

    from research.lab.tiingo_merge import write_yahoo_mid
    write_yahoo_mid(cached)

    for f in ("us_mid_bars.pkl", "us_mid.panel.pkl", "us_small.panel.pkl", "us_midsmall.panel.pkl"):
        p = os.path.join("data", "research", f)
        if os.path.exists(p):
            os.remove(p)
            print(f"removed {p}", flush=True)

    from research.lab.panel_mid import load_us_mid
    new = load_us_mid("midsmall")
    with open("data/research/us_midsmall.panel.pre_tiingo.pkl", "rb") as fh:
        old = pickle.load(fh)

    n, o = new.member.sum(axis=1), old.member.sum(axis=1)
    comp = pd.DataFrame({"post": n.resample("YE").last(), "pre": o.resample("YE").last()}).dropna()
    comp["delta"] = comp["post"] - comp["pre"]
    comp.index = comp.index.year
    print("\nPIT members per year (pre -> post):")
    print(comp.to_string())
    from research.lab.fetch_us_mid import load_bars
    restored = [t for t in cached if t in new.close.columns
                and new.member[t].sum() > 0 and load_bars(t) is not None
                and t not in old.close.columns]
    print(f"\nnew PIT-covered names not in old panel: {len(restored)}")
    print(new.close.shape, flush=True)
    print("REBUILD DONE", flush=True)


if __name__ == "__main__":
    main()