fix(data): store raw price panels and refresh a trailing window on every update

The append-only updater only fetched when last_date+1 < local date, never
revisited appended rows and forward-filled before saving. First-seen values
(partial intraday bars, late or missing prints) were frozen permanently: vs
Yahoo, 22% (US), 58% (CN) and 27% (ETF) of recent days had >0.5% daily-return
errors, CN never received same-day data, and HK tickers created US-holiday rows.

Panels are now stored raw (NaN = no print) and forward-filled on read; every
update re-downloads the last 30 days for the requested tickers, rescales older
history when Yahoo's split/dividend adjustment moves, drops rows printed by a
minority of tickers, and writes atomically under a per-market file lock.
Tickers Yahoo has no data for are retried weekly instead of every run.
rebuild() repairs panels written by the old updater.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.gitignore

@@ -44,3 +44,8 @@ docs/superpowers/
 # OS
 .DS_Store
 Thumbs.db
+
+# Price panel locks, repair backups and Yahoo availability markers
+data/.*.lock
+data/backup_*/
+data/*_unavailable.json

data_manager.py

@@ -2,37 +2,78 @@
 Persistent local data store for price history.
 
 Layout:
-    data/us.csv        — S&P 500 + SPY adjusted close prices
+    data/{market}.csv       — adjusted close prices
-    data/us_open.csv   — S&P 500 + SPY adjusted open prices
+    data/{market}_open.csv  — adjusted open prices
-    data/cn.csv        — CSI 300 + 000300.SS adjusted close prices
+    data/{market}_unavailable.json — tickers Yahoo returned no data for (retried weekly)
-    data/cn_open.csv   — CSI 300 + 000300.SS adjusted open prices
 
-On first run: downloads full 10-year history.
+Panels are stored raw: a date/ticker Yahoo has no bar for stays NaN, so a
-On subsequent runs: reads existing file, downloads only new dates, appends.
+missing print is distinguishable from an unchanged price. ``load()`` and
-New tickers (index rebalance) are backfilled automatically.
+``update()`` forward-fill on read; ``load_raw()`` exposes the stored values.
+
+Every ``update()`` re-downloads a trailing ``REFRESH_DAYS`` window and
+overwrites it for the requested tickers. Partial intraday bars, late prints and
+rows that turn out not to be trading days are therefore corrected on the next
+run instead of being frozen. When Yahoo's split/dividend adjustment shifts the
+overlap, that ticker's older history is rescaled so returns stay continuous.
 """
 
+import fcntl
+import json
 import os
+from contextlib import contextmanager
 from datetime import datetime, timedelta
 
+import numpy as np
 import pandas as pd
 import yfinance as yf
 
 DATA_DIR = "data"
 
+# Trailing calendar days re-downloaded and overwritten on every update.
+REFRESH_DAYS = 30
+# A row is kept only if at least this fraction of tickers alive on that date
+# have a bar. Drops e.g. US-holiday rows created by HK-listed tickers.
+MIN_ROW_COVERAGE = 0.5
+# Relative change in the adjusted-price overlap that triggers a rescale.
+RESCALE_TOLERANCE = 1e-3
+UNAVAILABLE_RETRY_DAYS = 7
+
 
 def _data_path(market: str, price_type: str = "close") -> str:
     suffix = "" if price_type == "close" else f"_{price_type}"
     return os.path.join(DATA_DIR, f"{market}{suffix}.csv")
 
 
-def load(market: str, price_type: str = "close") -> pd.DataFrame | None:
+def load_raw(market: str, price_type: str = "close") -> pd.DataFrame | None:
-    """Load existing data file, or None if it doesn't exist."""
+    """Load the stored panel without filling missing bars, or None if absent."""
     path = _data_path(market, price_type)
     if not os.path.exists(path):
         return None
-    df = pd.read_csv(path, index_col=0, parse_dates=True)
+    return pd.read_csv(path, index_col=0, parse_dates=True)
-    return df
+
+
+def load(market: str, price_type: str = "close") -> pd.DataFrame | None:
+    """Load the stored panel with missing bars forward-filled."""
+    raw = load_raw(market, price_type)
+    return None if raw is None else raw.ffill()
+
+
+def _write_atomic(df: pd.DataFrame, path: str) -> None:
+    tmp = f"{path}.tmp"
+    df.to_csv(tmp)
+    os.replace(tmp, path)
+
+
+@contextmanager
+def _panel_lock(market: str):
+    """Serialize updates of one market panel across processes (monitor, bridge, scripts)."""
+    os.makedirs(DATA_DIR, exist_ok=True)
+    with open(os.path.join(DATA_DIR, f".{market}.lock"), "w") as fh:
+        fcntl.flock(fh, fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            fcntl.flock(fh, fcntl.LOCK_UN)
 
 
 def _download(tickers: list[str], start: str, end: str | None = None,
@@ -73,38 +114,6 @@ def _download(tickers: list[str], start: str, end: str | None = None,
     return result
 
 
-def _download_period(tickers: list[str], period: str,
-                     fields: list[str] | None = None) -> dict[str, pd.DataFrame]:
-    """Download using period-based API (for small gaps)."""
-    if fields is None:
-        fields = ["Close"]
-    raw = yf.download(tickers, period=period, auto_adjust=True)
-    if raw.empty:
-        return {f: pd.DataFrame() for f in fields}
-    result = {}
-    for field in fields:
-        if field in raw.columns.get_level_values(0) if isinstance(raw.columns, pd.MultiIndex) else field in raw.columns:
-            selected = raw[field]
-            if isinstance(selected, pd.Series):
-                result[field] = selected.to_frame(name=tickers[0])
-            else:
-                result[field] = selected
-        else:
-            result[field] = pd.DataFrame()
-    return result
-
-
-def _clean(data: pd.DataFrame) -> pd.DataFrame:
-    """Drop tickers with >50% missing data, forward-fill, drop all-NaN rows."""
-    good = data.columns[data.notna().mean() > 0.5]
-    dropped = set(data.columns) - set(good)
-    if dropped:
-        print(f"--- Dropped {len(dropped)} tickers with >50% missing data ---")
-    data = data[good]
-    data = data.ffill().dropna(how="all")
-    return data
-
-
 def _clean_market_data(data: pd.DataFrame, field: str) -> pd.DataFrame:
     """Clean market data while preserving volume gaps."""
     good = data.columns[data.notna().mean() > 0.5]
@@ -165,145 +174,210 @@ def update_market_data(market: str, tickers: list[str], fields: list[str]) -> di
     return cleaned
 
 
+
+def _unavailable_path(market: str) -> str:
+    return os.path.join(DATA_DIR, f"{market}_unavailable.json")
+
+
+def _load_unavailable(market: str) -> dict[str, str]:
+    try:
+        with open(_unavailable_path(market)) as f:
+            return json.load(f)
+    except (OSError, ValueError):
+        return {}
+
+
+def _save_unavailable(market: str, unavailable: dict[str, str]) -> None:
+    with open(_unavailable_path(market), "w") as f:
+        json.dump(unavailable, f, indent=0, sort_keys=True)
+
+
+def _drop_sparse_rows(df: pd.DataFrame) -> pd.DataFrame:
+    """Drop rows where too few of the tickers alive on that date have a bar."""
+    df = df.dropna(how="all")
+    if df.empty:
+        return df
+    notna = df.notna().to_numpy()
+    n = len(df)
+    has_any = notna.any(axis=0)
+    first = np.where(has_any, notna.argmax(axis=0), n)
+    last = np.where(has_any, n - 1 - notna[::-1].argmax(axis=0), -1)
+    pos = np.arange(n)[:, None]
+    alive = (pos >= first) & (pos <= last)
+    coverage = notna.sum(axis=1) / np.maximum(alive.sum(axis=1), 1)
+    keep = coverage >= MIN_ROW_COVERAGE
+    if not keep.all():
+        dropped = [str(d.date()) for d in df.index[~keep]]
+        print(f"--- Dropped {len(dropped)} sparse non-trading rows: {dropped[:5]} ---")
+    return df[keep]
+
+
+def _adjustment_ratio(old: pd.Series, new: pd.Series) -> float | None:
+    """Factor to rescale older history when Yahoo re-adjusted the overlap window.
+
+    Uses the median ratio over the earliest overlapping bars so a couple of
+    bad stored values cannot fake an adjustment.
+    """
+    both = pd.concat([old, new], axis=1, keys=["old", "new"]).dropna()
+    both = both[both["old"] > 0].head(5)
+    if len(both) < 2:
+        return None
+    ratio = float((both["new"] / both["old"]).median())
+    if not np.isfinite(ratio) or abs(ratio - 1.0) <= RESCALE_TOLERANCE:
+        return None
+    return ratio
+
+
+def _merge_refresh(existing: pd.DataFrame | None, new: pd.DataFrame,
+                   window_start: pd.Timestamp | None) -> pd.DataFrame:
+    """Overwrite requested tickers from ``window_start`` on with freshly downloaded bars."""
+    new = new.dropna(axis=1, how="all")
+    if existing is None or existing.empty:
+        return _drop_sparse_rows(new.sort_index())
+    out = existing.copy()
+    if window_start is None:
+        window_start = new.index.min() if not new.empty else out.index.max()
+    rescaled = []
+    for col in new.columns:
+        if col not in out.columns:
+            continue
+        ratio = _adjustment_ratio(out.loc[out.index >= window_start, col], new[col])
+        if ratio is not None:
+            out.loc[out.index < window_start, col] *= ratio
+            rescaled.append(f"{col}x{ratio:.4f}")
+    if rescaled:
+        print(f"--- Rescaled history for re-adjusted tickers: {rescaled[:8]} ---")
+    out = out.reindex(index=out.index.union(new.index),
+                      columns=list(out.columns) + [c for c in new.columns if c not in out.columns])
+    window = out.index >= window_start
+    out.loc[window, new.columns] = new.reindex(out.index[window]).to_numpy()
+    return _drop_sparse_rows(out.sort_index())
+
+
+def _fill_open(close: pd.DataFrame, open_raw: pd.DataFrame | None) -> pd.DataFrame:
+    """Align open prices to the close panel; fall back to close where open is missing."""
+    if open_raw is None or open_raw.empty:
+        return close.copy()
+    aligned = open_raw.reindex(index=close.index, columns=close.columns).ffill()
+    return aligned.fillna(close)
+
+
 def update(market: str, tickers: list[str],
            with_open: bool = False) -> pd.DataFrame | tuple[pd.DataFrame, pd.DataFrame]:
     """
-    Load existing data, download new dates and new tickers, save back.
+    Refresh the stored panel for ``tickers`` and return it forward-filled.
 
-    Parameters
+    - First run: downloads 10 years of history.
-    ----------
+    - Every run: re-downloads the last ``REFRESH_DAYS`` calendar days for the
-    with_open : bool
+      requested tickers already in the panel and overwrites them.
-        If True, also download/maintain Open prices and return (close, open) tuple.
+    - New tickers are backfilled over the panel's full date range; tickers
+      Yahoo has no data for are remembered and retried weekly.
 
-    Returns
+    Returns the close panel, or ``(close, open)`` when ``with_open`` is True.
-    -------
-    pd.DataFrame or (close_df, open_df)
-        Price data ready for backtesting.
     """
-    os.makedirs(DATA_DIR, exist_ok=True)
     fields = ["Close", "Open"] if with_open else ["Close"]
+    with _panel_lock(market):
+        close_raw = load_raw(market, "close")
+        open_raw = load_raw(market, "open") if with_open else None
+        unavailable = _load_unavailable(market)
+        today = datetime.now()
 
-    existing_close = load(market, "close")
+        if close_raw is None:
-    existing_open = load(market, "open") if with_open else None
+            start = (today - timedelta(days=365 * 10)).strftime("%Y-%m-%d")
+            downloaded = _download(tickers, start=start, fields=fields)
+            close_raw = _merge_refresh(None, downloaded["Close"], None)
+            if with_open:
+                open_raw = _merge_refresh(None, downloaded.get("Open", pd.DataFrame()), None)
+        else:
+            if with_open and open_raw is None:
+                print("--- Downloading Open prices for the full panel (first time) ---")
+                start = close_raw.index[0].strftime("%Y-%m-%d")
+                opens = _download(list(close_raw.columns), start=start, fields=["Open"])
+                open_raw = _merge_refresh(None, opens["Open"], None)
 
-    # If we need open prices but don't have them cached, force a full re-download
+            known = [t for t in tickers if t in close_raw.columns]
-    need_full_open = with_open and existing_open is None and existing_close is not None
+            if known:
-
+                window_start = close_raw.index[-1] - pd.Timedelta(days=REFRESH_DAYS)
-    if existing_close is None:
+                fresh = _download(known, start=window_start.strftime("%Y-%m-%d"), fields=fields)
-        # First run: download full 10-year history
+                if not fresh["Close"].empty:
-        start = (datetime.now() - timedelta(days=365 * 10)).strftime("%Y-%m-%d")
+                    close_raw = _merge_refresh(close_raw, fresh["Close"], window_start)
-        downloaded = _download(tickers, start=start, fields=fields)
+                    if with_open:
-        close_data = downloaded["Close"]
+                        open_raw = _merge_refresh(open_raw, fresh.get("Open", pd.DataFrame()), window_start)
-        open_data = downloaded.get("Open", pd.DataFrame()) if with_open else None
-    elif need_full_open:
-        # Have close data but need open data — download open prices for full range
-        close_data = existing_close
-        start = close_data.index[0].strftime("%Y-%m-%d")
-        known_tickers = [t for t in close_data.columns if t in tickers]
-        print(f"--- Downloading Open prices for {len(known_tickers)} tickers (first time) ---")
-        downloaded = _download(known_tickers, start=start, fields=["Open"])
-        open_data = downloaded.get("Open", pd.DataFrame())
-        last_date = close_data.index[-1]
-
-        # Still need to append new close dates and backfill new tickers
-        next_day = (last_date + timedelta(days=1)).strftime("%Y-%m-%d")
-        today = datetime.now().strftime("%Y-%m-%d")
-        if next_day < today:
-            if known_tickers:
-                gap_days = (datetime.now() - last_date.to_pydatetime().replace(tzinfo=None)).days
-                if gap_days <= 7:
-                    print(f"--- Fetching last {gap_days + 5}d to append ---")
-                    new = _download_period(known_tickers, f"{gap_days + 5}d", fields=fields)
-                    new_close = new["Close"]
-                    new_close = new_close[new_close.index > last_date]
                 else:
-                    new = _download(known_tickers, start=next_day, fields=fields)
+                    print("--- Refresh download returned no data; keeping stored panel ---")
-                    new_close = new["Close"]
-                if not new_close.empty:
-                    close_data = pd.concat([close_data, new_close]).sort_index()
-                    close_data = close_data[~close_data.index.duplicated(keep="last")]
-                    print(f"--- Appended {len(new_close)} new days ---")
-                else:
-                    print("--- No new trading days to append ---")
 
-        new_tickers = [t for t in tickers if t not in close_data.columns]
+            cutoff = (today - timedelta(days=UNAVAILABLE_RETRY_DAYS)).strftime("%Y-%m-%d")
-        if new_tickers:
+            new_tickers = [t for t in tickers
-            start = close_data.index[0].strftime("%Y-%m-%d")
+                           if t not in close_raw.columns and unavailable.get(t, "") < cutoff]
-            backfill = _download(new_tickers, start=start, fields=fields)
+            if new_tickers:
-            backfill_close = backfill["Close"]
+                start = close_raw.index[0].strftime("%Y-%m-%d")
-            backfill_open = backfill.get("Open", pd.DataFrame())
+                backfill = _download(new_tickers, start=start, fields=fields)
-            if not backfill_close.empty:
+                downloaded_any = not backfill["Close"].dropna(how="all").empty
-                backfill_close = backfill_close.reindex(close_data.index)
+                got = backfill["Close"].dropna(axis=1, how="all")
-                close_data = pd.concat([close_data, backfill_close], axis=1)
+                got = got.reindex(index=close_raw.index)
-                if not backfill_open.empty and open_data is not None:
+                got = got.dropna(axis=1, how="all")
-                    backfill_open = backfill_open.reindex(open_data.index)
+                for t in new_tickers:
-                    open_data = pd.concat([open_data, backfill_open], axis=1)
+                    if t in got.columns:
-                print(f"--- Backfilled {len(new_tickers)} new tickers ---")
+                        unavailable.pop(t, None)
-    else:
+                    elif downloaded_any:
-        close_data = existing_close
+                        # Only a ticker missing from an otherwise successful
-        open_data = existing_open if with_open else None
+                        # download is unavailable; a failed request is retried.
-        last_date = close_data.index[-1]
+                        unavailable[t] = today.strftime("%Y-%m-%d")
+                if not got.empty:
+                    close_raw = pd.concat([close_raw, got], axis=1)
+                    if with_open and open_raw is not None:
+                        opens = backfill.get("Open", pd.DataFrame())
+                        opens = opens.reindex(index=open_raw.index, columns=got.columns)
+                        open_raw = pd.concat([open_raw, opens], axis=1)
+                    print(f"--- Backfilled {got.shape[1]} new tickers ---")
+                missing = len(new_tickers) - got.shape[1]
+                if missing:
+                    print(f"--- {missing} tickers have no Yahoo data; retry after "
+                          f"{UNAVAILABLE_RETRY_DAYS} days ---")
 
-        # 1) Append new dates for existing tickers
+        close_raw = close_raw.loc[:, ~close_raw.columns.duplicated()]
-        next_day = (last_date + timedelta(days=1)).strftime("%Y-%m-%d")
+        _write_atomic(close_raw, _data_path(market, "close"))
-        today = datetime.now().strftime("%Y-%m-%d")
+        print(f"--- Saved {close_raw.shape[0]} days x {close_raw.shape[1]} tickers to "
-        if next_day < today:
+              f"{_data_path(market, 'close')} ---")
-            known_tickers = [t for t in close_data.columns if t in tickers]
+        if with_open and open_raw is not None:
-            if known_tickers:
+            open_raw = open_raw.loc[:, ~open_raw.columns.duplicated()]
-                gap_days = (datetime.now() - last_date.to_pydatetime().replace(tzinfo=None)).days
+            _write_atomic(open_raw, _data_path(market, "open"))
-                if gap_days <= 7:
+        _save_unavailable(market, unavailable)
-                    print(f"--- Fetching last {gap_days + 5}d to append ---")
-                    new = _download_period(known_tickers, f"{gap_days + 5}d", fields=fields)
-                    new_close = new["Close"]
-                    new_close = new_close[new_close.index > last_date]
-                    new_open = new.get("Open", pd.DataFrame())
-                    if not new_open.empty:
-                        new_open = new_open[new_open.index > last_date]
-                else:
-                    new = _download(known_tickers, start=next_day, fields=fields)
-                    new_close = new["Close"]
-                    new_open = new.get("Open", pd.DataFrame())
 
-                if not new_close.empty:
+    close = close_raw.ffill()
-                    close_data = pd.concat([close_data, new_close]).sort_index()
+    if with_open:
-                    close_data = close_data[~close_data.index.duplicated(keep="last")]
+        return close, _fill_open(close, open_raw)
-                    print(f"--- Appended {len(new_close)} new days ---")
+    return close
-                    if with_open and not new_open.empty and open_data is not None:
-                        open_data = pd.concat([open_data, new_open]).sort_index()
-                        open_data = open_data[~open_data.index.duplicated(keep="last")]
-                else:
-                    print("--- No new trading days to append ---")
 
-        # 2) Backfill any new tickers not in existing data
-        new_tickers = [t for t in tickers if t not in close_data.columns]
-        if new_tickers:
-            start = close_data.index[0].strftime("%Y-%m-%d")
-            backfill = _download(new_tickers, start=start, fields=fields)
-            backfill_close = backfill["Close"]
-            backfill_open = backfill.get("Open", pd.DataFrame())
-            if not backfill_close.empty:
-                backfill_close = backfill_close.reindex(close_data.index)
-                close_data = pd.concat([close_data, backfill_close], axis=1)
-                print(f"--- Backfilled {len(new_tickers)} new tickers ---")
-                if with_open and not backfill_open.empty and open_data is not None:
-                    backfill_open = backfill_open.reindex(open_data.index)
-                    open_data = pd.concat([open_data, backfill_open], axis=1)
 
-    # Clean and save close
+def rebuild(market: str, with_open: bool = True, years: int = 10) -> None:
-    close_data = _clean(close_data)
+    """Re-download the full history of every ticker in a stored panel.
-    close_path = _data_path(market, "close")
-    close_data.to_csv(close_path)
-    print(f"--- Saved {close_data.shape[0]} days x {close_data.shape[1]} tickers to {close_path} ---")
 
-    if with_open and open_data is not None and not open_data.empty:
+    Used to repair panels written by the old append-only updater. Tickers
-        # Align open data to same tickers/dates as close
+    Yahoo no longer serves keep their stored history.
-        open_data = open_data.reindex(index=close_data.index, columns=close_data.columns)
+    """
-        open_data = open_data.ffill().dropna(how="all")
+    with _panel_lock(market):
-        # Where open is missing, fall back to close
+        old_close = load_raw(market, "close")
-        open_data = open_data.fillna(close_data)
+        old_open = load_raw(market, "open")
-        open_path = _data_path(market, "open")
+        if old_close is None:
-        open_data.to_csv(open_path)
+            raise FileNotFoundError(_data_path(market, "close"))
-        print(f"--- Saved {open_data.shape[0]} days x {open_data.shape[1]} tickers to {open_path} ---")
+        tickers = list(old_close.columns)
-        return close_data, open_data
+        start = (datetime.now() - timedelta(days=365 * years)).strftime("%Y-%m-%d")
-
+        fields = ["Close", "Open"] if with_open else ["Close"]
-    return close_data
+        fresh = _download(tickers, start=start, fields=fields)
+        close = _merge_refresh(None, fresh["Close"], None)
+        kept = [t for t in tickers if t not in close.columns]
+        if kept:
+            close = close.combine_first(old_close[kept].reindex(close.index))
+            print(f"--- Kept stored history for {len(kept)} tickers Yahoo no longer serves ---")
+        close = close[[t for t in tickers if t in close.columns]]
+        _write_atomic(close, _data_path(market, "close"))
+        print(f"--- Rebuilt {market}: {close.shape[0]} days x {close.shape[1]} tickers ---")
+        if with_open:
+            opens = fresh.get("Open", pd.DataFrame()).dropna(axis=1, how="all")
+            opens = opens.reindex(index=close.index)
+            if old_open is not None:
+                missing = [t for t in close.columns if t not in opens.columns and t in old_open.columns]
+                if missing:
+                    opens = pd.concat([opens, old_open[missing].reindex(close.index)], axis=1)
+            _write_atomic(opens, _data_path(market, "open"))