""" Persistent local data store for price history. Layout: data/{market}.csv — adjusted close prices data/{market}_open.csv — adjusted open prices data/{market}_unavailable.json — tickers Yahoo returned no data for (retried weekly) Panels are stored raw: a date/ticker Yahoo has no bar for stays NaN, so a missing print is distinguishable from an unchanged price. ``load()`` and ``update()`` forward-fill on read; ``load_raw()`` exposes the stored values. Every ``update()`` re-downloads a trailing ``REFRESH_DAYS`` window and overwrites it for the requested tickers. Partial intraday bars, late prints and rows that turn out not to be trading days are therefore corrected on the next run instead of being frozen. When Yahoo's split/dividend adjustment shifts the overlap, that ticker's older history is rescaled so returns stay continuous. """ import fcntl import json import os from contextlib import contextmanager from datetime import datetime, timedelta import numpy as np import pandas as pd import yfinance as yf DATA_DIR = "data" # Trailing calendar days re-downloaded and overwritten on every update. REFRESH_DAYS = 30 # A row is kept only if at least this fraction of tickers alive on that date # have a bar. Drops e.g. US-holiday rows created by HK-listed tickers. MIN_ROW_COVERAGE = 0.5 # A ticker counts as alive for this many rows after its last bar, so a newest # row printed by only a few tickers (late data) is still judged sparse. ALIVE_GRACE_ROWS = 10 # Relative change in the adjusted-price overlap that triggers a rescale. RESCALE_TOLERANCE = 1e-3 UNAVAILABLE_RETRY_DAYS = 7 def _data_path(market: str, price_type: str = "close") -> str: suffix = "" if price_type == "close" else f"_{price_type}" return os.path.join(DATA_DIR, f"{market}{suffix}.csv") def load_raw(market: str, price_type: str = "close") -> pd.DataFrame | None: """Load the stored panel without filling missing bars, or None if absent.""" path = _data_path(market, price_type) if not os.path.exists(path): return None return pd.read_csv(path, index_col=0, parse_dates=True) def load(market: str, price_type: str = "close") -> pd.DataFrame | None: """Load the stored panel with missing bars forward-filled.""" raw = load_raw(market, price_type) return None if raw is None else raw.ffill() def _write_atomic(df: pd.DataFrame, path: str) -> None: tmp = f"{path}.tmp" df.to_csv(tmp) os.replace(tmp, path) @contextmanager def _panel_lock(market: str): """Serialize updates of one market panel across processes (monitor, bridge, scripts).""" os.makedirs(DATA_DIR, exist_ok=True) with open(os.path.join(DATA_DIR, f".{market}.lock"), "w") as fh: fcntl.flock(fh, fcntl.LOCK_EX) try: yield finally: fcntl.flock(fh, fcntl.LOCK_UN) def _download(tickers: list[str], start: str, end: str | None = None, fields: list[str] | None = None) -> dict[str, pd.DataFrame]: """ Download price data from Yahoo Finance. Parameters ---------- fields : list of str, optional Price fields to extract, e.g. ["Close", "Open"]. Defaults to ["Close"]. Returns ------- dict mapping field name to DataFrame (columns = tickers). """ if fields is None: fields = ["Close"] print(f"--- Downloading {len(tickers)} tickers from {start}{f' to {end}' if end else ''} ---") kwargs = dict(auto_adjust=True) if end: kwargs["end"] = end raw = yf.download(tickers, start=start, **kwargs) if raw.empty: return {f: pd.DataFrame() for f in fields} result = {} for field in fields: if field in raw.columns.get_level_values(0) if isinstance(raw.columns, pd.MultiIndex) else field in raw.columns: selected = raw[field] if isinstance(selected, pd.Series): result[field] = selected.to_frame(name=tickers[0]) else: result[field] = selected else: result[field] = pd.DataFrame() return result def _clean_market_data(data: pd.DataFrame, field: str) -> pd.DataFrame: """Clean market data while preserving volume gaps.""" good = data.columns[data.notna().mean() > 0.5] dropped = set(data.columns) - set(good) if dropped: print(f"--- Dropped {len(dropped)} tickers with >50% missing data ---") data = data[good] if field == "volume": return data return data.ffill().dropna(how="all") def _merge_market_panel(existing: pd.DataFrame | None, new_data: pd.DataFrame) -> pd.DataFrame: """Merge new data into an existing cached panel, preserving old columns and dates.""" if existing is None or existing.empty: merged = new_data.copy() elif new_data.empty: merged = existing.copy() else: merged = existing.combine_first(new_data) merged.loc[new_data.index, new_data.columns] = new_data merged = merged.sort_index() merged = merged[~merged.index.duplicated(keep="last")] return merged def update_market_data(market: str, tickers: list[str], fields: list[str]) -> dict[str, pd.DataFrame]: """Download, clean, persist, and return market data panels for requested Yahoo fields.""" field_aliases = { "close": "Close", "open": "Open", "high": "High", "low": "Low", "volume": "Volume", } normalized_fields = [] yahoo_fields = [] for field in fields: normalized = field.lower() if normalized not in field_aliases: raise ValueError(f"Unsupported market data field: {field}") normalized_fields.append(normalized) yahoo_fields.append(field_aliases[normalized]) os.makedirs(DATA_DIR, exist_ok=True) start = (datetime.now() - timedelta(days=365 * 10)).strftime("%Y-%m-%d") downloaded = _download(tickers, start=start, fields=yahoo_fields) cleaned = {} for normalized, yahoo_field in zip(normalized_fields, yahoo_fields): data = _clean_market_data(downloaded.get(yahoo_field, pd.DataFrame()), normalized) existing = load(market, normalized) data = _merge_market_panel(existing, data) path = _data_path(market, normalized) data.to_csv(path) print(f"--- Saved {data.shape[0]} days x {data.shape[1]} tickers to {path} ---") cleaned[normalized] = data return cleaned def _unavailable_path(market: str) -> str: return os.path.join(DATA_DIR, f"{market}_unavailable.json") def _load_unavailable(market: str) -> dict[str, str]: try: with open(_unavailable_path(market)) as f: return json.load(f) except (OSError, ValueError): return {} def _save_unavailable(market: str, unavailable: dict[str, str]) -> None: with open(_unavailable_path(market), "w") as f: json.dump(unavailable, f, indent=0, sort_keys=True) def _drop_sparse_rows(df: pd.DataFrame) -> pd.DataFrame: """Drop rows where too few of the tickers alive on that date have a bar.""" df = df.dropna(how="all") if df.empty: return df notna = df.notna().to_numpy() n = len(df) has_any = notna.any(axis=0) first = np.where(has_any, notna.argmax(axis=0), n) last = np.where(has_any, n - 1 - notna[::-1].argmax(axis=0), -1) pos = np.arange(n)[:, None] alive = (pos >= first) & (pos <= last + ALIVE_GRACE_ROWS) coverage = notna.sum(axis=1) / np.maximum(alive.sum(axis=1), 1) keep = coverage >= MIN_ROW_COVERAGE if not keep.all(): dropped = [str(d.date()) for d in df.index[~keep]] print(f"--- Dropped {len(dropped)} sparse non-trading rows: {dropped[:5]} ---") return df[keep] def _adjustment_ratio(old: pd.Series, new: pd.Series) -> float | None: """Factor to rescale older history when Yahoo re-adjusted the overlap window. Uses the median ratio over the earliest overlapping bars so a couple of bad stored values cannot fake an adjustment. """ both = pd.concat([old, new], axis=1, keys=["old", "new"]).dropna() both = both[both["old"] > 0].head(5) if len(both) < 2: return None ratio = float((both["new"] / both["old"]).median()) if not np.isfinite(ratio) or abs(ratio - 1.0) <= RESCALE_TOLERANCE: return None return ratio def _merge_refresh(existing: pd.DataFrame | None, new: pd.DataFrame, window_start: pd.Timestamp | None) -> pd.DataFrame: """Overwrite requested tickers from ``window_start`` on with freshly downloaded bars.""" new = new.dropna(axis=1, how="all") if existing is None or existing.empty: return _drop_sparse_rows(new.sort_index()) out = existing.copy() if window_start is None: window_start = new.index.min() if not new.empty else out.index.max() rescaled = [] for col in new.columns: if col not in out.columns: continue ratio = _adjustment_ratio(out.loc[out.index >= window_start, col], new[col]) if ratio is not None: out.loc[out.index < window_start, col] *= ratio rescaled.append(f"{col}x{ratio:.4f}") if rescaled: print(f"--- Rescaled history for re-adjusted tickers: {rescaled[:8]} ---") out = out.reindex(index=out.index.union(new.index), columns=list(out.columns) + [c for c in new.columns if c not in out.columns]) window = out.index >= window_start out.loc[window, new.columns] = new.reindex(out.index[window]).to_numpy() return _drop_sparse_rows(out.sort_index()) def _fill_open(close: pd.DataFrame, open_raw: pd.DataFrame | None) -> pd.DataFrame: """Align quotes without manufacturing a print from a stale or later price.""" if open_raw is None: return pd.DataFrame(np.nan, index=close.index, columns=close.columns) return open_raw.reindex(index=close.index, columns=close.columns) YAHOO_FIELDS = {"close": "Close", "open": "Open", "high": "High", "low": "Low", "volume": "Volume"} def _backfill(tickers: list[str], start: str, yahoo_fields: list[str], index: pd.DatetimeIndex) -> tuple[dict[str, pd.DataFrame], list[str]]: """Download new tickers over ``index``; return (panels, tickers still missing). yfinance reports a ticker it failed to fetch (rate limit, proxy error) the same way as a ticker it has no data for, so tickers missing from the first batch are downloaded once more on their own before being given up on. """ def fetch(batch: list[str]) -> dict[str, pd.DataFrame]: raw = _download(batch, start=start, fields=yahoo_fields) close = raw["Close"].dropna(axis=1, how="all").reindex(index=index).dropna(axis=1, how="all") return {f: raw.get(f, pd.DataFrame()).reindex(index=index, columns=close.columns) for f in yahoo_fields} got = fetch(tickers) missing = [t for t in tickers if t not in got["Close"].columns] if missing and len(missing) < len(tickers): retry = fetch(missing) if not retry["Close"].empty: got = {f: pd.concat([got[f], retry[f]], axis=1) for f in yahoo_fields} missing = [t for t in missing if t not in retry["Close"].columns] return got, missing def update(market: str, tickers: list[str], with_open: bool = False, fields: list[str] | None = None): """ Refresh the stored panels for ``tickers`` and return them forward-filled. - First run: downloads 10 years of history. - Every run: re-downloads the last ``REFRESH_DAYS`` calendar days for the requested tickers already in the panel and overwrites them. - New tickers are backfilled over the panel's full date range; tickers Yahoo has no data for are remembered and retried weekly. - A field requested for the first time (e.g. ``high``) is downloaded for every ticker in the stored close panel. Returns the close panel, ``(close, open)`` when ``with_open`` is True, or ``{field: panel}`` when ``fields`` is given. Open/high/low fall back to the close where a bar is missing; volume is returned raw (NaN = no print). """ names = list(fields) if fields is not None else (["close", "open"] if with_open else ["close"]) if "close" not in names: names = ["close"] + names unknown = [f for f in names if f not in YAHOO_FIELDS] if unknown: raise ValueError(f"Unsupported market data field(s): {unknown}") yahoo = [YAHOO_FIELDS[f] for f in names] with _panel_lock(market): raw = {f: load_raw(market, f) for f in names} unavailable = _load_unavailable(market) today = datetime.now() if raw["close"] is None: start = (today - timedelta(days=365 * 10)).strftime("%Y-%m-%d") downloaded = _download(tickers, start=start, fields=yahoo) for f, y in zip(names, yahoo): raw[f] = _merge_refresh(None, downloaded.get(y, pd.DataFrame()), None) else: close_raw = raw["close"] first = [(f, y) for f, y in zip(names, yahoo) if raw[f] is None] if first: print(f"--- Downloading {[y for _, y in first]} for the full panel (first time) ---") got = _download(list(close_raw.columns), start=close_raw.index[0].strftime("%Y-%m-%d"), fields=[y for _, y in first]) for f, y in first: raw[f] = _merge_refresh(None, got[y], None) known = [t for t in tickers if t in close_raw.columns] if known: window_start = close_raw.index[-1] - pd.Timedelta(days=REFRESH_DAYS) fresh = _download(known, start=window_start.strftime("%Y-%m-%d"), fields=yahoo) if not fresh["Close"].empty: for f, y in zip(names, yahoo): raw[f] = _merge_refresh(raw[f], fresh.get(y, pd.DataFrame()), window_start) else: print("--- Refresh download returned no data; keeping stored panel ---") cutoff = (today - timedelta(days=UNAVAILABLE_RETRY_DAYS)).strftime("%Y-%m-%d") new_tickers = [t for t in tickers if t not in raw["close"].columns and unavailable.get(t, "") < cutoff] if new_tickers: got, missing = _backfill(new_tickers, raw["close"].index[0].strftime("%Y-%m-%d"), yahoo, raw["close"].index) added = list(got["Close"].columns) for t in added: unavailable.pop(t, None) # A batch that returned nothing at all is a failed request, not # evidence the tickers have no data: retry those next run. if added: for t in missing: unavailable[t] = today.strftime("%Y-%m-%d") if added: for f, y in zip(names, yahoo): panel = got[y].reindex(index=raw[f].index) raw[f] = pd.concat([raw[f], panel], axis=1) print(f"--- Backfilled {len(added)} new tickers ---") if missing: print(f"--- {len(missing)} tickers have no Yahoo data; retry after " f"{UNAVAILABLE_RETRY_DAYS} days ---") for f in names: raw[f] = raw[f].loc[:, ~raw[f].columns.duplicated()] _write_atomic(raw[f], _data_path(market, f)) print(f"--- Saved {raw['close'].shape[0]} days x {raw['close'].shape[1]} tickers to " f"{_data_path(market, 'close')} ---") _save_unavailable(market, unavailable) close = raw["close"].ffill() out = {"close": close} for f in names: if f in ("open", "high", "low"): out[f] = _fill_open(close, raw[f]) elif f == "volume": out[f] = raw[f].reindex(index=close.index, columns=close.columns) if fields is not None: return out if with_open: return close, out["open"] return close def load_fields(market: str, fields: list[str]) -> dict[str, pd.DataFrame]: """Stored panels without refreshing, aligned and filled the way ``update`` returns them.""" close = load(market, "close") out = {"close": close} for f in fields: if f == "close": continue stored = load_raw(market, f) if f == "volume": out[f] = (stored.reindex(index=close.index, columns=close.columns) if stored is not None else pd.DataFrame(np.nan, index=close.index, columns=close.columns)) else: out[f] = _fill_open(close, stored) return out def rebuild(market: str, with_open: bool = True, years: int = 10) -> None: """Re-download the full history of every ticker in a stored panel. Used to repair panels written by the old append-only updater. Tickers Yahoo no longer serves keep their stored history. """ with _panel_lock(market): old_close = load_raw(market, "close") old_open = load_raw(market, "open") if old_close is None: raise FileNotFoundError(_data_path(market, "close")) tickers = list(old_close.columns) start = (datetime.now() - timedelta(days=365 * years)).strftime("%Y-%m-%d") fields = ["Close", "Open"] if with_open else ["Close"] fresh = _download(tickers, start=start, fields=fields) close = _merge_refresh(None, fresh["Close"], None) kept = [t for t in tickers if t not in close.columns] if kept: close = close.combine_first(old_close[kept].reindex(close.index)) print(f"--- Kept stored history for {len(kept)} tickers Yahoo no longer serves ---") close = close[[t for t in tickers if t in close.columns]] _write_atomic(close, _data_path(market, "close")) print(f"--- Rebuilt {market}: {close.shape[0]} days x {close.shape[1]} tickers ---") if with_open: opens = fresh.get("Open", pd.DataFrame()).dropna(axis=1, how="all") opens = opens.reindex(index=close.index) if old_open is not None: missing = [t for t in close.columns if t not in opens.columns and t in old_open.columns] if missing: opens = pd.concat([opens, old_open[missing].reindex(close.index)], axis=1) _write_atomic(opens, _data_path(market, "open"))