gahow/quant-trading
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

live: keep new sessions on subset refreshes; replay books from a backfilled day

Browse Source 
A refresh of a few tickers judged a new date's coverage against every
ticker in the stored panel, so the date was dropped as sparse; it also
re-ran the sparse filter over the whole history. Coverage is now checked
only inside the refresh window, with unrequested tickers counted where they
already have a bar.

When a session that was skipped earlier appears in the panel, the trader
rewinds the ledger to that day (undoing later trades, or restoring a legacy
snapshot) and replays every session from it in order. State files are
written atomically, and the monitor refreshes volume with close/open.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-10-12 05:16:16 -07:00
parent 2e69bc8c7a
commit 31155fdcfb
3 changed files with 132 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
data_manager.py
View File

@@ -195,7 +195,7 @@ def _save_unavailable(market: str, unavailable: dict[str, str]) -> None:
json.dump(unavailable, f, indent=0, sort_keys=True)
def _drop_sparse_rows(df: pd.DataFrame) -> pd.DataFrame:
def _drop_sparse_rows(df: pd.DataFrame, refreshed_columns=None) -> pd.DataFrame:
"""Drop rows where too few of the tickers alive on that date have a bar."""
df = df.dropna(how="all")
if df.empty:
@@ -207,6 +207,9 @@ def _drop_sparse_rows(df: pd.DataFrame) -> pd.DataFrame:
last = np.where(has_any, n - 1 - notna[::-1].argmax(axis=0), -1)
pos = np.arange(n)[:, None]
alive = (pos >= first) & (pos <= last + ALIVE_GRACE_ROWS)
if refreshed_columns is not None:
unrequested = ~df.columns.isin(refreshed_columns)
alive[:, unrequested] = notna[:, unrequested]
coverage = notna.sum(axis=1) / np.maximum(alive.sum(axis=1), 1)
keep = coverage >= MIN_ROW_COVERAGE
if not keep.all():
@@ -234,6 +237,7 @@ def _adjustment_ratio(old: pd.Series, new: pd.Series) -> float | None:
def _merge_refresh(existing: pd.DataFrame | None, new: pd.DataFrame,
window_start: pd.Timestamp | None) -> pd.DataFrame:
"""Overwrite requested tickers from ``window_start`` on with freshly downloaded bars."""
refreshed_columns = new.columns
new = new.dropna(axis=1, how="all")
if existing is None or existing.empty:
return _drop_sparse_rows(new.sort_index())
@@ -254,7 +258,10 @@ def _merge_refresh(existing: pd.DataFrame | None, new: pd.DataFrame,
columns=list(out.columns) + [c for c in new.columns if c not in out.columns])
window = out.index >= window_start
out.loc[window, new.columns] = new.reindex(out.index[window]).to_numpy()
return _drop_sparse_rows(out.sort_index())
# Unrequested columns count only where an actual bar already exists.
# They cannot make a newly fetched date look sparse; older history stays intact.
valid = _drop_sparse_rows(out.loc[window], refreshed_columns=refreshed_columns).index
return out.loc[~window | out.index.isin(valid)].sort_index()
def _fill_open(close: pd.DataFrame, open_raw: pd.DataFrame | None) -> pd.DataFrame: