diff --git a/research/lab/fetch_options.py b/research/lab/fetch_options.py index baeb16e..a072eaf 100644 --- a/research/lab/fetch_options.py +++ b/research/lab/fetch_options.py @@ -106,6 +106,10 @@ OCC = re.compile(r"^[A-Z]{1,5}\d{6,7}[CP]\d{8}$") # what the bars endpoint acc BAR_COLS = ["date", "underlying", "symbol", "type", "expiry", "strike", "o", "h", "l", "c", "v", "n", "vw"] +class BackendTimeout(RuntimeError): + """HTTP 504 on a request for many symbols: the caller splits the batch.""" + + class Api: """Paced GET with retries. ``yield_to_snapshot`` sleeps through 16:10-16:55 New York on weekdays so a backfill leaves the rate limit to the daily run.""" @@ -140,6 +144,8 @@ class Api: time.sleep(2 ** attempt) continue self.n += 1 + if r.status_code == 504 and params.get("symbols", "").count(",") >= 50: + raise BackendTimeout(r.text[:200]) # the same batch times out again (SNDK, 1000 contracts) if r.status_code == 429 or r.status_code >= 500: time.sleep(2 ** attempt) continue @@ -238,14 +244,23 @@ def option_bars(api: Api, symbols: list[str], start: str, end: str, timeframe: s e = (dt.datetime.combine(dt.date.fromisoformat(end), dt.time(0), NY) if len(end) == 10 else dt.datetime.fromisoformat(end)) end = min(e, dt.datetime.now(NY) - dt.timedelta(minutes=16)).isoformat() + + def fetch(chunk: list[str]) -> list[dict]: + params = {"symbols": ",".join(chunk), "timeframe": timeframe, "start": start, "end": end, "limit": 10000} + got = [] + try: + for bars in api.pages(f"{DATA}/v1beta1/options/bars", params, "bars"): + for sym, bl in bars.items(): + got += [{"symbol": sym, "t": b["t"], "o": b["o"], "h": b["h"], "l": b["l"], "c": b["c"], + "v": b["v"], "n": b.get("n"), "vw": b.get("vw")} for b in bl] + except BackendTimeout: # restart the chunk as two halves + h = len(chunk) // 2 + return fetch(chunk[:h]) + fetch(chunk[h:]) + return got + rows = [] for i in range(0, len(symbols), 1000): - params = {"symbols": ",".join(symbols[i:i + 1000]), "timeframe": timeframe, "start": start, "end": end, - "limit": 10000} - for bars in api.pages(f"{DATA}/v1beta1/options/bars", params, "bars"): - for sym, bl in bars.items(): - rows += [{"symbol": sym, "t": b["t"], "o": b["o"], "h": b["h"], "l": b["l"], "c": b["c"], - "v": b["v"], "n": b.get("n"), "vw": b.get("vw")} for b in bl] + rows += fetch(symbols[i:i + 1000]) return rows