diff --git a/Makefile b/Makefile index aa0373f..4e956e4 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,9 @@ check: $(RUN) check # On the server: probe the Gateway and restart it if it has lost IBKR's servers. +# The trade job passes GATEWAY_ARGS=--history to require historical bars too. gateway: - sh deploy/ensure-gateway.sh + sh deploy/ensure-gateway.sh $(GATEWAY_ARGS) bars: $(RUN) bars diff --git a/deploy/ensure-gateway.sh b/deploy/ensure-gateway.sh index 36a94b0..f39d97f 100755 --- a/deploy/ensure-gateway.sh +++ b/deploy/ensure-gateway.sh @@ -4,9 +4,12 @@ # with its API port still open (2026-09-29, for 9 hours): every request times # out, yet java keeps running, so systemd's Restart= never fires. # Runs before each job (ExecStartPre=make gateway) and by hand as `make gateway`. +# Arguments go to the probe: `--history` also requires daily bars, because the +# historical data farm can fail while account requests work (2026-10-12). cd "$(dirname "$0")/.." || exit 1 -probe() { uv run ibkr-quant gateway; } +PROBE_ARGS="$*" +probe() { uv run ibkr-quant gateway $PROBE_ARGS; } probe && exit 0 # Gateway rides out short outages by reconnecting on its own; give it a minute. diff --git a/deploy/systemd/ibkr-quant-trade.service b/deploy/systemd/ibkr-quant-trade.service index 4f09150..5219a20 100644 --- a/deploy/systemd/ibkr-quant-trade.service +++ b/deploy/systemd/ibkr-quant-trade.service @@ -1,17 +1,20 @@ [Unit] Description=ibkr-quant daily run: today's bars, then MOC orders on rebalance sessions After=ibgateway.service -# make gateway restarts a Gateway that has lost IBKR. Retries cover today's bar -# arriving late (delayed data) and Gateway restarts; make submit refuses to -# double-submit while orders are open. -StartLimitIntervalSec=7200 -StartLimitBurst=6 +# make gateway restarts a Gateway that has lost IBKR or its historical data +# farm. Retries cover today's bar arriving late (delayed data) and Gateway +# restarts; make submit refuses to double-submit while orders are open. +# Every 10 minutes from 09:50 until about 15:30 ET (34 starts), inside the +# 09:31-15:45 order window: on 2026-10-12 six retries 5 minutes apart gave up +# after 35 minutes while the data farm was down. +StartLimitIntervalSec=21600 +StartLimitBurst=34 [Service] Type=oneshot WorkingDirectory=%h/projects/ibkr-quant Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin -ExecStartPre=/usr/bin/make gateway +ExecStartPre=/usr/bin/make gateway GATEWAY_ARGS=--history ExecStart=/usr/bin/make bars submit Restart=on-failure -RestartSec=300 +RestartSec=600 diff --git a/src/ibkr_quant/cli.py b/src/ibkr_quant/cli.py index ee35002..1d07626 100644 --- a/src/ibkr_quant/cli.py +++ b/src/ibkr_quant/cli.py @@ -93,11 +93,22 @@ async def check(cfg: Config) -> None: print("Delayed data is already requested; check market data permissions and availability in IB Gateway/TWS.") -async def probe_gateway(cfg: Config) -> None: +async def probe_gateway(cfg: Config, history: bool = False) -> None: # Broker's connect fails unless the Gateway answers positions, orders and # account updates, which needs a live session with IBKR's servers. async with Broker(dataclasses.replace(cfg.ib, client_id=cfg.ib.health_client_id)) as broker: - print(f"IB Gateway OK: account {broker.account}, server version {broker.ib.client.serverVersion()}") + status = f"account {broker.account}, server version {broker.ib.client.serverVersion()}" + if history: + # The historical data farm can fail on its own: on 2026-10-12 account requests + # worked while every bar request timed out for 35 minutes, until the Gateway's + # daily restart. Without this check `make gateway` never restarted it. + contracts, _ = await broker.qualify_stocks(["SPY"]) + bars = (await broker.daily_bars(contracts, "5 D", timeout=30))[0] if contracts else None + if bars is None or bars.empty: + raise SystemExit(f"IB Gateway answers account requests ({status}) but returned no SPY bars " + "within 30 s; its historical data farm is not serving") + status += f", historical data OK (SPY through {bars['date'].max()})" + print(f"IB Gateway OK: {status}") async def update_bars(cfg: Config) -> None: @@ -279,7 +290,8 @@ def main() -> None: commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("universe", help="download S&P 500 constituents") commands.add_parser("check", help="connect and show account, positions, open orders, SPY price") - commands.add_parser("gateway", help="exit non-zero unless IB Gateway answers account requests") + gateway_parser = commands.add_parser("gateway", help="exit non-zero unless IB Gateway answers account requests") + gateway_parser.add_argument("--history", action="store_true", help="also require SPY daily bars from the historical data farm") commands.add_parser("bars", help="download adjusted daily bars for the universe") rebalance_parser = commands.add_parser("rebalance", help="plan orders (dry run unless --submit)") rebalance_parser.add_argument("--submit", action="store_true") @@ -299,7 +311,7 @@ def main() -> None: case "check": asyncio.run(check(cfg)) case "gateway": - asyncio.run(probe_gateway(cfg)) + asyncio.run(probe_gateway(cfg, args.history)) case "bars": asyncio.run(update_bars(cfg)) case "rebalance": diff --git a/tests/test_check.py b/tests/test_check.py index 8c92d16..2dcfc38 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -62,3 +62,24 @@ def test_check_explains_snapshot_result(monkeypatch, capsys, mode, error, price, assert absent not in output if error: assert f"IBKR {error[0]}: {error[1]}" in output + + +@pytest.mark.parametrize("bars, ok", [(0, False), (5, True)]) +def test_gateway_probe_requires_historical_bars(monkeypatch, capsys, bars, ok): + import pandas as pd + cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml") + broker = Broker(cfg.ib) + broker.account = "DU123" + frame = pd.DataFrame({"date": pd.date_range("2026-10-06", periods=bars), "symbol": "SPY", "close": 1.0}) + monkeypatch.setattr(Broker, "__aenter__", AsyncMock(return_value=broker)) + monkeypatch.setattr(Broker, "__aexit__", AsyncMock(return_value=None)) + monkeypatch.setattr(cli, "Broker", lambda _: broker) + monkeypatch.setattr(broker.ib.client, "serverVersion", Mock(return_value=178)) + monkeypatch.setattr(broker, "qualify_stocks", AsyncMock(return_value=({"SPY": Stock("SPY", "SMART", "USD")}, []))) + monkeypatch.setattr(broker, "daily_bars", AsyncMock(return_value=(frame, [] if bars else ["SPY"]))) + if ok: + asyncio.run(cli.probe_gateway(cfg, history=True)) + assert "historical data OK" in capsys.readouterr().out + else: + with pytest.raises(SystemExit, match="historical data farm"): + asyncio.run(cli.probe_gateway(cfg, history=True))