Restart the Gateway when its historical data farm stops serving

On 2026-10-12 the Gateway answered account requests but every bar request
timed out from 09:50 to 10:24 ET: make gateway reported OK, so the Gateway was
never restarted, and six retries 5 minutes apart gave up after 35 minutes. The
data came back only with the Gateway's daily restart.

ibkr-quant gateway --history also requires SPY daily bars within 30 s; the
trade job's ExecStartPre passes it, so ensure-gateway.sh restarts a Gateway
whose data farm is down. The trade job now retries every 10 minutes until
about 15:30 ET (34 starts in 6 h) instead of giving up after 35 minutes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Makefile

@@ -20,8 +20,9 @@ check:
 	$(RUN) check
 
 # On the server: probe the Gateway and restart it if it has lost IBKR's servers.
+# The trade job passes GATEWAY_ARGS=--history to require historical bars too.
 gateway:
-	sh deploy/ensure-gateway.sh
+	sh deploy/ensure-gateway.sh $(GATEWAY_ARGS)
 
 bars:
 	$(RUN) bars

deploy/ensure-gateway.sh

@@ -4,9 +4,12 @@
 # with its API port still open (2026-09-29, for 9 hours): every request times
 # out, yet java keeps running, so systemd's Restart= never fires.
 # Runs before each job (ExecStartPre=make gateway) and by hand as `make gateway`.
+# Arguments go to the probe: `--history` also requires daily bars, because the
+# historical data farm can fail while account requests work (2026-10-12).
 cd "$(dirname "$0")/.." || exit 1
 
-probe() { uv run ibkr-quant gateway; }
+PROBE_ARGS="$*"
+probe() { uv run ibkr-quant gateway $PROBE_ARGS; }
 
 probe && exit 0
 # Gateway rides out short outages by reconnecting on its own; give it a minute.

deploy/systemd/ibkr-quant-trade.service

@@ -1,17 +1,20 @@
 [Unit]
 Description=ibkr-quant daily run: today's bars, then MOC orders on rebalance sessions
 After=ibgateway.service
-# make gateway restarts a Gateway that has lost IBKR. Retries cover today's bar
-# arriving late (delayed data) and Gateway restarts; make submit refuses to
-# double-submit while orders are open.
-StartLimitIntervalSec=7200
-StartLimitBurst=6
+# make gateway restarts a Gateway that has lost IBKR or its historical data
+# farm. Retries cover today's bar arriving late (delayed data) and Gateway
+# restarts; make submit refuses to double-submit while orders are open.
+# Every 10 minutes from 09:50 until about 15:30 ET (34 starts), inside the
+# 09:31-15:45 order window: on 2026-10-12 six retries 5 minutes apart gave up
+# after 35 minutes while the data farm was down.
+StartLimitIntervalSec=21600
+StartLimitBurst=34
 
 [Service]
 Type=oneshot
 WorkingDirectory=%h/projects/ibkr-quant
 Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
-ExecStartPre=/usr/bin/make gateway
+ExecStartPre=/usr/bin/make gateway GATEWAY_ARGS=--history
 ExecStart=/usr/bin/make bars submit
 Restart=on-failure
-RestartSec=300
+RestartSec=600

src/ibkr_quant/cli.py

@@ -93,11 +93,22 @@ async def check(cfg: Config) -> None:
                 print("Delayed data is already requested; check market data permissions and availability in IB Gateway/TWS.")
 
 
-async def probe_gateway(cfg: Config) -> None:
+async def probe_gateway(cfg: Config, history: bool = False) -> None:
     # Broker's connect fails unless the Gateway answers positions, orders and
     # account updates, which needs a live session with IBKR's servers.
     async with Broker(dataclasses.replace(cfg.ib, client_id=cfg.ib.health_client_id)) as broker:
-        print(f"IB Gateway OK: account {broker.account}, server version {broker.ib.client.serverVersion()}")
+        status = f"account {broker.account}, server version {broker.ib.client.serverVersion()}"
+        if history:
+            # The historical data farm can fail on its own: on 2026-10-12 account requests
+            # worked while every bar request timed out for 35 minutes, until the Gateway's
+            # daily restart. Without this check `make gateway` never restarted it.
+            contracts, _ = await broker.qualify_stocks(["SPY"])
+            bars = (await broker.daily_bars(contracts, "5 D", timeout=30))[0] if contracts else None
+            if bars is None or bars.empty:
+                raise SystemExit(f"IB Gateway answers account requests ({status}) but returned no SPY bars "
+                                 "within 30 s; its historical data farm is not serving")
+            status += f", historical data OK (SPY through {bars['date'].max()})"
+        print(f"IB Gateway OK: {status}")
 
 
 async def update_bars(cfg: Config) -> None:
@@ -279,7 +290,8 @@ def main() -> None:
     commands = parser.add_subparsers(dest="command", required=True)
     commands.add_parser("universe", help="download S&P 500 constituents")
     commands.add_parser("check", help="connect and show account, positions, open orders, SPY price")
-    commands.add_parser("gateway", help="exit non-zero unless IB Gateway answers account requests")
+    gateway_parser = commands.add_parser("gateway", help="exit non-zero unless IB Gateway answers account requests")
+    gateway_parser.add_argument("--history", action="store_true", help="also require SPY daily bars from the historical data farm")
     commands.add_parser("bars", help="download adjusted daily bars for the universe")
     rebalance_parser = commands.add_parser("rebalance", help="plan orders (dry run unless --submit)")
     rebalance_parser.add_argument("--submit", action="store_true")
@@ -299,7 +311,7 @@ def main() -> None:
         case "check":
             asyncio.run(check(cfg))
         case "gateway":
-            asyncio.run(probe_gateway(cfg))
+            asyncio.run(probe_gateway(cfg, args.history))
         case "bars":
             asyncio.run(update_bars(cfg))
         case "rebalance":

tests/test_check.py

@@ -62,3 +62,24 @@ def test_check_explains_snapshot_result(monkeypatch, capsys, mode, error, price,
     assert absent not in output
     if error:
         assert f"IBKR {error[0]}: {error[1]}" in output
+
+
+@pytest.mark.parametrize("bars, ok", [(0, False), (5, True)])
+def test_gateway_probe_requires_historical_bars(monkeypatch, capsys, bars, ok):
+    import pandas as pd
+    cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml")
+    broker = Broker(cfg.ib)
+    broker.account = "DU123"
+    frame = pd.DataFrame({"date": pd.date_range("2026-10-06", periods=bars), "symbol": "SPY", "close": 1.0})
+    monkeypatch.setattr(Broker, "__aenter__", AsyncMock(return_value=broker))
+    monkeypatch.setattr(Broker, "__aexit__", AsyncMock(return_value=None))
+    monkeypatch.setattr(cli, "Broker", lambda _: broker)
+    monkeypatch.setattr(broker.ib.client, "serverVersion", Mock(return_value=178))
+    monkeypatch.setattr(broker, "qualify_stocks", AsyncMock(return_value=({"SPY": Stock("SPY", "SMART", "USD")}, [])))
+    monkeypatch.setattr(broker, "daily_bars", AsyncMock(return_value=(frame, [] if bars else ["SPY"])))
+    if ok:
+        asyncio.run(cli.probe_gateway(cfg, history=True))
+        assert "historical data OK" in capsys.readouterr().out
+    else:
+        with pytest.raises(SystemExit, match="historical data farm"):
+            asyncio.run(cli.probe_gateway(cfg, history=True))