diff --git a/Makefile b/Makefile index d52384e..8ed0bda 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # open, so start after 09:47 ET. RUN = uv run ibkr-quant -.PHONY: test universe check bars plan submit fills backtest install-units +.PHONY: test universe check gateway bars plan submit fills backtest install-units test: uv run pytest -q @@ -17,6 +17,10 @@ universe: check: $(RUN) check +# On the server: probe the Gateway and restart it if it has lost IBKR's servers. +gateway: + sh deploy/ensure-gateway.sh + bars: $(RUN) bars diff --git a/config.toml b/config.toml index 49ec4d1..bcc1b2b 100644 --- a/config.toml +++ b/config.toml @@ -2,6 +2,7 @@ host = "127.0.0.1" port = 4002 # IB Gateway: paper 4002 / live 4001; TWS: paper 7497 / live 7496 client_id = 17 +health_client_id = 18 # `ibkr-quant gateway` probe; differs so it never collides with a running job account = "" # empty = the single account managed by this login market_data_type = 3 # 1 = live (needs subscription), 3 = delayed (free, for testing) allow_live = false # refuse to connect to non-paper accounts unless true diff --git a/deploy/ensure-gateway.sh b/deploy/ensure-gateway.sh new file mode 100755 index 0000000..36a94b0 --- /dev/null +++ b/deploy/ensure-gateway.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Probe IB Gateway and restart it when it stays unable to serve account requests. +# Gateway can lose IBKR's servers, fail the reconnect and sit on an error dialog +# with its API port still open (2026-09-29, for 9 hours): every request times +# out, yet java keeps running, so systemd's Restart= never fires. +# Runs before each job (ExecStartPre=make gateway) and by hand as `make gateway`. +cd "$(dirname "$0")/.." || exit 1 + +probe() { uv run ibkr-quant gateway; } + +probe && exit 0 +# Gateway rides out short outages by reconnecting on its own; give it a minute. +sleep 60 +probe && exit 0 + +echo "restarting ibgateway.service" >&2 +systemctl --user restart ibgateway.service || exit 1 +# IBC logged in 28 s after a restart on 2026-09-29; leave room for a slow proxy. +deadline=$(($(date +%s) + 300)) +while [ "$(date +%s)" -lt "$deadline" ]; do + sleep 20 + probe && exit 0 +done +echo "IB Gateway still not serving requests 5 min after the restart" >&2 +exit 1 diff --git a/deploy/systemd/ibkr-quant-fills.service b/deploy/systemd/ibkr-quant-fills.service index 87db9ad..ab1f987 100644 --- a/deploy/systemd/ibkr-quant-fills.service +++ b/deploy/systemd/ibkr-quant-fills.service @@ -1,9 +1,15 @@ [Unit] Description=ibkr-quant: record today's executions After=ibgateway.service +# Executions are only reported for today, so retry a failed Gateway for an hour. +StartLimitIntervalSec=3600 +StartLimitBurst=4 [Service] Type=oneshot WorkingDirectory=%h/projects/ibkr-quant Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +ExecStartPre=/usr/bin/make gateway ExecStart=/usr/bin/make fills +Restart=on-failure +RestartSec=300 diff --git a/deploy/systemd/ibkr-quant-trade.service b/deploy/systemd/ibkr-quant-trade.service index bbf98f6..5403191 100644 --- a/deploy/systemd/ibkr-quant-trade.service +++ b/deploy/systemd/ibkr-quant-trade.service @@ -1,8 +1,9 @@ [Unit] Description=ibkr-quant daily rebalance: today's bars, then MOC orders After=ibgateway.service -# Retries cover today's bar arriving late (delayed data) and Gateway restarts; -# make submit refuses to double-submit while orders are open. +# make gateway restarts a Gateway that has lost IBKR. Retries cover today's bar +# arriving late (delayed data) and Gateway restarts; make submit refuses to +# double-submit while orders are open. StartLimitIntervalSec=7200 StartLimitBurst=6 @@ -10,6 +11,7 @@ StartLimitBurst=6 Type=oneshot WorkingDirectory=%h/projects/ibkr-quant Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +ExecStartPre=/usr/bin/make gateway ExecStart=/usr/bin/make bars submit Restart=on-failure RestartSec=300 diff --git a/src/ibkr_quant/broker.py b/src/ibkr_quant/broker.py index 28c78e1..eb3f93a 100644 --- a/src/ibkr_quant/broker.py +++ b/src/ibkr_quant/broker.py @@ -39,13 +39,27 @@ class Broker: self.account = "" async def __aenter__(self) -> "Broker": + address = f"{self.cfg.host}:{self.cfg.port}" try: + # A Gateway that has lost IBKR's servers still accepts API clients but + # never answers the startup sync; without raiseSyncErrors the connect + # succeeds with an empty portfolio(), which rebalance reads as no holdings. await self.ib.connectAsync( - self.cfg.host, self.cfg.port, self.cfg.client_id, timeout=15, account=self.cfg.account + self.cfg.host, + self.cfg.port, + self.cfg.client_id, + timeout=15, + account=self.cfg.account, + raiseSyncErrors=True, ) except ConnectionRefusedError: + raise SystemExit(f"nothing listening on {address}; start IB Gateway/TWS and enable the API port") from None + except TimeoutError: + raise SystemExit(f"IB Gateway on {address} did not finish the API handshake; it may still be logging in") from None + except ConnectionError as e: raise SystemExit( - f"nothing listening on {self.cfg.host}:{self.cfg.port}; start IB Gateway/TWS and enable the API port" + f"IB Gateway on {address} accepted the connection but failed {e}; it has likely lost IBKR's " + "servers (`make gateway` checks it and restarts it)" ) from None accounts = self.ib.managedAccounts() if self.cfg.account: diff --git a/src/ibkr_quant/cli.py b/src/ibkr_quant/cli.py index 2cf757b..1f2d86a 100644 --- a/src/ibkr_quant/cli.py +++ b/src/ibkr_quant/cli.py @@ -44,6 +44,13 @@ async def check(cfg: Config) -> None: print("no SPY price: check the market data subscription or set ib.market_data_type = 3") +async def probe_gateway(cfg: Config) -> None: + # Broker's connect fails unless the Gateway answers positions, orders and + # account updates, which needs a live session with IBKR's servers. + async with Broker(dataclasses.replace(cfg.ib, client_id=cfg.ib.health_client_id)) as broker: + print(f"IB Gateway OK: account {broker.account}, server version {broker.ib.client.serverVersion()}") + + async def update_bars(cfg: Config) -> None: symbols = load_universe(cfg.data.universe) async with Broker(cfg.ib) as broker: @@ -201,6 +208,7 @@ def main() -> None: commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("universe", help="download S&P 500 constituents") commands.add_parser("check", help="connect and show account, positions, open orders, SPY price") + commands.add_parser("gateway", help="exit non-zero unless IB Gateway answers account requests") commands.add_parser("bars", help="download adjusted daily bars for the universe") rebalance_parser = commands.add_parser("rebalance", help="plan orders (dry run unless --submit)") rebalance_parser.add_argument("--submit", action="store_true") @@ -219,6 +227,8 @@ def main() -> None: update_universe(cfg) case "check": asyncio.run(check(cfg)) + case "gateway": + asyncio.run(probe_gateway(cfg)) case "bars": asyncio.run(update_bars(cfg)) case "rebalance": diff --git a/src/ibkr_quant/config.py b/src/ibkr_quant/config.py index 00b4fd3..18e0e5f 100644 --- a/src/ibkr_quant/config.py +++ b/src/ibkr_quant/config.py @@ -8,6 +8,7 @@ class IBConfig: host: str port: int client_id: int + health_client_id: int account: str market_data_type: int allow_live: bool