Restart IB Gateway before each job when it has lost IBKR's servers

On 2026-09-29 Gateway lost its connection, failed the reconnect's SSL
handshake and sat on an error dialog for 9 hours with its API port open:
clients connected, every startup request timed out, and systemd never
restarted it because java kept running.

- Broker connects with raiseSyncErrors, so a failed startup sync exits
  instead of continuing with an empty portfolio(), which rebalance would
  read as holding nothing.
- `ibkr-quant gateway` probes the Gateway under its own health_client_id.
- deploy/ensure-gateway.sh (`make gateway`) probes, waits a minute for a
  self-reconnect, then restarts ibgateway.service and waits for it.
- The trade and fills units run `make gateway` as ExecStartPre; fills
  now retries on failure like trade does.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Makefile

@@ -6,7 +6,7 @@
 # open, so start after 09:47 ET.
 RUN = uv run ibkr-quant
 
-.PHONY: test universe check bars plan submit fills backtest install-units
+.PHONY: test universe check gateway bars plan submit fills backtest install-units
 
 test:
 	uv run pytest -q
@@ -17,6 +17,10 @@ universe:
 check:
 	$(RUN) check
 
+# On the server: probe the Gateway and restart it if it has lost IBKR's servers.
+gateway:
+	sh deploy/ensure-gateway.sh
+
 bars:
 	$(RUN) bars
 

config.toml

@@ -2,6 +2,7 @@
 host = "127.0.0.1"
 port = 4002             # IB Gateway: paper 4002 / live 4001; TWS: paper 7497 / live 7496
 client_id = 17
+health_client_id = 18   # `ibkr-quant gateway` probe; differs so it never collides with a running job
 account = ""            # empty = the single account managed by this login
 market_data_type = 3    # 1 = live (needs subscription), 3 = delayed (free, for testing)
 allow_live = false      # refuse to connect to non-paper accounts unless true

deploy/ensure-gateway.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+# Probe IB Gateway and restart it when it stays unable to serve account requests.
+# Gateway can lose IBKR's servers, fail the reconnect and sit on an error dialog
+# with its API port still open (2026-09-29, for 9 hours): every request times
+# out, yet java keeps running, so systemd's Restart= never fires.
+# Runs before each job (ExecStartPre=make gateway) and by hand as `make gateway`.
+cd "$(dirname "$0")/.." || exit 1
+
+probe() { uv run ibkr-quant gateway; }
+
+probe && exit 0
+# Gateway rides out short outages by reconnecting on its own; give it a minute.
+sleep 60
+probe && exit 0
+
+echo "restarting ibgateway.service" >&2
+systemctl --user restart ibgateway.service || exit 1
+# IBC logged in 28 s after a restart on 2026-09-29; leave room for a slow proxy.
+deadline=$(($(date +%s) + 300))
+while [ "$(date +%s)" -lt "$deadline" ]; do
+    sleep 20
+    probe && exit 0
+done
+echo "IB Gateway still not serving requests 5 min after the restart" >&2
+exit 1

deploy/systemd/ibkr-quant-fills.service

@@ -1,9 +1,15 @@
 [Unit]
 Description=ibkr-quant: record today's executions
 After=ibgateway.service
+# Executions are only reported for today, so retry a failed Gateway for an hour.
+StartLimitIntervalSec=3600
+StartLimitBurst=4
 
 [Service]
 Type=oneshot
 WorkingDirectory=%h/projects/ibkr-quant
 Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
+ExecStartPre=/usr/bin/make gateway
 ExecStart=/usr/bin/make fills
+Restart=on-failure
+RestartSec=300

deploy/systemd/ibkr-quant-trade.service

@@ -1,8 +1,9 @@
 [Unit]
 Description=ibkr-quant daily rebalance: today's bars, then MOC orders
 After=ibgateway.service
-# Retries cover today's bar arriving late (delayed data) and Gateway restarts;
-# make submit refuses to double-submit while orders are open.
+# make gateway restarts a Gateway that has lost IBKR. Retries cover today's bar
+# arriving late (delayed data) and Gateway restarts; make submit refuses to
+# double-submit while orders are open.
 StartLimitIntervalSec=7200
 StartLimitBurst=6
 
@@ -10,6 +11,7 @@ StartLimitBurst=6
 Type=oneshot
 WorkingDirectory=%h/projects/ibkr-quant
 Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
+ExecStartPre=/usr/bin/make gateway
 ExecStart=/usr/bin/make bars submit
 Restart=on-failure
 RestartSec=300

src/ibkr_quant/broker.py

@@ -39,13 +39,27 @@ class Broker:
         self.account = ""
 
     async def __aenter__(self) -> "Broker":
+        address = f"{self.cfg.host}:{self.cfg.port}"
         try:
+            # A Gateway that has lost IBKR's servers still accepts API clients but
+            # never answers the startup sync; without raiseSyncErrors the connect
+            # succeeds with an empty portfolio(), which rebalance reads as no holdings.
             await self.ib.connectAsync(
-                self.cfg.host, self.cfg.port, self.cfg.client_id, timeout=15, account=self.cfg.account
+                self.cfg.host,
+                self.cfg.port,
+                self.cfg.client_id,
+                timeout=15,
+                account=self.cfg.account,
+                raiseSyncErrors=True,
             )
         except ConnectionRefusedError:
+            raise SystemExit(f"nothing listening on {address}; start IB Gateway/TWS and enable the API port") from None
+        except TimeoutError:
+            raise SystemExit(f"IB Gateway on {address} did not finish the API handshake; it may still be logging in") from None
+        except ConnectionError as e:
             raise SystemExit(
-                f"nothing listening on {self.cfg.host}:{self.cfg.port}; start IB Gateway/TWS and enable the API port"
+                f"IB Gateway on {address} accepted the connection but failed {e}; it has likely lost IBKR's "
+                "servers (`make gateway` checks it and restarts it)"
             ) from None
         accounts = self.ib.managedAccounts()
         if self.cfg.account:

src/ibkr_quant/cli.py

@@ -44,6 +44,13 @@ async def check(cfg: Config) -> None:
             print("no SPY price: check the market data subscription or set ib.market_data_type = 3")
 
 
+async def probe_gateway(cfg: Config) -> None:
+    # Broker's connect fails unless the Gateway answers positions, orders and
+    # account updates, which needs a live session with IBKR's servers.
+    async with Broker(dataclasses.replace(cfg.ib, client_id=cfg.ib.health_client_id)) as broker:
+        print(f"IB Gateway OK: account {broker.account}, server version {broker.ib.client.serverVersion()}")
+
+
 async def update_bars(cfg: Config) -> None:
     symbols = load_universe(cfg.data.universe)
     async with Broker(cfg.ib) as broker:
@@ -201,6 +208,7 @@ def main() -> None:
     commands = parser.add_subparsers(dest="command", required=True)
     commands.add_parser("universe", help="download S&P 500 constituents")
     commands.add_parser("check", help="connect and show account, positions, open orders, SPY price")
+    commands.add_parser("gateway", help="exit non-zero unless IB Gateway answers account requests")
     commands.add_parser("bars", help="download adjusted daily bars for the universe")
     rebalance_parser = commands.add_parser("rebalance", help="plan orders (dry run unless --submit)")
     rebalance_parser.add_argument("--submit", action="store_true")
@@ -219,6 +227,8 @@ def main() -> None:
             update_universe(cfg)
         case "check":
             asyncio.run(check(cfg))
+        case "gateway":
+            asyncio.run(probe_gateway(cfg))
         case "bars":
             asyncio.run(update_bars(cfg))
         case "rebalance":

src/ibkr_quant/config.py

@@ -8,6 +8,7 @@ class IBConfig:
     host: str
     port: int
     client_id: int
+    health_client_id: int
     account: str
     market_data_type: int
     allow_live: bool