Restart IB Gateway before each job when it has lost IBKR's servers

On 2026-09-29 Gateway lost its connection, failed the reconnect's SSL
handshake and sat on an error dialog for 9 hours with its API port open:
clients connected, every startup request timed out, and systemd never
restarted it because java kept running.

- Broker connects with raiseSyncErrors, so a failed startup sync exits
  instead of continuing with an empty portfolio(), which rebalance would
  read as holding nothing.
- `ibkr-quant gateway` probes the Gateway under its own health_client_id.
- deploy/ensure-gateway.sh (`make gateway`) probes, waits a minute for a
  self-reconnect, then restarts ibgateway.service and waits for it.
- The trade and fills units run `make gateway` as ExecStartPre; fills
  now retries on failure like trade does.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/ibkr_quant/broker.py

@@ -39,13 +39,27 @@ class Broker:
         self.account = ""
 
     async def __aenter__(self) -> "Broker":
+        address = f"{self.cfg.host}:{self.cfg.port}"
         try:
+            # A Gateway that has lost IBKR's servers still accepts API clients but
+            # never answers the startup sync; without raiseSyncErrors the connect
+            # succeeds with an empty portfolio(), which rebalance reads as no holdings.
             await self.ib.connectAsync(
-                self.cfg.host, self.cfg.port, self.cfg.client_id, timeout=15, account=self.cfg.account
+                self.cfg.host,
+                self.cfg.port,
+                self.cfg.client_id,
+                timeout=15,
+                account=self.cfg.account,
+                raiseSyncErrors=True,
             )
         except ConnectionRefusedError:
+            raise SystemExit(f"nothing listening on {address}; start IB Gateway/TWS and enable the API port") from None
+        except TimeoutError:
+            raise SystemExit(f"IB Gateway on {address} did not finish the API handshake; it may still be logging in") from None
+        except ConnectionError as e:
             raise SystemExit(
-                f"nothing listening on {self.cfg.host}:{self.cfg.port}; start IB Gateway/TWS and enable the API port"
+                f"IB Gateway on {address} accepted the connection but failed {e}; it has likely lost IBKR's "
+                "servers (`make gateway` checks it and restarts it)"
             ) from None
         accounts = self.ib.managedAccounts()
         if self.cfg.account:

src/ibkr_quant/cli.py

@@ -44,6 +44,13 @@ async def check(cfg: Config) -> None:
             print("no SPY price: check the market data subscription or set ib.market_data_type = 3")
 
 
+async def probe_gateway(cfg: Config) -> None:
+    # Broker's connect fails unless the Gateway answers positions, orders and
+    # account updates, which needs a live session with IBKR's servers.
+    async with Broker(dataclasses.replace(cfg.ib, client_id=cfg.ib.health_client_id)) as broker:
+        print(f"IB Gateway OK: account {broker.account}, server version {broker.ib.client.serverVersion()}")
+
+
 async def update_bars(cfg: Config) -> None:
     symbols = load_universe(cfg.data.universe)
     async with Broker(cfg.ib) as broker:
@@ -201,6 +208,7 @@ def main() -> None:
     commands = parser.add_subparsers(dest="command", required=True)
     commands.add_parser("universe", help="download S&P 500 constituents")
     commands.add_parser("check", help="connect and show account, positions, open orders, SPY price")
+    commands.add_parser("gateway", help="exit non-zero unless IB Gateway answers account requests")
     commands.add_parser("bars", help="download adjusted daily bars for the universe")
     rebalance_parser = commands.add_parser("rebalance", help="plan orders (dry run unless --submit)")
     rebalance_parser.add_argument("--submit", action="store_true")
@@ -219,6 +227,8 @@ def main() -> None:
             update_universe(cfg)
         case "check":
             asyncio.run(check(cfg))
+        case "gateway":
+            asyncio.run(probe_gateway(cfg))
         case "bars":
             asyncio.run(update_bars(cfg))
         case "rebalance":

src/ibkr_quant/config.py

@@ -8,6 +8,7 @@ class IBConfig:
     host: str
     port: int
     client_id: int
+    health_client_id: int
     account: str
     market_data_type: int
     allow_live: bool