From 35d750f0a5c3baddde44b39c24be99c22030a643 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 13 Oct 2026 14:30:41 -0700 Subject: [PATCH] Accept paper MOC orders IBKR holds as PendingSubmit; stop retrying after the order window On 2026-10-13 the 09:53 ET run placed six MOC orders and failed: the paper account reports MOC orders as PendingSubmit until the closing auction and sent no acknowledgement within 30 s, so confirm_orders treated them as rejected. IBKR listed all six with permanent ids and filled them at the close, as it did the 14 PendingSubmit orders of 2026-09-29. Every retry then stopped at "6 open orders already exist", and after 15:45 ET at the order window, and the start limit's 6-hour window reset at 15:50 ET, so the job restarted every 10 minutes into the evening (42 restarts by 17:28 ET). - confirm_orders accepts a PendingSubmit order once IBKR lists it with a permanent id, re-reading this client's open orders every 5 s for up to 60 s. - rebalance --submit succeeds without sending anything when every open order comes from today's runs and IBKR still holds every order those runs sent; a missing one still fails. - The trade unit skips starts after 15:45 ET (ExecCondition), which systemd does not restart. Co-Authored-By: Claude Opus 5.5 --- deploy/systemd/ibkr-quant-trade.service | 12 ++-- src/ibkr_quant/broker.py | 25 ++++++-- src/ibkr_quant/cli.py | 11 ++++ tests/test_order_confirmation.py | 82 +++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 9 deletions(-) diff --git a/deploy/systemd/ibkr-quant-trade.service b/deploy/systemd/ibkr-quant-trade.service index 5219a20..2e68c8f 100644 --- a/deploy/systemd/ibkr-quant-trade.service +++ b/deploy/systemd/ibkr-quant-trade.service @@ -3,10 +3,11 @@ Description=ibkr-quant daily run: today's bars, then MOC orders on rebalance ses After=ibgateway.service # make gateway restarts a Gateway that has lost IBKR or its historical data # farm. Retries cover today's bar arriving late (delayed data) and Gateway -# restarts; make submit refuses to double-submit while orders are open. -# Every 10 minutes from 09:50 until about 15:30 ET (34 starts), inside the -# 09:31-15:45 order window: on 2026-10-12 six retries 5 minutes apart gave up -# after 35 minutes while the data farm was down. +# restarts; make submit refuses to double-submit while orders are open and +# succeeds once IBKR holds every order an earlier run sent. +# Every 10 minutes from 09:50 ET until the 15:45 end of the order window: on +# 2026-10-12 six retries 5 minutes apart gave up after 35 minutes while the +# data farm was down. The start limit is a backstop; ExecCondition ends the day. StartLimitIntervalSec=21600 StartLimitBurst=34 @@ -14,6 +15,9 @@ StartLimitBurst=34 Type=oneshot WorkingDirectory=%h/projects/ibkr-quant Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +# Past 15:45 ET make submit cannot send orders; a failed condition skips the +# start without counting as a failure, so it is not restarted. +ExecCondition=/bin/sh -c 'test "$(TZ=America/New_York date +%%H%%M)" -lt 1545' ExecStartPre=/usr/bin/make gateway GATEWAY_ARGS=--history ExecStart=/usr/bin/make bars submit Restart=on-failure diff --git a/src/ibkr_quant/broker.py b/src/ibkr_quant/broker.py index 6c3b173..a0d4ddd 100644 --- a/src/ibkr_quant/broker.py +++ b/src/ibkr_quant/broker.py @@ -170,17 +170,32 @@ class Broker: order.orderRef = ref return self.ib.placeOrder(contract, order) - async def confirm_orders(self, trades: list[Trade], timeout: float = 30) -> None: - accepted = {OrderStatus.PreSubmitted, OrderStatus.Submitted, OrderStatus.Filled} - terminal = accepted | OrderStatus.DoneStates + async def confirm_orders(self, trades: list[Trade], timeout: float = 60, refresh_every: float = 5) -> None: + # The paper account reports MOC orders as PendingSubmit until the closing auction: on + # 2026-09-29 and 10-13 every order stayed PendingSubmit and the 09-29 ones all filled at + # the close. Such an order counts as accepted once IBKR lists it with a permanent id, + # which the 10-13 run never saw unprompted, so this client's open orders are re-read. + acknowledged = {OrderStatus.PreSubmitted, OrderStatus.Submitted, OrderStatus.Filled} + + def accepted(t: Trade) -> bool: + status = t.orderStatus.status + return status in acknowledged or (status == OrderStatus.PendingSubmit and t.order.permId != 0) + loop = asyncio.get_running_loop() deadline = loop.time() + timeout - while any(t.orderStatus.status not in terminal for t in trades): + next_refresh = loop.time() + refresh_every + while not all(accepted(t) or t.orderStatus.status in OrderStatus.DoneStates for t in trades): remaining = deadline - loop.time() if remaining <= 0: break + if loop.time() >= next_refresh: + try: + await asyncio.wait_for(self.ib.reqOpenOrdersAsync(), remaining) + except TimeoutError: + break + next_refresh = loop.time() + refresh_every await asyncio.sleep(min(0.1, remaining)) - failed = [t for t in trades if t.orderStatus.status not in accepted] + failed = [t for t in trades if not accepted(t)] if failed: details = [] for trade in failed: diff --git a/src/ibkr_quant/cli.py b/src/ibkr_quant/cli.py index 1d07626..f4563d7 100644 --- a/src/ibkr_quant/cli.py +++ b/src/ibkr_quant/cli.py @@ -190,6 +190,17 @@ async def rebalance(cfg: Config, submit: bool) -> None: raise SystemExit(f"{now:%H:%M} ET is outside the order window {ORDER_WINDOW[0]}-{ORDER_WINDOW[1]}") if open_trades := await broker.open_trades(): + # A run whose confirmation timed out exits non-zero and the trade service retries it; + # if IBKR holds every order that run sent, the session's orders are in. + refs = {t.order.orderRef for t in open_trades} + sent = [cfg.root / "runs" / ref.removeprefix("ibkr-quant ") / "orders.csv" for ref in refs] + if all(ref.startswith(f"ibkr-quant {now:%Y%m%d}-") for ref in refs) and all(p.exists() for p in sent): + sent_ids = set(pd.concat(pd.read_csv(p) for p in sent)["order_id"]) + if missing := sent_ids - {t.order.orderId for t in open_trades}: + raise SystemExit(f"today's orders {sorted(missing)} are no longer open at IBKR " + "(rejected, cancelled or filled); see `make check`") + print(f"{len(open_trades)} orders from {', '.join(sorted(refs))} are open at IBKR; not submitting again") + return raise SystemExit(f"{len(open_trades)} open orders already exist; cancel or let them finish first") contracts, unknown = await broker.qualify_stocks([o.symbol for o in orders]) if unknown: diff --git a/tests/test_order_confirmation.py b/tests/test_order_confirmation.py index 181974c..af721dd 100644 --- a/tests/test_order_confirmation.py +++ b/tests/test_order_confirmation.py @@ -34,6 +34,34 @@ def test_confirmation_accepts_broker_acknowledged_orders(status): asyncio.run(Broker(cfg.ib).confirm_orders([trade(status)], timeout=0)) +def test_confirmation_accepts_pending_moc_orders_that_ibkr_lists(): + # The paper account keeps MOC orders PendingSubmit until the close (2026-09-29, 10-13). + async def run(): + order = trade("PendingSubmit") + cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml") + broker = Broker(cfg.ib) + + async def open_orders(): + order.order.permId = 114178004 + return [order] + broker.ib.reqOpenOrdersAsync = AsyncMock(side_effect=open_orders) + await broker.confirm_orders([order], timeout=1, refresh_every=0.01) + assert order.orderStatus.status == "PendingSubmit" + broker.ib.reqOpenOrdersAsync.assert_awaited() + asyncio.run(run()) + + +def test_confirmation_rejects_pending_orders_ibkr_never_lists(): + async def run(): + cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml") + broker = Broker(cfg.ib) + broker.ib.reqOpenOrdersAsync = AsyncMock(return_value=[]) + with pytest.raises(RuntimeError, match="AAA order 41: PendingSubmit"): + await broker.confirm_orders([trade("PendingSubmit")], timeout=0.5, refresh_every=0.01) + broker.ib.reqOpenOrdersAsync.assert_awaited() + asyncio.run(run()) + + @pytest.mark.parametrize("status", ["Inactive", "Cancelled", "ApiCancelled", "PendingSubmit", "ApiPending"]) def test_confirmation_rejects_terminal_failure_and_missing_acknowledgement(status): cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml") @@ -82,3 +110,57 @@ def test_rebalance_records_partial_failure_and_exits_nonzero(monkeypatch, tmp_pa orders = pd.read_csv(next((tmp_path / "runs").glob("*/orders.csv"))) assert dict(zip(orders.symbol, orders.status)) == {"AAA": "Submitted", "BBB": "Inactive"} assert "accepted 2 orders" not in capsys.readouterr().out + + +def submit_with_open_orders(monkeypatch, tmp_path, open_trades, earlier_run="20261012-095323", earlier_ids=(64, 65)): + """Run `rebalance --submit` at 10:00 ET on 2026-10-12 while `open_trades` are open at IBKR.""" + cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml") + cfg = dataclasses.replace(cfg, root=tmp_path, strategy=dataclasses.replace(cfg.strategy, top_n=2)) + now = dt.datetime(2026, 10, 12, 10, 0, tzinfo=cli.NEW_YORK) + run_dir = tmp_path / "runs" / earlier_run + run_dir.mkdir(parents=True) + pd.DataFrame({"symbol": ["AAA", "BBB"][:len(earlier_ids)], "order_id": list(earlier_ids), + "status": "PendingSubmit"}).to_csv(run_dir / "orders.csv", index=False) + + class Clock(dt.datetime): + @classmethod + def now(cls, tz=None): + return now + + monkeypatch.setattr(cli.dt, "datetime", Clock) + idx = pd.bdate_range(end=now.date(), periods=300) + panel = pd.DataFrame({"AAA": 100.0, "BBB": 100.0}, index=idx) + monkeypatch.setattr(cli, "load_universe", lambda _: ["AAA", "BBB"]) + monkeypatch.setattr(cli, "load_bars", lambda *_: (panel, panel)) + monkeypatch.setattr(cli, "sessions_until_rebalance", lambda *_: 0) + monkeypatch.setattr(cli, "signal_overnight_recovery", lambda *_: panel.iloc[:-1]) + broker = Broker(cfg.ib) + monkeypatch.setattr(Broker, "__aenter__", AsyncMock(return_value=broker)) + monkeypatch.setattr(Broker, "__aexit__", AsyncMock(return_value=None)) + monkeypatch.setattr(cli, "Broker", lambda _: broker) + monkeypatch.setattr(broker, "account_state", AsyncMock(return_value=AccountState("DU123", 10000, 10000, []))) + monkeypatch.setattr(broker, "open_trades", AsyncMock(return_value=open_trades)) + monkeypatch.setattr(broker, "place", lambda *_: pytest.fail("placed an order while orders were open")) + asyncio.run(cli.rebalance(cfg, submit=True)) + + +def open_order(order_id, ref): + return Trade(Stock("AAA", "SMART", "USD"), Order(orderId=order_id, permId=114178000 + order_id, orderRef=ref), + OrderStatus(status="PendingSubmit")) + + +def test_retry_succeeds_when_ibkr_holds_every_order_an_earlier_run_sent(monkeypatch, tmp_path, capsys): + ref = "ibkr-quant 20261012-095323" + submit_with_open_orders(monkeypatch, tmp_path, [open_order(64, ref), open_order(65, ref)]) + assert f"2 orders from {ref} are open at IBKR; not submitting again" in capsys.readouterr().out + + +def test_retry_fails_when_an_order_from_the_earlier_run_is_gone(monkeypatch, tmp_path): + with pytest.raises(SystemExit, match=r"today's orders \[65\] are no longer open"): + submit_with_open_orders(monkeypatch, tmp_path, [open_order(64, "ibkr-quant 20261012-095323")]) + + +@pytest.mark.parametrize("ref", ["ibkr-quant 20261009-095323", "", "manual"]) +def test_retry_refuses_open_orders_it_did_not_send_today(monkeypatch, tmp_path, ref): + with pytest.raises(SystemExit, match="1 open orders already exist"): + submit_with_open_orders(monkeypatch, tmp_path, [open_order(64, ref)])