Accept paper MOC orders IBKR holds as PendingSubmit; stop retrying after the order window

On 2026-10-13 the 09:53 ET run placed six MOC orders and failed: the paper
account reports MOC orders as PendingSubmit until the closing auction and sent
no acknowledgement within 30 s, so confirm_orders treated them as rejected.
IBKR listed all six with permanent ids and filled them at the close, as it did
the 14 PendingSubmit orders of 2026-09-29. Every retry then stopped at "6 open
orders already exist", and after 15:45 ET at the order window, and the start
limit's 6-hour window reset at 15:50 ET, so the job restarted every 10 minutes
into the evening (42 restarts by 17:28 ET).

- confirm_orders accepts a PendingSubmit order once IBKR lists it with a
  permanent id, re-reading this client's open orders every 5 s for up to 60 s.
- rebalance --submit succeeds without sending anything when every open order
  comes from today's runs and IBKR still holds every order those runs sent;
  a missing one still fails.
- The trade unit skips starts after 15:45 ET (ExecCondition), which systemd
  does not restart.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

deploy/systemd/ibkr-quant-trade.service

@@ -3,10 +3,11 @@ Description=ibkr-quant daily run: today's bars, then MOC orders on rebalance ses
 After=ibgateway.service
 # make gateway restarts a Gateway that has lost IBKR or its historical data
 # farm. Retries cover today's bar arriving late (delayed data) and Gateway
-# restarts; make submit refuses to double-submit while orders are open.
-# Every 10 minutes from 09:50 until about 15:30 ET (34 starts), inside the
-# 09:31-15:45 order window: on 2026-10-12 six retries 5 minutes apart gave up
-# after 35 minutes while the data farm was down.
+# restarts; make submit refuses to double-submit while orders are open and
+# succeeds once IBKR holds every order an earlier run sent.
+# Every 10 minutes from 09:50 ET until the 15:45 end of the order window: on
+# 2026-10-12 six retries 5 minutes apart gave up after 35 minutes while the
+# data farm was down. The start limit is a backstop; ExecCondition ends the day.
 StartLimitIntervalSec=21600
 StartLimitBurst=34
 
@@ -14,6 +15,9 @@ StartLimitBurst=34
 Type=oneshot
 WorkingDirectory=%h/projects/ibkr-quant
 Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
+# Past 15:45 ET make submit cannot send orders; a failed condition skips the
+# start without counting as a failure, so it is not restarted.
+ExecCondition=/bin/sh -c 'test "$(TZ=America/New_York date +%%H%%M)" -lt 1545'
 ExecStartPre=/usr/bin/make gateway GATEWAY_ARGS=--history
 ExecStart=/usr/bin/make bars submit
 Restart=on-failure

src/ibkr_quant/broker.py

@@ -170,17 +170,32 @@ class Broker:
         order.orderRef = ref
         return self.ib.placeOrder(contract, order)
 
-    async def confirm_orders(self, trades: list[Trade], timeout: float = 30) -> None:
-        accepted = {OrderStatus.PreSubmitted, OrderStatus.Submitted, OrderStatus.Filled}
-        terminal = accepted | OrderStatus.DoneStates
+    async def confirm_orders(self, trades: list[Trade], timeout: float = 60, refresh_every: float = 5) -> None:
+        # The paper account reports MOC orders as PendingSubmit until the closing auction: on
+        # 2026-09-29 and 10-13 every order stayed PendingSubmit and the 09-29 ones all filled at
+        # the close. Such an order counts as accepted once IBKR lists it with a permanent id,
+        # which the 10-13 run never saw unprompted, so this client's open orders are re-read.
+        acknowledged = {OrderStatus.PreSubmitted, OrderStatus.Submitted, OrderStatus.Filled}
+
+        def accepted(t: Trade) -> bool:
+            status = t.orderStatus.status
+            return status in acknowledged or (status == OrderStatus.PendingSubmit and t.order.permId != 0)
+
         loop = asyncio.get_running_loop()
         deadline = loop.time() + timeout
-        while any(t.orderStatus.status not in terminal for t in trades):
+        next_refresh = loop.time() + refresh_every
+        while not all(accepted(t) or t.orderStatus.status in OrderStatus.DoneStates for t in trades):
             remaining = deadline - loop.time()
             if remaining <= 0:
                 break
+            if loop.time() >= next_refresh:
+                try:
+                    await asyncio.wait_for(self.ib.reqOpenOrdersAsync(), remaining)
+                except TimeoutError:
+                    break
+                next_refresh = loop.time() + refresh_every
             await asyncio.sleep(min(0.1, remaining))
-        failed = [t for t in trades if t.orderStatus.status not in accepted]
+        failed = [t for t in trades if not accepted(t)]
         if failed:
             details = []
             for trade in failed:

src/ibkr_quant/cli.py

@@ -190,6 +190,17 @@ async def rebalance(cfg: Config, submit: bool) -> None:
             raise SystemExit(f"{now:%H:%M} ET is outside the order window {ORDER_WINDOW[0]}-{ORDER_WINDOW[1]}")
 
         if open_trades := await broker.open_trades():
+            # A run whose confirmation timed out exits non-zero and the trade service retries it;
+            # if IBKR holds every order that run sent, the session's orders are in.
+            refs = {t.order.orderRef for t in open_trades}
+            sent = [cfg.root / "runs" / ref.removeprefix("ibkr-quant ") / "orders.csv" for ref in refs]
+            if all(ref.startswith(f"ibkr-quant {now:%Y%m%d}-") for ref in refs) and all(p.exists() for p in sent):
+                sent_ids = set(pd.concat(pd.read_csv(p) for p in sent)["order_id"])
+                if missing := sent_ids - {t.order.orderId for t in open_trades}:
+                    raise SystemExit(f"today's orders {sorted(missing)} are no longer open at IBKR "
+                                     "(rejected, cancelled or filled); see `make check`")
+                print(f"{len(open_trades)} orders from {', '.join(sorted(refs))} are open at IBKR; not submitting again")
+                return
             raise SystemExit(f"{len(open_trades)} open orders already exist; cancel or let them finish first")
         contracts, unknown = await broker.qualify_stocks([o.symbol for o in orders])
         if unknown:

tests/test_order_confirmation.py

@@ -34,6 +34,34 @@ def test_confirmation_accepts_broker_acknowledged_orders(status):
     asyncio.run(Broker(cfg.ib).confirm_orders([trade(status)], timeout=0))
 
 
+def test_confirmation_accepts_pending_moc_orders_that_ibkr_lists():
+    # The paper account keeps MOC orders PendingSubmit until the close (2026-09-29, 10-13).
+    async def run():
+        order = trade("PendingSubmit")
+        cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml")
+        broker = Broker(cfg.ib)
+
+        async def open_orders():
+            order.order.permId = 114178004
+            return [order]
+        broker.ib.reqOpenOrdersAsync = AsyncMock(side_effect=open_orders)
+        await broker.confirm_orders([order], timeout=1, refresh_every=0.01)
+        assert order.orderStatus.status == "PendingSubmit"
+        broker.ib.reqOpenOrdersAsync.assert_awaited()
+    asyncio.run(run())
+
+
+def test_confirmation_rejects_pending_orders_ibkr_never_lists():
+    async def run():
+        cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml")
+        broker = Broker(cfg.ib)
+        broker.ib.reqOpenOrdersAsync = AsyncMock(return_value=[])
+        with pytest.raises(RuntimeError, match="AAA order 41: PendingSubmit"):
+            await broker.confirm_orders([trade("PendingSubmit")], timeout=0.5, refresh_every=0.01)
+        broker.ib.reqOpenOrdersAsync.assert_awaited()
+    asyncio.run(run())
+
+
 @pytest.mark.parametrize("status", ["Inactive", "Cancelled", "ApiCancelled", "PendingSubmit", "ApiPending"])
 def test_confirmation_rejects_terminal_failure_and_missing_acknowledgement(status):
     cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml")
@@ -82,3 +110,57 @@ def test_rebalance_records_partial_failure_and_exits_nonzero(monkeypatch, tmp_pa
     orders = pd.read_csv(next((tmp_path / "runs").glob("*/orders.csv")))
     assert dict(zip(orders.symbol, orders.status)) == {"AAA": "Submitted", "BBB": "Inactive"}
     assert "accepted 2 orders" not in capsys.readouterr().out
+
+
+def submit_with_open_orders(monkeypatch, tmp_path, open_trades, earlier_run="20261012-095323", earlier_ids=(64, 65)):
+    """Run `rebalance --submit` at 10:00 ET on 2026-10-12 while `open_trades` are open at IBKR."""
+    cfg = load_config(Path(__file__).resolve().parents[1] / "config.toml")
+    cfg = dataclasses.replace(cfg, root=tmp_path, strategy=dataclasses.replace(cfg.strategy, top_n=2))
+    now = dt.datetime(2026, 10, 12, 10, 0, tzinfo=cli.NEW_YORK)
+    run_dir = tmp_path / "runs" / earlier_run
+    run_dir.mkdir(parents=True)
+    pd.DataFrame({"symbol": ["AAA", "BBB"][:len(earlier_ids)], "order_id": list(earlier_ids),
+                  "status": "PendingSubmit"}).to_csv(run_dir / "orders.csv", index=False)
+
+    class Clock(dt.datetime):
+        @classmethod
+        def now(cls, tz=None):
+            return now
+
+    monkeypatch.setattr(cli.dt, "datetime", Clock)
+    idx = pd.bdate_range(end=now.date(), periods=300)
+    panel = pd.DataFrame({"AAA": 100.0, "BBB": 100.0}, index=idx)
+    monkeypatch.setattr(cli, "load_universe", lambda _: ["AAA", "BBB"])
+    monkeypatch.setattr(cli, "load_bars", lambda *_: (panel, panel))
+    monkeypatch.setattr(cli, "sessions_until_rebalance", lambda *_: 0)
+    monkeypatch.setattr(cli, "signal_overnight_recovery", lambda *_: panel.iloc[:-1])
+    broker = Broker(cfg.ib)
+    monkeypatch.setattr(Broker, "__aenter__", AsyncMock(return_value=broker))
+    monkeypatch.setattr(Broker, "__aexit__", AsyncMock(return_value=None))
+    monkeypatch.setattr(cli, "Broker", lambda _: broker)
+    monkeypatch.setattr(broker, "account_state", AsyncMock(return_value=AccountState("DU123", 10000, 10000, [])))
+    monkeypatch.setattr(broker, "open_trades", AsyncMock(return_value=open_trades))
+    monkeypatch.setattr(broker, "place", lambda *_: pytest.fail("placed an order while orders were open"))
+    asyncio.run(cli.rebalance(cfg, submit=True))
+
+
+def open_order(order_id, ref):
+    return Trade(Stock("AAA", "SMART", "USD"), Order(orderId=order_id, permId=114178000 + order_id, orderRef=ref),
+                 OrderStatus(status="PendingSubmit"))
+
+
+def test_retry_succeeds_when_ibkr_holds_every_order_an_earlier_run_sent(monkeypatch, tmp_path, capsys):
+    ref = "ibkr-quant 20261012-095323"
+    submit_with_open_orders(monkeypatch, tmp_path, [open_order(64, ref), open_order(65, ref)])
+    assert f"2 orders from {ref} are open at IBKR; not submitting again" in capsys.readouterr().out
+
+
+def test_retry_fails_when_an_order_from_the_earlier_run_is_gone(monkeypatch, tmp_path):
+    with pytest.raises(SystemExit, match=r"today's orders \[65\] are no longer open"):
+        submit_with_open_orders(monkeypatch, tmp_path, [open_order(64, "ibkr-quant 20261012-095323")])
+
+
+@pytest.mark.parametrize("ref", ["ibkr-quant 20261009-095323", "", "manual"])
+def test_retry_refuses_open_orders_it_did_not_send_today(monkeypatch, tmp_path, ref):
+    with pytest.raises(SystemExit, match="1 open orders already exist"):
+        submit_with_open_orders(monkeypatch, tmp_path, [open_order(64, ref)])