Complete harness-vs-naive ablation: harness 3x faster + stops; naive nondeterministic

Full naive run (dash1) reached the same TP4=0.34 optimum as the harness but took 6
iters (vs 2), never stopped (full budget), and spent trials 2-5 on worse TP2+runtime
detours. The other naive run (dash0) wandered runtime-only on TP1, found nothing, and
crashed the engine. Refined conclusion (matches paper §7.3): a strong model can
sometimes find the right knob unaided, so the harness's value is reliability + speed +
stop discipline, not that naive always fails. Harness: 2 iters-to-best, stopped at 4,
no regression. Naive: 3x slower at best, no stop, failed at worst.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

configs/examples/dash0_qwen27b_ablation_harness_on.json

@@ -0,0 +1,179 @@
+{
+  "study_id": "dash0-qwen27b-ablation-harness-on",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "qwen3.5-27b-256k-0223-internal",
+    "served_model_name": "qwen35-27b-aituner"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "latest-release-on-dash0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18082,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 180,
+    "launch_args": [
+      "serve",
+      "/home/admin/resource/model/464482ce/qwen3.5-27b/256k-0223-internal"
+    ],
+    "base_envs": {
+      "VLLM_DISABLE_COMPILE_CACHE": "1",
+      "DS_LLM_IGNORE_WARMUP": "1",
+      "DS_LLM_IGNORE_CHECK_WARMUP": "1",
+      "VLLM_ENABLE_MODEL_RUNNER_WARMUP": "1",
+      "VLLM_GDN_USE_FUSED_QKVZBA_KERNEL": "0",
+      "PARAM_TOTAL_MAX": "262144",
+      "PARAM_IN_LENGTH_MAX": "262144",
+      "PARAM_MAX_LENGTH_MAX": "131072",
+      "DS_LLM_MAX_THINK_TOKENS": "81920",
+      "DS_LLM_GRACEFUL_SHUTDOWN_WAIT_SECONDS": "600",
+      "VLLM_FP8_USE_BLADNN": "1",
+      "VLLM_MOE_USE_BLADNN": "1",
+      "VLLM_GDN_USE_BLADNN": "0",
+      "VLLM_USE_V1": "1",
+      "VLLM_IS_HYBRID_MODEL": "1",
+      "VLLM_ENABLE_TORCH_COMPILE": "1",
+      "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
+      "VLLM_QUANTIZE_ROUTED_EXPERTS_ONLY": "1",
+      "VLLM_USE_FLASHINFER_SAMPLER": "0",
+      "VLLM_DP_MASTER_PORT": "9528",
+      "VLLM_RESPONSE_TIMEOUT": "300",
+      "VLLM_LOG_REQ_KV_LENS": "1",
+      "DS_LLM_GRACEFUL_SHUTDOWN_KEEP_SECONDS": "600",
+      "CUDA_VISIBLE_DEVICES": "2,3,4,5,6,7"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18082,
+      "served-model-name": "qwen35-27b-aituner",
+      "trust-remote-code": true,
+      "dtype": "bfloat16",
+      "gpu-memory-utilization": 0.9,
+      "enable-prefix-caching": true,
+      "mamba-cache-mode": "light",
+      "distributed-executor-backend": "mp",
+      "block-size": 64,
+      "enable-chunked-prefill": true,
+      "max-num-batched-tokens": 8192,
+      "disable-cascade-attn": true,
+      "max-model-len": 262144,
+      "speculative-config": "{\"method\":\"qwen3_next_vl_mtp\",\"num_speculative_tokens\":3}",
+      "mm-processor-cache-gb": 0,
+      "limit-mm-per-prompt": "{\"image\":256,\"video\":64}",
+      "compilation-config": "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"use_inductor\":false,\"pass_config\":{\"fuse_norm_quant\":false,\"fuse_act_quant\":false,\"fuse_attn_quant\":false}}",
+      "mamba-cache-dtype": "float32",
+      "skip-mm-profiling": true,
+      "quantization": "fp8",
+      "tensor-parallel-size": 1,
+      "disable-log-requests": true
+    },
+    "tunable_envs": [
+      "VLLM_ENABLE_TORCH_COMPILE"
+    ],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "block-size",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1
+      ]
+    },
+    "python_executable": "python3"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 32,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 0.5,
+    "early_stop_max_lag_s": 45.0,
+    "early_stop_max_elapsed_s": 320.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    },
+    "completion_tokens_override": 128
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "linear_ms",
+      "intercept_ms": 4000,
+      "per_token_ms": 0.125
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.25,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325,
+    "inherit_incumbent_floor": true
+  },
+  "llm": {
+    "system_prompt": "Propose a single engine config patch that increases the maximum feasible sampling_u under the SLO target. Favor launch-safe changes grounded in the incumbent result and only propose knobs that plausibly improve throughput above the incumbent request rate.",
+    "max_history_trials": 8,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.4",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 180
+    },
+    "use_harness": true
+  }
+}

configs/examples/dash0_qwen27b_ablation_naive_off.json

@@ -0,0 +1,179 @@
+{
+  "study_id": "dash0-qwen27b-ablation-naive-off",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "qwen3.5-27b-256k-0223-internal",
+    "served_model_name": "qwen35-27b-aituner"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "latest-release-on-dash0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18082,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 180,
+    "launch_args": [
+      "serve",
+      "/home/admin/resource/model/464482ce/qwen3.5-27b/256k-0223-internal"
+    ],
+    "base_envs": {
+      "VLLM_DISABLE_COMPILE_CACHE": "1",
+      "DS_LLM_IGNORE_WARMUP": "1",
+      "DS_LLM_IGNORE_CHECK_WARMUP": "1",
+      "VLLM_ENABLE_MODEL_RUNNER_WARMUP": "1",
+      "VLLM_GDN_USE_FUSED_QKVZBA_KERNEL": "0",
+      "PARAM_TOTAL_MAX": "262144",
+      "PARAM_IN_LENGTH_MAX": "262144",
+      "PARAM_MAX_LENGTH_MAX": "131072",
+      "DS_LLM_MAX_THINK_TOKENS": "81920",
+      "DS_LLM_GRACEFUL_SHUTDOWN_WAIT_SECONDS": "600",
+      "VLLM_FP8_USE_BLADNN": "1",
+      "VLLM_MOE_USE_BLADNN": "1",
+      "VLLM_GDN_USE_BLADNN": "0",
+      "VLLM_USE_V1": "1",
+      "VLLM_IS_HYBRID_MODEL": "1",
+      "VLLM_ENABLE_TORCH_COMPILE": "1",
+      "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
+      "VLLM_QUANTIZE_ROUTED_EXPERTS_ONLY": "1",
+      "VLLM_USE_FLASHINFER_SAMPLER": "0",
+      "VLLM_DP_MASTER_PORT": "9528",
+      "VLLM_RESPONSE_TIMEOUT": "300",
+      "VLLM_LOG_REQ_KV_LENS": "1",
+      "DS_LLM_GRACEFUL_SHUTDOWN_KEEP_SECONDS": "600",
+      "CUDA_VISIBLE_DEVICES": "2,3,4,5,6,7"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18082,
+      "served-model-name": "qwen35-27b-aituner",
+      "trust-remote-code": true,
+      "dtype": "bfloat16",
+      "gpu-memory-utilization": 0.9,
+      "enable-prefix-caching": true,
+      "mamba-cache-mode": "light",
+      "distributed-executor-backend": "mp",
+      "block-size": 64,
+      "enable-chunked-prefill": true,
+      "max-num-batched-tokens": 8192,
+      "disable-cascade-attn": true,
+      "max-model-len": 262144,
+      "speculative-config": "{\"method\":\"qwen3_next_vl_mtp\",\"num_speculative_tokens\":3}",
+      "mm-processor-cache-gb": 0,
+      "limit-mm-per-prompt": "{\"image\":256,\"video\":64}",
+      "compilation-config": "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"use_inductor\":false,\"pass_config\":{\"fuse_norm_quant\":false,\"fuse_act_quant\":false,\"fuse_attn_quant\":false}}",
+      "mamba-cache-dtype": "float32",
+      "skip-mm-profiling": true,
+      "quantization": "fp8",
+      "tensor-parallel-size": 1,
+      "disable-log-requests": true
+    },
+    "tunable_envs": [
+      "VLLM_ENABLE_TORCH_COMPILE"
+    ],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "block-size",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1
+      ]
+    },
+    "python_executable": "python3"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 32,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 0.5,
+    "early_stop_max_lag_s": 45.0,
+    "early_stop_max_elapsed_s": 320.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    },
+    "completion_tokens_override": 128
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "linear_ms",
+      "intercept_ms": 4000,
+      "per_token_ms": 0.125
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.25,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325,
+    "inherit_incumbent_floor": true
+  },
+  "llm": {
+    "system_prompt": "Propose a single engine config patch that increases the maximum feasible sampling_u under the SLO target. Favor launch-safe changes grounded in the incumbent result and only propose knobs that plausibly improve throughput above the incumbent request rate.",
+    "max_history_trials": 8,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.4",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 180
+    },
+    "use_harness": false
+  }
+}

docs/harness-ablation/harness-vs-naive-20260616.md

@@ -0,0 +1,99 @@
+# Harness vs naive (use_harness on/off) — convergence ablation — 2026-06-16/17
+
+Controlled ablation of the paper's "harness" (domain-knowledge knob-family steering):
+the same agentic loop with `llm.use_harness=true` vs `false` (= the paper's naive
+agentic tuner: free-form LLM proposals, no `Harnesses:` prompt section, no
+deterministic guided proposals, no Stop-B validator/veto). Same workload, model, SLO,
+substrate — the only difference is `use_harness` (configs
+`dash0_qwen27b_ablation_harness_on.json` / `..._naive_off.json`, verified to differ
+only in that flag + study_id).
+
+- Model: dense Qwen3.5-27B, vLLM 0.11.1, 8×H20 (dash0 and dash1 share the cpfs mount).
+- Workload: chat 0–8k, length-aware TTFT SLO (4s + L_in/8k) + TPOT ≤ 50 ms, pass ≥ 95%.
+- Substrate (process comparison, not absolute peak-rate): `replay_time_scale=0.5`,
+  `completion_tokens_override=128`, Stop-A on, `search.high=0.25`, 6 probes, max-trials 6,
+  **`--skip-baseline`** (the low-capacity TP1 auto-baseline is infeasible under this
+  SLO+compression and would trip `baseline_all_infeasible`; skipping it lets both loops
+  climb from their first proposal).
+- This measures the tuning *process* (which knob family, convergence speed, stop
+  discipline), not a validated peak-rate.
+
+## Harness ON — converged in 2 iterations, then stopped
+| iter | proposer | config | per_gpu | outcome |
+| --- | --- | --- | --- | --- |
+| 1 | LLM (harness-guided) | TP2 | 0.247 | feasible |
+| 2 | harness (deterministic) | **TP4** | **0.340** | feasible — incumbent |
+| 3 | harness | TP4 + chunked-prefill + mbt=16384 | 0.333 | worse → rejected |
+| (—) | LLM | `should_stop` | — | **VETOED** ("decode TPOT still the bottleneck; adjacent probes weak") |
+| 4 | LLM | TP2 + DP2 | 0.194 | worse → rejected |
+| (—) | LLM | `should_stop` | **STOP** | honored after veto budget |
+
+Best **TP4 @ 0.340**; iters-to-best = **2**; ran **4 trials then stopped** (Stop-B +
+one veto of a premature stop); no regression.
+
+## Naive OFF — nondeterministic; reaches the optimum slowly at best, fails at worst
+
+The naive (free-form) `gpt-5.4` loop behaved very differently across two runs — it has
+no harness steering and no stop logic:
+
+**Run A (dash0, interrupted by an outage at trial-5):** kept **TP=1** the whole time and
+cycled runtime knobs (`max-num-batched-tokens` 16k→65k, `max-num-seqs`, caching). All
+trials **infeasible** (same `tpot>50` + `ttft>budget`), trial-4 **crashed the engine**
+(OOM at mbt=65536). Found **no feasible config** in 5 trials — never tried raising TP.
+
+**Run B (dash1, full budget):**
+| iter | config | per_gpu | note |
+| --- | --- | --- | --- |
+| 1 | TP2 | 0.247 | feasible |
+| 2 | TP2 + max-num-seqs=32 | 0.218 | worse |
+| 3 | TP2 + mbt=12288 | 0.218 | worse |
+| 4 | TP2 (re-proposal) | 0.218 | no gain |
+| 5 | TP2 + gpu-mem-util=0.85 | 0.218 | worse |
+| 6 | **TP4** | **0.340** | reaches the optimum — at the last trial |
+
+Best **TP4 @ 0.340** — the *same* optimum as the harness — but iters-to-best = **6**,
+it used the **entire budget with no early stop**, and trials 2–5 were detours (TP2 +
+runtime tweaks, all worse than trial-1) before it stumbled onto TP4.
+
+## Comparison
+
+| | Harness ON | Naive OFF (B, dash1) | Naive OFF (A, dash0) |
+| --- | --- | --- | --- |
+| best per-GPU | 0.340 (TP4) | 0.340 (TP4) | none (failed) |
+| iters-to-best | **2** | 6 | — |
+| trials used | **4 (stopped)** | 6 (full budget, no stop) | 5 (interrupted) |
+| stopped early? | yes (Stop-B + veto) | no | — |
+| wasted trials | 2 (post-best refinements) | 4 (TP2+runtime detours) | 5 (runtime-only, infeasible) |
+| path to optimum | direct (TP2→TP4) | slow (TP2→runtime detour→TP4) | wrong family (runtime on TP1) |
+
+## Interpretation (honest)
+
+The bottleneck is **compute** (decode TPOT + prefill queueing), which only a
+compute-adding knob (**tensor parallelism**) fixes. Findings:
+
+1. **A strong frontier model can sometimes find the right knob unaided** — naive run B
+   eventually reached TP4 = 0.34, the same optimum as the harness. This matches the
+   paper's own caveat (§7.3): stronger models reduce, but do not remove, the need for
+   structured guidance. So the harness's value is **not** "naive always fails."
+2. **The harness's value is reliability, speed, and stop discipline.** With the harness:
+   converged in **2 iters** and **stopped at 4** (recognized convergence; vetoed a
+   premature stop). Naive: **3× slower** to the same answer (6 iters), **never stopped**
+   (burned the full budget on detours), and in run A **failed outright** — never tried
+   TP, found nothing, crashed the engine. Naive is **nondeterministic and unreliable**;
+   the harness is fast, monotone (no regression), and self-terminating.
+3. This reproduces the paper's Figure-18 story: the harness converges in a few
+   iterations and stops, while the naive agentic tuner wastes the budget (and can fail
+   to converge entirely).
+
+## Caveats
+
+- Compressed substrate (scale=0.5, out=128) → per-GPU numbers are *process* comparators,
+  not validated peak-rates; the convergence behavior is the result. (The TP4 optimum
+  reproduced at 0.340 across the harness run and naive run B, a useful consistency check.)
+- One run per arm per host; naive is nondeterministic (runs A and B differ markedly),
+  which is itself part of the finding. The harness arm's deterministic guided proposal
+  (TP4 at iter 2) and validator veto are reproducible.
+- Infra notes: dash0 (LLM-gateway reachable) went down mid-experiment; dash1 shares the
+  cpfs and ran the completion. The codex `config.toml` points at a dash0-local proxy
+  (`127.0.0.1:11235`); on dash1 the LLM endpoint must be reached directly (set empty
+  `*_proxy` env) — see `scripts/run_naive_d1.sh`.

docs/two-stop-summary.md

@@ -0,0 +1,63 @@
+# Two-stop work — summary (feat/two-stop)
+
+Aligns the tuning harness with paper.pdf by implementing and validating the two
+distinct stopping mechanisms the paper conflates, plus a length-aware SLO and two
+harness-robustness fixes. 117 unit tests pass.
+
+## The two stops (they are different things)
+
+- **Stop-A — evaluation sufficiency** (per replay / probe): how much trace to replay
+  before a measurement is trustworthy. Criterion: the replayed prefix's offered
+  L-C-A converges to the full set's. Saves per-evaluation GPU time.
+- **Stop-B — tuning convergence** (across iterations): whether any config-improvement
+  opportunity remains; stop if not. Saves iterations.
+
+## What was built
+
+| Area | Summary | Commit |
+| --- | --- | --- |
+| Unify L-C-A | The prompt's `workload_lca_profile` is now the canonical 10-dim `lca.py` vector, not an ad-hoc re-derivation | `6f8e3c9` |
+| Session-coherent load axis | `sampling_u` assigned per session (parent_chat_id chain) so thresholding keeps/drops whole multi-turn sessions, preserving intra-session KV reuse (C) across load levels | `0f15bbc` |
+| Stop-A | `lca.find_convergence_prefix` (deterministic offered-L-C-A convergence prefix + C-gate: never declare infeasible on a cold cache); `spec.AdaptiveStopSpec` (default off); `worker._adaptive_replay_set` truncates replay + writes a certificate | `51a9e4a` |
+| Stop-A boundary guard | Re-measure the full window when a truncated probe's pass-rate is within `boundary_delta` of the SLO target (fixes feasibility-knee false positives) | `dfc823f` |
+| Stop-B authority | `harness._stop_authority` (mirrors the deterministic validator); `study tune` honors an LLM `should_stop` only if the validator agrees, else a bounded veto | `a8f9034` |
+| Length-aware SLO | `linear_ms` rule: `threshold = intercept_ms + per_token_ms·L_in` (e.g. 4s + L_in/8k) | `ed2bbe0` |
+| Robustness: timeout | HTTP stream now wraps `OSError`/`TimeoutError` as `HttpClientError` — a request exceeding `request_timeout_s` is a failed request, not a crashed trial | `2fcaf80` |
+| Robustness: SIGTERM | `run_trial` installs a SIGTERM handler so killing `study tune` tears down the engine (and EngineCore workers) instead of orphaning it / leaking GPU memory | `b17b213` |
+| Tooling | `stop_a_calibration.py` (CPU convergence curve), `stop_a_validate.py` (offline truncation-fidelity) | `08e53fd`, `3af1d84` |
+
+A subagent code review found no blocking bugs (and independently validated session
+coherence against a real trace); three minor fixes applied in `43125f4`.
+
+## Validation results (real GPU runs, dash0 H20)
+
+- **Stop-A** (`stop-a-validation-20260615.md`): CPU calibration reproduces the paper's
+  C-slowest ordering; GPU fidelity check on Qwen3-30B-A3B saves ~52% replay at τ_c=0.90
+  with 3/4 probe verdicts preserved; the one mismatch is a feasibility-knee false
+  positive that the **boundary guard fixes** — with the guard, best threshold matches
+  full replay exactly while still saving ~38% replay.
+- **27B TP sweep** (`qwen27b-tp-sweep-20260616.md`): under the length-aware SLO, dense
+  Qwen3.5-27B per-GPU rises sharply with TP — TP1 0.065 → TP2 0.29 → TP4 ≥0.91 — the
+  **opposite** of the MoE 30B-A3B (TP1 best per-GPU). TP1 is TPOT-bound.
+- **Stop-B** (`stop-b-e2e-20260615.md`, `stop-b-e2e-27b-20260616.md`): the 30B run
+  shows the deterministic `validator` stop + a premature-LLM-stop **veto**; the 27B run
+  (real gpt-5.4 loop) shows a genuine **improving climb** TP1 0.123 → TP2 0.29 → TP4
+  1.00 req/s/GPU (8.1×), each a correctly-diagnosed gain, then correctly **rejecting** a
+  TP4 runtime tweak that measured worse (no regression). The SIGTERM fix was validated
+  in practice (clean teardown, no leak).
+
+## Open items (next round)
+
+- **Harness convergence ablation (NOT yet done on this branch).** The paper's harness
+  result — domain-knowledge knob-family rules steering the LLM and cutting iterations —
+  has only *qualitative* evidence here (the 27B climb shows correct steering) plus older
+  smoke-regime ablations (`qwen27b-chat-0-8k-harness-fig18.md`: iters-to-best 4→2). A
+  controlled `use_harness=true` vs `false` (naive tuner) comparison on the 27B is the
+  missing quantified result.
+- **Loop efficiency**: at scale=1.0 infeasible high-θ probes burn the
+  `early_stop_max_elapsed_s=900` cap (a TP4 trial took ~3h). Lower it to ~300s for
+  practical agentic loops.
+- **dash0 GPUs 0/1** still hold leaked memory (pre-fix orphans) — needs a root
+  `nvidia-smi --gpu-reset`.
+- Deferred: more robust C feature for the low-reuse regime; Stop-C cross-day retune
+  trigger (paper Q1); §7 baselines (SCOOT / naive / community).

scripts/ablation_trajectory.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Extract a per-iteration trajectory table from an ablation study store.
+
+Usage: python3 ablation_trajectory.py <study_store_dir>
+Prints iter, proposal source/name, config_patch summary, per_gpu, status,
+and the running incumbent per_gpu. Read-only.
+"""
+import json
+import sys
+from pathlib import Path
+
+
+def topo(patch):
+    fp = (patch or {}).get("flag_patch", {}) or {}
+    ep = (patch or {}).get("env_patch", {}) or {}
+    parts = []
+    for k, label in (
+        ("tensor-parallel-size", "TP"),
+        ("data-parallel-size", "DP"),
+        ("expert-parallel-size", "EP"),
+    ):
+        if k in fp:
+            parts.append(f"{label}{fp[k]}")
+    runtime = {
+        k: v
+        for k, v in fp.items()
+        if k not in ("tensor-parallel-size", "data-parallel-size", "expert-parallel-size")
+    }
+    runtime.update({f"env:{k}": v for k, v in ep.items()})
+    base = "+".join(parts) if parts else "baseline-topo"
+    if runtime:
+        base += " | " + ", ".join(f"{k}={v}" for k, v in runtime.items())
+    return base
+
+
+def main():
+    store = Path(sys.argv[1])
+    state = json.load(open(store / "state.json"))
+    print(f"study_id: {state.get('study_id')}")
+    print(f"best_trial: {state.get('best_trial_id')}  best_per_gpu: {state.get('best_request_rate_per_gpu')}")
+    print(f"stop_reason: {state.get('tuning_stop_reason')!r}")
+    print(f"stop_diagnosis: {state.get('tuning_stop_diagnosis')!r}")
+    print(f"stop_details: {json.dumps(state.get('tuning_stop_details'), ensure_ascii=False)}")
+    print()
+    incumbent = None
+    hdr = f"{'iter':<5}{'trial':<11}{'status':<14}{'per_gpu':<10}{'incumbent':<11}config"
+    print(hdr)
+    print("-" * len(hdr))
+    for i, t in enumerate(state.get("trials", []), 1):
+        pg = t.get("best_request_rate_per_gpu")
+        if pg is not None and (incumbent is None or pg > incumbent):
+            incumbent = pg
+        pgs = f"{pg:.4f}" if isinstance(pg, (int, float)) else str(pg)
+        incs = f"{incumbent:.4f}" if isinstance(incumbent, (int, float)) else str(incumbent)
+        print(
+            f"{i:<5}{t.get('trial_id',''):<11}{str(t.get('status','')):<14}{pgs:<10}{incs:<11}{topo(t.get('config_patch'))}"
+        )
+    # also dump proposals dir to see what was *proposed* (incl. vetoed/failed)
+    pdir = store / "proposals"
+    if pdir.exists():
+        print("\n-- proposal files (chronological) --")
+        for p in sorted(pdir.glob("*.json")):
+            try:
+                pr = json.load(open(p))
+            except Exception:
+                continue
+            print(f"  {p.stem}: should_stop={pr.get('should_stop')} | {topo(pr.get('config_patch'))}")
+
+
+if __name__ == "__main__":
+    main()

scripts/run_ablation_pair.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Run the harness-ON then naive-OFF tuning loops sequentially (use_harness ablation),
+# then drop a DONE marker. Run from the repo root on the GPU host.
+set -u
+export OPENAI_API_KEY=$(python3 -c 'import json,pathlib;print(json.load(open(pathlib.Path.home()/".codex/auth.json"))["OPENAI_API_KEY"])')
+mkdir -p .aituner
+rm -rf .aituner/abl-harness .aituner/abl-naive .aituner/ABLATION_DONE
+
+echo "=== harness ON  start $(date -Is) ==="
+PYTHONPATH=src python3 -m aituner.cli study tune \
+  --spec configs/examples/dash0_qwen27b_ablation_harness_on.json \
+  --store-root .aituner/abl-harness --max-trials 6 --skip-baseline > .aituner/abl-harness.log 2>&1
+echo "=== harness ON  done  $(date -Is) ==="
+
+echo "=== naive OFF   start $(date -Is) ==="
+PYTHONPATH=src python3 -m aituner.cli study tune \
+  --spec configs/examples/dash0_qwen27b_ablation_naive_off.json \
+  --store-root .aituner/abl-naive --max-trials 6 --skip-baseline > .aituner/abl-naive.log 2>&1
+echo "=== naive OFF   done  $(date -Is) ==="
+
+touch .aituner/ABLATION_DONE

scripts/run_naive_d1.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Complete the naive-OFF ablation arm to full budget on a fresh store (dash1).
+set -u
+export OPENAI_API_KEY=$(python3 -c 'import json,pathlib;print(json.load(open(pathlib.Path.home()/".codex/auth.json"))["OPENAI_API_KEY"])')
+# codex config.toml points at a local proxy (127.0.0.1:11235) that exists on dash0 but
+# not dash1; the LLM endpoint is reachable directly, so force a direct connection.
+export http_proxy= https_proxy= all_proxy= HTTP_PROXY= HTTPS_PROXY= ALL_PROXY= no_proxy='*'
+mkdir -p .aituner
+rm -rf .aituner/abl-naive-d1 .aituner/ABL_NAIVE_D1_DONE
+echo "=== naive OFF (dash1) start $(date -Is) ==="
+PYTHONPATH=src python3 -m aituner.cli study tune \
+  --spec configs/examples/dash0_qwen27b_ablation_naive_off.json \
+  --store-root .aituner/abl-naive-d1 --max-trials 6 --skip-baseline > .aituner/abl-naive-d1.log 2>&1
+echo "=== naive OFF (dash1) done  $(date -Is) ==="
+touch .aituner/ABL_NAIVE_D1_DONE