Report held-out active intervention result

Entry-point summary of both closed campaigns: verdicts, corrected
readings (P5 supersedes P3's P10-vs-P04 magnitude), methodological
findings (co-location SLO validity, arrival uniformization, compile-
factor env poisoning), artifact map, and GPU accounting.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.gitignore

@@ -4,6 +4,7 @@
 .aituner-tight/
 .aituner-prefill/
 .aituner-compare/
+.aituner-run-configs/
 .env
 __pycache__/
 *.pyc
@@ -13,3 +14,8 @@ configs/examples/dash0_llm_10min_study_run[0-9]*.json
 configs/examples/dash0_smoke_study_run*.json
 infra/gpu_fleet/config/fleet.toml
 infra/gpu_fleet/config/jobs.toml
+# Raw Layer-1 telemetry streams (507 MB; decision-bearing JSON/log evidence stays tracked)
+runs/**/*.jsonl
+.ruff_cache/
+# Recovered dash1 interaction-run stores (100 MB raw tune logs, kept on disk only)
+recovered-stores/

AGENTS.md

@@ -6,6 +6,10 @@
 - Hardware expectation: 8 NVIDIA H20 GPUs.
 - SSH check: use `ssh dash0` before scheduling or debugging remote runs.
 - Remote project path: `/home/admin/cpfs/wjh/aituner/aituner`.
+- If remote downloads are slow or fail, start the proxy from the remote `wjh`
+  home directory with `./auto_proxy.sh`, then run downloads in a shell where
+  `proxyOn` from `~/.bashrc` has been applied. If `autossh` is unavailable,
+  `ssh -Nf proxy` provides the same local `127.0.0.1:11235` tunnel.
 
 ## Local/remote sync workflow
 

configs/examples/dash0_qwen27b_ablation_harness_on.json

@@ -0,0 +1,180 @@
+{
+  "study_id": "dash0-qwen27b-ablation-harness-on",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "qwen3.5-27b-256k-0223-internal",
+    "served_model_name": "qwen35-27b-aituner"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "latest-release-on-dash0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18082,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 180,
+    "launch_args": [
+      "serve",
+      "/home/admin/resource/model/464482ce/qwen3.5-27b/256k-0223-internal"
+    ],
+    "base_envs": {
+      "VLLM_DISABLE_COMPILE_CACHE": "1",
+      "DS_LLM_IGNORE_WARMUP": "1",
+      "DS_LLM_IGNORE_CHECK_WARMUP": "1",
+      "VLLM_ENABLE_MODEL_RUNNER_WARMUP": "1",
+      "VLLM_GDN_USE_FUSED_QKVZBA_KERNEL": "0",
+      "PARAM_TOTAL_MAX": "262144",
+      "PARAM_IN_LENGTH_MAX": "262144",
+      "PARAM_MAX_LENGTH_MAX": "131072",
+      "DS_LLM_MAX_THINK_TOKENS": "81920",
+      "DS_LLM_GRACEFUL_SHUTDOWN_WAIT_SECONDS": "600",
+      "VLLM_FP8_USE_BLADNN": "1",
+      "VLLM_MOE_USE_BLADNN": "1",
+      "VLLM_GDN_USE_BLADNN": "0",
+      "VLLM_USE_V1": "1",
+      "VLLM_IS_HYBRID_MODEL": "1",
+      "VLLM_ENABLE_TORCH_COMPILE": "1",
+      "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
+      "VLLM_QUANTIZE_ROUTED_EXPERTS_ONLY": "1",
+      "VLLM_USE_FLASHINFER_SAMPLER": "0",
+      "VLLM_DP_MASTER_PORT": "9528",
+      "VLLM_RESPONSE_TIMEOUT": "300",
+      "VLLM_LOG_REQ_KV_LENS": "1",
+      "DS_LLM_GRACEFUL_SHUTDOWN_KEEP_SECONDS": "600",
+      "CUDA_VISIBLE_DEVICES": "2,3,4,5,6,7"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18082,
+      "served-model-name": "qwen35-27b-aituner",
+      "trust-remote-code": true,
+      "dtype": "bfloat16",
+      "gpu-memory-utilization": 0.9,
+      "enable-prefix-caching": true,
+      "mamba-cache-mode": "light",
+      "distributed-executor-backend": "mp",
+      "block-size": 64,
+      "enable-chunked-prefill": true,
+      "max-num-batched-tokens": 8192,
+      "disable-cascade-attn": true,
+      "max-model-len": 262144,
+      "speculative-config": "{\"method\":\"qwen3_next_vl_mtp\",\"num_speculative_tokens\":3}",
+      "mm-processor-cache-gb": 0,
+      "limit-mm-per-prompt": "{\"image\":256,\"video\":64}",
+      "compilation-config": "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"use_inductor\":false,\"pass_config\":{\"fuse_norm_quant\":false,\"fuse_act_quant\":false,\"fuse_attn_quant\":false}}",
+      "mamba-cache-dtype": "float32",
+      "skip-mm-profiling": true,
+      "quantization": "fp8",
+      "tensor-parallel-size": 1,
+      "disable-log-requests": true
+    },
+    "tunable_envs": [
+      "VLLM_ENABLE_TORCH_COMPILE"
+    ],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "block-size",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1
+      ]
+    },
+    "python_executable": "python3"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 32,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 0.8775,
+    "early_stop_max_lag_s": 45.0,
+    "early_stop_max_elapsed_s": 1000.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "linear_ms",
+      "intercept_ms": 4000,
+      "per_token_ms": 0.125
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.15,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325,
+    "inherit_incumbent_floor": true
+  },
+  "llm": {
+    "system_prompt": "Propose a single engine config patch that increases the maximum feasible sampling_u under the SLO target. Favor launch-safe changes grounded in the incumbent result and only propose knobs that plausibly improve throughput above the incumbent request rate.",
+    "max_history_trials": 8,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.5",
+      "base_url": "https://ai.gahow.org/v1",
+      "wire_api": "chat.completions",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 180
+    },
+    "use_harness": true
+  }
+}

configs/examples/dash0_qwen27b_ablation_naive_off.json

@@ -0,0 +1,180 @@
+{
+  "study_id": "dash0-qwen27b-ablation-naive-off",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "qwen3.5-27b-256k-0223-internal",
+    "served_model_name": "qwen35-27b-aituner"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "latest-release-on-dash0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18082,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 180,
+    "launch_args": [
+      "serve",
+      "/home/admin/resource/model/464482ce/qwen3.5-27b/256k-0223-internal"
+    ],
+    "base_envs": {
+      "VLLM_DISABLE_COMPILE_CACHE": "1",
+      "DS_LLM_IGNORE_WARMUP": "1",
+      "DS_LLM_IGNORE_CHECK_WARMUP": "1",
+      "VLLM_ENABLE_MODEL_RUNNER_WARMUP": "1",
+      "VLLM_GDN_USE_FUSED_QKVZBA_KERNEL": "0",
+      "PARAM_TOTAL_MAX": "262144",
+      "PARAM_IN_LENGTH_MAX": "262144",
+      "PARAM_MAX_LENGTH_MAX": "131072",
+      "DS_LLM_MAX_THINK_TOKENS": "81920",
+      "DS_LLM_GRACEFUL_SHUTDOWN_WAIT_SECONDS": "600",
+      "VLLM_FP8_USE_BLADNN": "1",
+      "VLLM_MOE_USE_BLADNN": "1",
+      "VLLM_GDN_USE_BLADNN": "0",
+      "VLLM_USE_V1": "1",
+      "VLLM_IS_HYBRID_MODEL": "1",
+      "VLLM_ENABLE_TORCH_COMPILE": "1",
+      "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
+      "VLLM_QUANTIZE_ROUTED_EXPERTS_ONLY": "1",
+      "VLLM_USE_FLASHINFER_SAMPLER": "0",
+      "VLLM_DP_MASTER_PORT": "9528",
+      "VLLM_RESPONSE_TIMEOUT": "300",
+      "VLLM_LOG_REQ_KV_LENS": "1",
+      "DS_LLM_GRACEFUL_SHUTDOWN_KEEP_SECONDS": "600",
+      "CUDA_VISIBLE_DEVICES": "2,3,4,5,6,7"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18082,
+      "served-model-name": "qwen35-27b-aituner",
+      "trust-remote-code": true,
+      "dtype": "bfloat16",
+      "gpu-memory-utilization": 0.9,
+      "enable-prefix-caching": true,
+      "mamba-cache-mode": "light",
+      "distributed-executor-backend": "mp",
+      "block-size": 64,
+      "enable-chunked-prefill": true,
+      "max-num-batched-tokens": 8192,
+      "disable-cascade-attn": true,
+      "max-model-len": 262144,
+      "speculative-config": "{\"method\":\"qwen3_next_vl_mtp\",\"num_speculative_tokens\":3}",
+      "mm-processor-cache-gb": 0,
+      "limit-mm-per-prompt": "{\"image\":256,\"video\":64}",
+      "compilation-config": "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"use_inductor\":false,\"pass_config\":{\"fuse_norm_quant\":false,\"fuse_act_quant\":false,\"fuse_attn_quant\":false}}",
+      "mamba-cache-dtype": "float32",
+      "skip-mm-profiling": true,
+      "quantization": "fp8",
+      "tensor-parallel-size": 1,
+      "disable-log-requests": true
+    },
+    "tunable_envs": [
+      "VLLM_ENABLE_TORCH_COMPILE"
+    ],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "block-size",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1
+      ]
+    },
+    "python_executable": "python3"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 32,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 0.8775,
+    "early_stop_max_lag_s": 45.0,
+    "early_stop_max_elapsed_s": 1000.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "linear_ms",
+      "intercept_ms": 4000,
+      "per_token_ms": 0.125
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.15,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325,
+    "inherit_incumbent_floor": true
+  },
+  "llm": {
+    "system_prompt": "Propose a single engine config patch that increases the maximum feasible sampling_u under the SLO target. Favor launch-safe changes grounded in the incumbent result and only propose knobs that plausibly improve throughput above the incumbent request rate.",
+    "max_history_trials": 8,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.5",
+      "base_url": "https://ai.gahow.org/v1",
+      "wire_api": "chat.completions",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 180
+    },
+    "use_harness": false
+  }
+}

configs/examples/dash0_qwen27b_stopB_loop.json

@@ -0,0 +1,177 @@
+{
+  "study_id": "dash0-qwen27b-stopB-loop-chat-0-8k",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "qwen3.5-27b-256k-0223-internal",
+    "served_model_name": "qwen35-27b-aituner"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "latest-release-on-dash0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18082,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 180,
+    "launch_args": [
+      "serve",
+      "/home/admin/resource/model/464482ce/qwen3.5-27b/256k-0223-internal"
+    ],
+    "base_envs": {
+      "VLLM_DISABLE_COMPILE_CACHE": "1",
+      "DS_LLM_IGNORE_WARMUP": "1",
+      "DS_LLM_IGNORE_CHECK_WARMUP": "1",
+      "VLLM_ENABLE_MODEL_RUNNER_WARMUP": "1",
+      "VLLM_GDN_USE_FUSED_QKVZBA_KERNEL": "0",
+      "PARAM_TOTAL_MAX": "262144",
+      "PARAM_IN_LENGTH_MAX": "262144",
+      "PARAM_MAX_LENGTH_MAX": "131072",
+      "DS_LLM_MAX_THINK_TOKENS": "81920",
+      "DS_LLM_GRACEFUL_SHUTDOWN_WAIT_SECONDS": "600",
+      "VLLM_FP8_USE_BLADNN": "1",
+      "VLLM_MOE_USE_BLADNN": "1",
+      "VLLM_GDN_USE_BLADNN": "0",
+      "VLLM_USE_V1": "1",
+      "VLLM_IS_HYBRID_MODEL": "1",
+      "VLLM_ENABLE_TORCH_COMPILE": "1",
+      "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
+      "VLLM_QUANTIZE_ROUTED_EXPERTS_ONLY": "1",
+      "VLLM_USE_FLASHINFER_SAMPLER": "0",
+      "VLLM_DP_MASTER_PORT": "9528",
+      "VLLM_RESPONSE_TIMEOUT": "300",
+      "VLLM_LOG_REQ_KV_LENS": "1",
+      "DS_LLM_GRACEFUL_SHUTDOWN_KEEP_SECONDS": "600",
+      "CUDA_VISIBLE_DEVICES": "2,3,4,5,6,7"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18082,
+      "served-model-name": "qwen35-27b-aituner",
+      "trust-remote-code": true,
+      "dtype": "bfloat16",
+      "gpu-memory-utilization": 0.9,
+      "enable-prefix-caching": true,
+      "mamba-cache-mode": "light",
+      "distributed-executor-backend": "mp",
+      "block-size": 64,
+      "enable-chunked-prefill": true,
+      "max-num-batched-tokens": 8192,
+      "disable-cascade-attn": true,
+      "max-model-len": 262144,
+      "speculative-config": "{\"method\":\"qwen3_next_vl_mtp\",\"num_speculative_tokens\":3}",
+      "mm-processor-cache-gb": 0,
+      "limit-mm-per-prompt": "{\"image\":256,\"video\":64}",
+      "compilation-config": "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"use_inductor\":false,\"pass_config\":{\"fuse_norm_quant\":false,\"fuse_act_quant\":false,\"fuse_attn_quant\":false}}",
+      "mamba-cache-dtype": "float32",
+      "skip-mm-profiling": true,
+      "quantization": "fp8",
+      "tensor-parallel-size": 1,
+      "disable-log-requests": true
+    },
+    "tunable_envs": [
+      "VLLM_ENABLE_TORCH_COMPILE"
+    ],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "block-size",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1
+      ]
+    },
+    "python_executable": "python3"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 32,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 1.0,
+    "early_stop_max_lag_s": 120.0,
+    "early_stop_max_elapsed_s": 900.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "linear_ms",
+      "intercept_ms": 4000,
+      "per_token_ms": 0.125
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.25,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325,
+    "inherit_incumbent_floor": true
+  },
+  "llm": {
+    "system_prompt": "Propose a single engine config patch that increases the maximum feasible sampling_u under the SLO target. Favor launch-safe changes grounded in the incumbent result and only propose knobs that plausibly improve throughput above the incumbent request rate.",
+    "max_history_trials": 8,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.4",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 180
+    }
+  }
+}

configs/examples/dash0_qwen27b_tp_ab.json

@@ -0,0 +1,176 @@
+{
+  "study_id": "dash0-qwen27b-tp-ab-chat-0-8k",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "qwen3.5-27b-256k-0223-internal",
+    "served_model_name": "qwen35-27b-aituner"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "latest-release-on-dash0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18082,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 180,
+    "launch_args": [
+      "serve",
+      "/home/admin/resource/model/464482ce/qwen3.5-27b/256k-0223-internal"
+    ],
+    "base_envs": {
+      "VLLM_DISABLE_COMPILE_CACHE": "1",
+      "DS_LLM_IGNORE_WARMUP": "1",
+      "DS_LLM_IGNORE_CHECK_WARMUP": "1",
+      "VLLM_ENABLE_MODEL_RUNNER_WARMUP": "1",
+      "VLLM_GDN_USE_FUSED_QKVZBA_KERNEL": "0",
+      "PARAM_TOTAL_MAX": "262144",
+      "PARAM_IN_LENGTH_MAX": "262144",
+      "PARAM_MAX_LENGTH_MAX": "131072",
+      "DS_LLM_MAX_THINK_TOKENS": "81920",
+      "DS_LLM_GRACEFUL_SHUTDOWN_WAIT_SECONDS": "600",
+      "VLLM_FP8_USE_BLADNN": "1",
+      "VLLM_MOE_USE_BLADNN": "1",
+      "VLLM_GDN_USE_BLADNN": "0",
+      "VLLM_USE_V1": "1",
+      "VLLM_IS_HYBRID_MODEL": "1",
+      "VLLM_ENABLE_TORCH_COMPILE": "1",
+      "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
+      "VLLM_QUANTIZE_ROUTED_EXPERTS_ONLY": "1",
+      "VLLM_USE_FLASHINFER_SAMPLER": "0",
+      "VLLM_DP_MASTER_PORT": "9528",
+      "VLLM_RESPONSE_TIMEOUT": "300",
+      "VLLM_LOG_REQ_KV_LENS": "1",
+      "DS_LLM_GRACEFUL_SHUTDOWN_KEEP_SECONDS": "600",
+      "CUDA_VISIBLE_DEVICES": "2,3,4,5,6,7"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18082,
+      "served-model-name": "qwen35-27b-aituner",
+      "trust-remote-code": true,
+      "dtype": "bfloat16",
+      "gpu-memory-utilization": 0.9,
+      "enable-prefix-caching": true,
+      "mamba-cache-mode": "light",
+      "distributed-executor-backend": "mp",
+      "block-size": 64,
+      "enable-chunked-prefill": true,
+      "max-num-batched-tokens": 8192,
+      "disable-cascade-attn": true,
+      "max-model-len": 262144,
+      "speculative-config": "{\"method\":\"qwen3_next_vl_mtp\",\"num_speculative_tokens\":3}",
+      "mm-processor-cache-gb": 0,
+      "limit-mm-per-prompt": "{\"image\":256,\"video\":64}",
+      "compilation-config": "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"use_inductor\":false,\"pass_config\":{\"fuse_norm_quant\":false,\"fuse_act_quant\":false,\"fuse_attn_quant\":false}}",
+      "mamba-cache-dtype": "float32",
+      "skip-mm-profiling": true,
+      "quantization": "fp8",
+      "tensor-parallel-size": 1,
+      "disable-log-requests": true
+    },
+    "tunable_envs": [
+      "VLLM_ENABLE_TORCH_COMPILE"
+    ],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "block-size",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1
+      ]
+    },
+    "python_executable": "python3"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 32,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 1.0,
+    "early_stop_max_lag_s": 120.0,
+    "early_stop_max_elapsed_s": 900.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "linear_ms",
+      "intercept_ms": 4000,
+      "per_token_ms": 0.125
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.125,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325
+  },
+  "llm": {
+    "system_prompt": "Propose a single engine config patch that increases the maximum feasible sampling_u under the SLO target. Favor launch-safe changes grounded in the incumbent result and only propose knobs that plausibly improve throughput above the incumbent request rate.",
+    "max_history_trials": 8,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.4",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 180
+    }
+  }
+}

configs/examples/dash0_qwen30b_a3b_stopA_fulldata.json

@@ -14,7 +14,7 @@
   "engine": {
     "engine_name": "vllm",
     "engine_version": "0.20.0",
-    "exec_path": "/tmp/wjh/venvs/vllm-0.20.0-cu129/bin/vllm",
+    "exec_path": "/usr/local/bin/vllm",
     "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
     "host": "127.0.0.1",
     "port": 18230,
@@ -33,7 +33,11 @@
     "base_flags": {
       "host": "127.0.0.1",
       "port": 18230,
-      "served-model-name": "qwen3-30b-a3b-community"
+      "served-model-name": "qwen3-30b-a3b-community",
+      "gpu-memory-utilization": 0.9,
+      "max-model-len": 16384,
+      "trust-remote-code": true,
+      "enable-prefix-caching": true
     },
     "tunable_envs": [],
     "tunable_flags": [
@@ -123,19 +127,12 @@
     "low": 0.0,
     "high": 0.125,
     "tolerance": 0.001,
-    "max_probes": 5,
+    "max_probes": 4,
     "sample_seed": 20260325
   },
   "llm": {
     "system_prompt": "Tune community vLLM 0.20.0 serving for Qwen3-30B-A3B. Start from the default vLLM engine configuration, use only launch-safe patches, and optimize request_rate_per_gpu under the configured SLO.",
     "max_history_trials": 8,
-    "use_harness": true,
-    "endpoint": {
-      "provider": "codex",
-      "model": "gpt-5.4",
-      "stream": true,
-      "api_key_env": "OPENAI_API_KEY",
-      "timeout_s": 240
-    }
+    "use_harness": false
   }
 }

configs/examples/dash0_qwen30b_a3b_stopA_on.json

@@ -0,0 +1,147 @@
+{
+  "study_id": "dash0-qwen30b-a3b-stopA-on-chat-0-8k",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "Qwen/Qwen3-30B-A3B",
+    "served_model_name": "qwen3-30b-a3b-community"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "0.20.0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18230,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 900,
+    "launch_args": [
+      "serve",
+      "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
+    ],
+    "base_envs": {
+      "CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7",
+      "HOME": "/tmp/wjh",
+      "XDG_CACHE_HOME": "/tmp/wjh/.cache"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18230,
+      "served-model-name": "qwen3-30b-a3b-community",
+      "gpu-memory-utilization": 0.9,
+      "max-model-len": 16384,
+      "trust-remote-code": true,
+      "enable-prefix-caching": true
+    },
+    "tunable_envs": [],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "enable-expert-parallel",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "block-size",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ]
+    },
+    "python_executable": "/tmp/wjh/venvs/vllm-0.20.0-cu129/bin/python"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "completion_tokens_override": 128,
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 64,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 1.0,
+    "early_stop_max_lag_s": 120.0,
+    "early_stop_max_elapsed_s": 900.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "step_ms",
+      "buckets": [
+        {
+          "max_input_tokens": 4096,
+          "threshold_ms": 2000
+        },
+        {
+          "max_input_tokens": 32768,
+          "threshold_ms": 4000
+        },
+        {
+          "threshold_ms": 6000
+        }
+      ]
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.125,
+    "tolerance": 0.001,
+    "max_probes": 4,
+    "sample_seed": 20260325
+  },
+  "llm": {
+    "system_prompt": "Tune community vLLM 0.20.0 serving for Qwen3-30B-A3B. Start from the default vLLM engine configuration, use only launch-safe patches, and optimize request_rate_per_gpu under the configured SLO.",
+    "max_history_trials": 8,
+    "use_harness": false
+  }
+}

configs/examples/dash0_qwen30b_a3b_stopB_e2e.json

@@ -0,0 +1,155 @@
+{
+  "study_id": "dash0-qwen30b-a3b-stopB-e2e-hi-chat-0-8k",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "Qwen/Qwen3-30B-A3B",
+    "served_model_name": "qwen3-30b-a3b-community"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "0.20.0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18230,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 900,
+    "launch_args": [
+      "serve",
+      "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
+    ],
+    "base_envs": {
+      "CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7",
+      "HOME": "/tmp/wjh",
+      "XDG_CACHE_HOME": "/tmp/wjh/.cache"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18230,
+      "served-model-name": "qwen3-30b-a3b-community",
+      "gpu-memory-utilization": 0.9,
+      "max-model-len": 16384,
+      "trust-remote-code": true,
+      "enable-prefix-caching": true
+    },
+    "tunable_envs": [],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "enable-expert-parallel",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "block-size",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ]
+    },
+    "python_executable": "/tmp/wjh/venvs/vllm-0.20.0-cu129/bin/python"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "completion_tokens_override": 128,
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 64,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "max_requests_per_probe": 512,
+    "replay_time_scale": 0.1,
+    "early_stop_max_lag_s": 120.0,
+    "early_stop_max_elapsed_s": 900.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "step_ms",
+      "buckets": [
+        {
+          "max_input_tokens": 4096,
+          "threshold_ms": 2000
+        },
+        {
+          "max_input_tokens": 32768,
+          "threshold_ms": 4000
+        },
+        {
+          "threshold_ms": 6000
+        }
+      ]
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 1.0,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325
+  },
+  "llm": {
+    "system_prompt": "Tune community vLLM 0.20.0 serving for Qwen3-30B-A3B. Start from the default vLLM engine configuration, use only launch-safe patches, and optimize request_rate_per_gpu under the configured SLO.",
+    "max_history_trials": 8,
+    "use_harness": true,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.4",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 240
+    }
+  }
+}

configs/examples/stopb_27b_ab/p1_tp1.json

@@ -0,0 +1,13 @@
+{
+  "observation": "baseline TP1 (deployed flags)",
+  "diagnosis": "deterministic TP A/B point",
+  "config_patch": {
+    "env_patch": {},
+    "flag_patch": {}
+  },
+  "expected_effects": [
+    "measure peak request_rate_per_gpu at this topology"
+  ],
+  "why_not_previous_failures": "n/a",
+  "should_stop": false
+}

configs/examples/stopb_27b_ab/p2_tp2.json

@@ -0,0 +1,15 @@
+{
+  "observation": "TP2",
+  "diagnosis": "deterministic TP A/B point",
+  "config_patch": {
+    "env_patch": {},
+    "flag_patch": {
+      "tensor-parallel-size": 2
+    }
+  },
+  "expected_effects": [
+    "measure peak request_rate_per_gpu at this topology"
+  ],
+  "why_not_previous_failures": "n/a",
+  "should_stop": false
+}

configs/examples/stopb_27b_ab/p3_tp4.json

@@ -0,0 +1,15 @@
+{
+  "observation": "TP4",
+  "diagnosis": "deterministic TP A/B point",
+  "config_patch": {
+    "env_patch": {},
+    "flag_patch": {
+      "tensor-parallel-size": 4
+    }
+  },
+  "expected_effects": [
+    "measure peak request_rate_per_gpu at this topology"
+  ],
+  "why_not_previous_failures": "n/a",
+  "should_stop": false
+}

configs/examples/tuning_report.example.json

@@ -0,0 +1,26 @@
+{
+  "report_id": "qwen27b-abl12-harness-vs-naive",
+  "output_root": "../../.aituner-reports/qwen27b-abl12-harness-vs-naive",
+  "target_fraction": 0.95,
+  "min_final_ratio": 0.98,
+  "cases": [
+    {
+      "case_id": "qwen27b-chat-0-8k-real-output",
+      "description": "12-trial harness-vs-naive ablation on the 0-8k chat window with real output lengths.",
+      "tags": ["qwen27b", "chat", "0-8k", "h20", "real-output"],
+      "budgets": [1, 2, 3, 4, 6, 8, 12],
+      "arms": [
+        {
+          "name": "harness",
+          "kind": "harness",
+          "study_root": "../../.aituner/abl12-harness/dash0-qwen27b-ablation-harness-on"
+        },
+        {
+          "name": "naive",
+          "kind": "naive",
+          "study_root": "../../.aituner/abl12-naive/dash0-qwen27b-ablation-naive-off"
+        }
+      ]
+    }
+  ]
+}

docs/action-aware-constraint-pilot-v0-protocol-20260714.md

@@ -0,0 +1,179 @@
+# Action-aware constraint pilot v0 protocol
+
+Status: **FROZEN BEFORE NEW GPU RUNS**.
+
+Date: 2026-07-14 (Asia/Singapore).
+
+## Headline question
+
+Can telemetry from one complete initial-config benchmark identify which of two
+competing knob families should be changed, before either target configuration
+is evaluated?
+
+This pilot tests a narrow prerequisite, not an end-to-end tuner claim.  It
+uses fields already present in the per-step OpProf stream to reconstruct exact
+zero-slack conditions for `max_num_seqs` (MNS) and
+`max_num_batched_tokens` (MBBT).  No new vLLM instrumentation is justified
+unless those action-conditioned conditions predict crossed real-system
+intervention responses.
+
+## Hypothesis
+
+I believe config-normalized scheduler constraints provide a stronger tuning
+signal than an aggregate queue symptom because the same waiting queue can be
+blocked by different admission limits.
+
+I will verify it by holding model, hardware, TP, request bands, arrival times,
+and offered load fixed while constructing two source configurations with
+different binding constraints.  From each source run alone, the larger
+exclusive binding fraction predicts the action family.  Both candidate
+actions are then measured on the same requests for the full 300-second replay.
+
+## Frozen platform and workload
+
+- Host: `dash0`, solo placement on GPUs 0-3, four NVIDIA H20 GPUs.
+- Model: Qwen3-30B-A3B BF16.
+- Engine: patched vLLM `0.24.1.dev3+opprof`, TP=4.
+- Workload: the three disjoint `mid` bands from
+  `chat_w20260312_1000`, 2.125 requests/s/GPU, 300-second arrival window,
+  exactly 128 output tokens.
+- SLO: the unchanged study TTFT/TPOT thresholds and 0.95 pass-rate target.
+- Every config starts one fresh server, performs the accepted 16-request
+  warm-up and the existing burn-in, then runs all three disjoint measured
+  bands in its frozen order.
+- SLO early stopping is disabled.  A measured run must drain all selected
+  requests and finish within the 450-second client deadline.
+
+## Frozen configuration and action matrix
+
+| ID | MNS | MBBT | Role |
+|---|---:|---:|---|
+| `b_base` | 64 | 256 | token-budget-bound source; operational gate runs first |
+| `a_base` | 16 | 8192 | MNS-bound source |
+| `shared` | 64 | 8192 | MNS action from A; MBBT action from B |
+| `b_mns` | 128 | 256 | competing MNS action from B |
+| `a_mbbt` | 16 | 16384 | competing MBBT action from A |
+
+The two decisions are therefore:
+
+```text
+Regime A: a_base -> {shared (increase MNS), a_mbbt (increase MBBT)}
+Regime B: b_base -> {b_mns (increase MNS), shared (increase MBBT)}
+```
+
+The candidate magnitudes are intentionally large in this feasibility pilot so
+that a missing crossed response is not explained by an imperceptibly small
+intervention.  This does not establish that these are production step sizes.
+
+Frozen config order is `b_base`, `a_base`, `shared`, `b_mns`, `a_mbbt`.
+Frozen repetition orders are respectively `123`, `231`, `312`, `132`, and
+`213`, reducing band/time alignment without reusing a server across configs.
+
+## Pre-action signal
+
+For each source run, let `waiting` include the normal and deferred waiting
+queues, and let `scheduled_tokens = prefill_tokens + decode_tokens`.
+
+```text
+mns_exclusive = waiting > 0
+                and running == configured MNS
+                and scheduled_tokens < configured MBBT
+
+mbbt_exclusive = waiting > 0
+                 and scheduled_tokens == configured MBBT
+                 and running < configured MNS
+
+both = waiting > 0
+       and running == configured MNS
+       and scheduled_tokens == configured MBBT
+```
+
+Each score is the fraction of all scheduler records in the measured interval
+that satisfies the condition.  The predicted action is the family with the
+larger exclusive fraction.  This uses no target telemetry or target outcome.
+KV usage and preemptions are reported as possible alternative constraints but
+are not silently reassigned to either score.
+
+These conditions reproduce two scheduler loop boundaries, but they are still
+a Level-0 proxy: they do not expose the exact request rejected at the boundary
+or run a shadow schedule.  The pilot explicitly tests whether that additional
+engine patch is warranted.
+
+## Outcomes and baselines
+
+Primary intervention outcome:
+
+```text
+SLO-goodput = full-run SLO pass count / 300-second arrival window
+```
+
+Also report pass rate, TTFT p50/p95/p99, TPOT p50/p95/p99, drain elapsed time,
+KV usage, preemptions, queue area, and CUDA-graph padding.
+
+Required decision baselines:
+
+1. always choose the MNS family;
+2. always choose the MBBT family;
+3. queue-pressure-only, which has no candidate-specific score and therefore
+   must use one frozen family for both regimes;
+4. the pre-action exclusive-binding prediction.
+
+This is a mechanism ablation.  It does not compare against a trained black-box
+tuner because two regimes are not a valid training surface.
+
+## Gates and failure meanings
+
+Data validity requires 15 uncensored measured runs, exact request/arrival/input
+hashes across each repetition, full request accounting, one continuous OpProf
+stream per config, zero dropped records, monotonic timestamps and step indices,
+nonnegative counters, bounded ratios, clean GPU placement, and config values in
+the result matching the server command.
+
+The crossed-response gate passes only if, in all three repetitions:
+
+- the MNS target has higher SLO-goodput than the MBBT target in Regime A;
+- the MBBT target has higher SLO-goodput than the MNS target in Regime B;
+- each winning target exceeds its competing target by at least 10% of the
+  source SLO-goodput.  A source with zero goodput makes the run invalid for
+  this relative gate rather than changing the denominator.
+
+The binding gate passes only if, in both regimes:
+
+- the predicted family matches the measured winning family in all three
+  repetitions;
+- the median winning-family exclusive fraction is at least 0.10;
+- it is at least 5x the median competing-family exclusive fraction;
+- the direction is unchanged under cumulative 25%, 50%, 75%, and 100%
+  checkpoints after the 25% checkpoint.
+
+Decision meanings:
+
+- `STOP_WORKLOAD_NOT_CROSSED`: candidate outcomes do not have different
+  winners; the experiment cannot test action selection.
+- `STOP_BINDING_NOT_PREDICTIVE`: outcomes cross but source-only constraint
+  scores do not select them; do not implement shadow scheduling from this
+  hypothesis.
+- `STOP_NO_NEW_INSTRUMENTATION_NEEDED`: the signal works but every required
+  field was already present; keep it as an analysis/tuner feature and do not
+  claim a new engine-instrumentation contribution.
+- `OPEN_EXACT_ATTRIBUTION_ABLATION`: the signal works but unresolved/both/KV
+  cases are material enough that exact rejection reasons could change a
+  decision.  Only this result authorizes a minimal vLLM attribution patch.
+
+Ambiguity is material only when, in either regime, the median
+`both + waiting_unresolved` fraction is at least the median absolute gap
+between the two exclusive fractions, or when any source run records a
+preemption or median source KV maximum is at least 0.90.  Otherwise all fields
+needed for the observed decision were already present and the result is
+`STOP_NO_NEW_INSTRUMENTATION_NEEDED`.
+
+No result from this development pilot is a paper-level E2E tuning claim.
+
+## Cost and stopping discipline
+
+- Hard cap: 8.0 H20-hours, including failed sessions.
+- Expected: 6.0-7.2 H20-hours and 90-110 minutes wall time.
+- `b_base` runs first.  If its first measured band cannot drain by 450 seconds,
+  the controller stops before any comparative analysis; MBBT=256 is then an
+  operationally invalid source, not negative evidence.
+- Any data red flag stops analysis before computing a tuning conclusion.

docs/action-aware-constraint-pilot-v1-protocol-20260714.md

@@ -0,0 +1,44 @@
+# Action-aware constraint pilot v1 amendment
+
+Status: **FROZEN AFTER V0 OPERATIONAL STOP AND BEFORE V1 GPU RUNS**.
+
+Date: 2026-07-14 (Asia/Singapore).
+
+The complete claim, workload, baselines, metrics, action matrix, analysis gates,
+and data-validity requirements remain those in the v0 protocol.  This amendment
+changes only the token-bound source severity and adds an operational burn-in
+gate.
+
+## Why v0 produced no comparative evidence
+
+The first v0 session used MNS=64 and MBBT=256.  During the 510-request,
+60-second burn-in, the client had run for 197 seconds and the engine still held
+64 requests: 13 running and 51 waiting.  The last step scheduled exactly 256
+tokens, KV usage was 0.01151, and there were zero preemptions.  No measured run
+or target configuration had started.
+
+The session was stopped and cleanly released all GPUs after consuming
+0.3859868995 H20-hours.  This is evidence that MBBT=256 is a real token-budget
+bottleneck, but it is not an admissible source for the 2.125 requests/s/GPU
+comparison because it cannot sustain the offered load.  V0 contributes no
+tuning label and none of its runtime data is reused by V1.
+
+Authoritative failure artifact:
+`/home/admin/cpfs/wjh/action-aware-constraint-v0-20260714/operational-stop-v0.json`.
+
+## V1 changes
+
+- `b_base`: MNS=64, MBBT **2048** instead of 256.
+- `b_mns`: MNS=128, MBBT **2048** instead of 256.
+- The B-family MBBT action is therefore 2048 -> 8192.
+- All five configurations and all three repetitions run fresh under a new V1
+  run root.
+- Before any measured run, every config's 510-request/60-second burn-in must
+  drain in at most **90 seconds**.  A slower config is an operational failure;
+  the controller stops before comparative analysis.
+- V1 incremental hard cap is 7.6140131005 H20-hours so that V0 plus V1 remains
+  within the original global 8.0 H20-hour cap.
+
+The V0 protocol's crossed-response and source-only binding gates are unchanged.
+In particular, V1 still requires three-of-three action-family predictions in
+both regimes and a different real winner in Regime A versus Regime B.

docs/action-aware-constraint-pilot-v2-protocol-20260714.md

@@ -0,0 +1,35 @@
+# Action-aware constraint pilot v2 amendment
+
+Status: **FROZEN AFTER V1 CONTROLLER STOP AND BEFORE V2 GPU RUNS**.
+
+Date: 2026-07-14 (Asia/Singapore).
+
+The V1 configuration matrix and every scientific gate remain unchanged.  V2
+fixes one controller bug and reruns every configuration and repetition fresh.
+
+## V1 controller failure
+
+The MNS64/MBBT2048 burn-in completed all 510 requests in 61.259 seconds, below
+the frozen 90-second operational limit.  However, the controller accidentally
+assigned the preceding 16-request warm-up result to `burnin_result`; its state
+therefore recorded 4.376 seconds and evaluated the wrong object.
+
+The first measured replay was terminated after 75 seconds, before it produced
+a result.  No target configuration had started.  V1 consumed
+0.3184109431 H20-hours and contributes no action label or telemetry to V2.
+
+## V2 correction and regression gate
+
+- The completed burn-in `run_client()` return value is assigned to
+  `burnin_result`.
+- A dedicated `burnin_gate()` rejects any non-anchor object, any request count
+  other than 510, and elapsed time above 90 seconds.
+- Unit tests explicitly pass a warm-up object and require rejection, then test
+  accepted and over-limit burn-ins.
+- All five configs and 15 measured runs use a new run root; no V0/V1 runtime
+  artifact is reused.
+
+V0 and V1 together consumed 0.7043978426 H20-hours.  V2's incremental hard cap
+is therefore 7.2956021574 H20-hours, preserving the original global 8.0
+H20-hour cap.  The authoritative accounting file is
+`runs/action-aware-v0/prior-attempts-v2.json`.

docs/action-aware-constraint-pilot-v2-results-20260714.md

@@ -0,0 +1,260 @@
+# Action-aware constraint pilot v2 results
+
+Date: 2026-07-14 (Asia/Singapore).
+
+Decision: **`STOP_WORKLOAD_NOT_CROSSED`**.
+
+The pilot produced one valid positive regime and one invalid-for-effect-size
+regime.  It supports continuing a narrower action-response investigation, but
+it does not justify an end-to-end telemetry-guided tuner claim or new engine
+instrumentation yet.
+
+## Question tested
+
+Given only a completed source run, can existing engine telemetry distinguish
+which of two one-knob interventions will improve SLO-goodput more?
+
+The frozen score counted scheduler steps with backlog where either MNS or MBBT
+was exclusively at its configured limit.  It made two pre-intervention
+predictions on the same workload and offered load:
+
+- Regime A: source `(MNS=16, MBBT=8192)` predicts increasing MNS to 64 over
+  increasing MBBT to 16384.
+- Regime B: source `(MNS=64, MBBT=2048)` predicts increasing MBBT to 8192 over
+  increasing MNS to 128.
+
+The primary outcome was 300-second SLO-goodput.  A predicted action had to beat
+the alternative on every paired request band by at least 10% of that band's
+source goodput.  Telemetry direction also had to remain stable at 25%, 50%,
+75%, and 100% of the replay.
+
+## Setup
+
+- Host: `dash0`, GPU 0-3 used exclusively; four NVIDIA H20 GPUs; TP=4.  GPU
+  4-7 remained idle to avoid co-location effects.
+- Model: Qwen3-30B-A3B BF16 at
+  `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`.
+- Runtime: patched vLLM `0.24.1.dev3+g668cfb7e2`, source commit `4b253fd`, with
+  OpProf Layer-1 telemetry.
+- Workload: `chat_w20260312_1000`, 2.125 requests/s/GPU, 300-second arrival
+  window, 128 output tokens, three disjoint request bands.
+- Five fresh-server configurations, three measured runs each, counter-rotated
+  repetition order, 16-request warm-up, and a 510-request/60-second burn-in.
+  SLO early stopping was disabled.
+- Exact request-id, arrival-order, and input-length hashes matched for every
+  paired comparison.
+
+The authoritative run root is
+`/home/admin/cpfs/wjh/action-aware-constraint-v2-20260714`.  The final audit is
+`pilot-audit-final.json`, SHA256
+`7ebe080fcc4970bef423bc587253d157e75aed1ea8b410bd37770c17708135ab`.
+
+## End-to-end result
+
+### Regime A: strong MNS constraint
+
+| Rep | Source 16/8192 | MBBT action 16/16384 | MNS action 64/8192 | MNS-only source steps | MBBT-only source steps | `(MNS action - MBBT action) / source` |
+|---:|---:|---:|---:|---:|---:|---:|
+| 1 | 4.710 | 7.687 | 8.500 | 79.85% | 0.038% | 17.27% |
+| 2 | 2.803 | 4.150 | 8.500 | 94.61% | 0.005% | 155.17% |
+| 3 | 2.227 | 3.943 | 8.500 | 79.87% | 0.005% | 204.64% |
+
+Units are SLO-goodput requests/s except the percentage columns.  The source
+prediction was stable at every phase checkpoint and correct in all three
+paired bands.  The predicted MNS action cleared the frozen 10% material-margin
+gate in all three bands.
+
+This is a real but limited positive result.  The source was an extreme case:
+MNS was full on nearly every scheduler step that retained backlog, TTFT p50 was
+1.64-5.61 seconds, KV usage remained below 2.45%, and no preemption occurred.
+An expert or a simple rule could identify this case without a learned tuner.
+
+### Regime B: MBBT direction at an outcome ceiling
+
+| Rep | Source 64/2048 | MNS action 128/2048 | MBBT action 64/8192 | MBBT-only source steps | `(MBBT action - MNS action) / source` |
+|---:|---:|---:|---:|---:|
+| 1 | 8.423 | 8.420 | 8.500 | 10.90% | 0.950% |
+| 2 | 8.447 | 8.500 | 8.500 | 12.03% | 0% |
+| 3 | 8.500 | 8.497 | 8.500 | 8.79% | 0.039% |
+
+The telemetry direction was stable in all phase checkpoints.  The predicted
+action won twice and tied once, while the wrong MNS action left the MBBT-only
+state intact.  However, the source already delivered 99.10-100% of the offered
+8.5 requests/s.  Even a perfect action could not reach the preregistered 10%
+margin.  Regime B therefore does not test material weak-signal value; it is a
+workload-selection failure, not evidence that telemetry does or does not help
+near a decision boundary.
+
+The missing preflight condition is mathematical.  For an effect threshold
+`delta` and offered goodput ceiling `G`, the source must satisfy
+`source <= G / (1 + delta)`.  Here `G=8.5` and `delta=0.10`, so any source above
+7.727 requests/s cannot possibly pass before either target is measured.
+
+## Why the exclusive-limit rule is incomplete
+
+The alternative MBBT action improved Regime A by 48.0%, 63.2%, and 77.1% over
+the source even though MBBT was almost never the exclusive backlog constraint.
+This rules out the binary interpretation "a knob that is not exclusively at
+its cap cannot help."
+
+Existing richer telemetry provides a plausible mechanism:
+
+| Rep | Split-prefill requests, source -> MBBT action | Prefill steps, source -> MBBT action | Prefill requests/step, source -> MBBT action | Prefix-hit rate, source -> MBBT action |
+|---:|---:|---:|---:|---:|
+| 1 | 41 -> 2 | 2324 -> 2022 | 1.071 -> 1.249 | 13.851% -> 13.747% |
+| 2 | 7 -> 0 | 2410 -> 2291 | 1.012 -> 1.076 | 13.078% -> 12.988% |
+| 3 | 12 -> 1 | 2377 -> 2270 | 1.018 -> 1.096 | 13.613% -> 13.604% |
+
+Increasing MBBT allows more prefill work to be packed into one iteration and
+nearly eliminates split prefills.  Under MNS=16, this can reduce the number of
+iterations for which long prompts occupy scarce running slots.  Prefix-cache
+hit rates differ by at most 0.104 percentage points, and exact workload hashes
+match, so neither explains the gain.  Step-duration p99 also remains similar;
+one 1.127-second decode-step outlier appears in `a_mbbt/rep2`, but the same
+action direction occurs in all three bands.
+
+This is a mechanism-consistent explanation, not a completed causal
+decomposition.  MBBT simultaneously changes total per-iteration token budget,
+per-request chunk size, and multi-request packing.  Instrumentation observes
+their joint response but cannot separate those effects without another
+intervention.
+
+## Instrumentation decision
+
+Do **not** add a new engine patch for this mechanism yet.  The existing OpProf
+stream already records submit/complete timestamps, prefill/decode composition,
+chunked-prefill categories, prefix hits, queues, KV usage, and CUDA graph mode.
+Those fields are sufficient to identify the interaction missed by the initial
+exclusive-limit rule.
+
+The next narrow mechanism ablation is available in the current vLLM runtime:
+
+1. `(MNS=16, MBBT=8192, long-prefill-threshold=0)` is the current source.
+2. `(MNS=16, MBBT=16384, long-prefill-threshold=8192)` keeps individual long
+   chunks at 8192 while increasing total packing budget.
+3. `(MNS=16, MBBT=16384, long-prefill-threshold=0)` is the current MBBT action.
+
+The runtime exposes `--long-prefill-token-threshold`; with a threshold of 8192,
+the second arm separates total packing headroom from the larger per-request
+chunk allowed by the third arm.  A formal test must rerun all three arms fresh
+with counter-rotated order rather than reuse today's endpoints.
+
+## Correct tuning-research route
+
+The pilot does not support turning the frozen equality checks into a larger
+rule tree.  The supported route is **intervention-calibrated, action-conditioned
+system identification**:
+
+```text
+source event sequence + normalized config delta
+    -> predicted distribution of Delta SLO-goodput and evaluation cost
+    -> uncertainty-aware next-config selection
+```
+
+The policy input should retain continuous distributions and phase evolution:
+queue/running residency, MNS and token slack, prefill/decode composition,
+partial-prefill occupancy, step time, KV state, and graph behavior.  Human
+bottleneck labels and hand-authored `if queue then increase MNS` mappings are
+not policy inputs.  Mechanism summaries remain audit and interpretation tools;
+the action response is learned from paired real interventions.
+
+The harness has a narrower, non-heuristic role:
+
+- define legal configurations and exact paired workloads;
+- reject source points without outcome headroom before a full sweep;
+- randomize/counter-rotate execution order and preserve failures/cost;
+- validate stream coverage, hashes, request accounting, and censoring;
+- expose target outcomes only after a source-only prediction is frozen;
+- evaluate fixed-budget regret and H20-hours, not explanation quality alone.
+
+The next tuning experiment should use non-extreme, non-ceiling source points
+and a local two-dimensional MNS/MBBT neighborhood.  A short run may screen load
+only; every inferential telemetry and outcome result remains a 300-second run.
+At least one held-out workload must be reserved before choosing features or
+thresholds.
+
+Primary evaluation is H20-hours/trials to reach 95% of the real local oracle
+and cost-normalized regret AUC.  Required baselines are random search,
+config/outcome-only sequential search, the current rule heuristic, and the
+same action-response model with telemetry removed.  Action-ranking accuracy is
+supporting evidence only.
+
+## What this pilot establishes and does not establish
+
+Established:
+
+- Long-window engine state can make a correct, phase-stable action-family
+  prediction in an extreme MNS-constrained regime.
+- A naive `queue > 0 -> increase MNS` rule would choose an ineffective action
+  in Regime B; action-conditioned state distinguishes the mechanism direction,
+  although the measured effect is immaterial at the selected load.
+- Binary exclusive-cap attribution misses a substantial MNS/MBBT interaction;
+  existing chunk/step telemetry reveals a plausible explanation.
+- Source outcome headroom must be an explicit experiment admission gate.
+
+Not established:
+
+- telemetry improves an end-to-end tuner over an outcome-only baseline;
+- weak or mixed constraints can be ranked with material gain;
+- the response transfers across workloads, models, TP, or hardware;
+- new engine instrumentation is necessary;
+- the chunking/packing breakdown is causal rather than mechanism-consistent.
+
+## Change and verification
+
+Reproduction:
+
+```bash
+python3 runs/action-aware-v0/test_pilot.py
+
+python3 runs/action-aware-v0/analyze_pilot.py \
+  --run-root /home/admin/cpfs/wjh/action-aware-constraint-v2-20260714/runs/pilot \
+  --manifest runs/action-aware-v0/pilot-manifest-v2.json \
+  --output /home/admin/cpfs/wjh/action-aware-constraint-v2-20260714/pilot-audit-final.json
+```
+
+The fresh GPU run used AITuner commit `c5ab073`; asynchronous coverage was
+corrected in `3facb18`; reproducible mechanism summaries were added in
+`2af22db`.  The raw run is unchanged across those analyzer-only commits.
+
+Change: added a crossed real-intervention controller and audit, fixed the
+burn-in result gate, corrected asynchronous per-step coverage accounting, and
+added reproducible step/chunk/prefix mechanism summaries.
+
+Expected effect: distinguish descriptive telemetry from source-only action
+predictions that survive paired real interventions.
+
+Verification: local and remote action-aware test suites pass; all five sessions
+completed; all stream/footer and request-accounting invariants pass; the final
+analyzer was run twice and produced byte-identical output.
+
+Result: Regime A passes; Regime B is invalid for the frozen effect-size test;
+the global decision is `STOP_WORKLOAD_NOT_CROSSED`.
+
+Remaining risk: one model, one TP, one trace family, three bands, two action
+families, and deliberately constructed endpoints are development evidence
+only.  The strong positive regime is too obvious to support a paper claim.
+
+## Data sanity
+
+- Measured runs: n=15; elapsed 300.610-317.350 seconds; 15 distinct.  Pass
+  rate min/max 0.2620/1.0 with 11 distinct values; SLO-goodput min/max
+  2.2267/8.5 requests/s with 11 distinct values.
+- Telemetry intervals: n=15; records min/max 13,621/23,711; 15 distinct.
+  Start gaps min/max 0.0412/0.1227 seconds; end gaps 0.00053/0.0649; uncovered
+  internal gaps 0/0.3231.  One submit gap reached 1.1190 seconds but was fully
+  covered by a 1.1269-second recorded execution; contiguous indices and zero
+  drops were preserved.
+- Sessions: n=5; cost min/max 1.1691/1.2730 H20-hours; 5 distinct.  V2 cost
+  was 6.0862 H20-hours; V0/V1/V2 total was 6.7906, below the 8.0 cap.
+- Regime-A split-prefill observations: n=6; min/max 0/41 requests; 6 distinct.
+  Prefix-hit rates: n=6; min/max 0.12988/0.13851; 6 distinct.
+- Checked invariants: non-negative counters and durations; ratios in `[0,1]`;
+  exact request, arrival, and length hashes; 2550/2550 request accounting per
+  measured run; uncensored outcomes; outcomes across configurations not all
+  identical;
+  five complete streams; monotonic timestamps; contiguous step indices; zero
+  drops; footer/sidecar agreement; chunk-token accounting; bounded prefix hits;
+  no OOM, controller error, or residual GPU allocation.  No unresolved red
+  flag remains.  The three identical shared goodputs are reported as the
+  offered-load ceiling, not treated as independent performance variation.

docs/active-intervention-v0-protocol-20260715.md

@@ -0,0 +1,136 @@
+# Active intervention + measurement v0 protocol
+
+Date: 2026-07-15 (Asia/Singapore)
+
+Status: **FROZEN BEFORE THE `chat_w20260313_1000` GPU RUN**.
+
+## Research question
+
+This experiment asks whether a tuner conditioned on direct engine-state
+trajectories can choose both a measurement horizon and a coupled configuration
+intervention with lower real-GPU cost than the same tuner using only external
+prefix outcomes.
+
+The contribution is not the controller, legality checks, telemetry collection,
+or the ridge model.  The route remains open only if engine state changes an
+actual decision and reduces cost-to-near-oracle on unseen workloads.
+
+## Development result that motivates, but does not pass, the route
+
+The frozen trace-12 dataset contains 72 examples: six source decisions, four
+measurement checkpoints, and `noop/MNS/MBBT` actions.  Features are direct
+continuous Layer-1 state summaries; cap-exclusive and bottleneck labels are
+excluded.  Leave-one-repetition-out sequential replay uses the same model,
+candidate set, confidence rule, and checkpoint set for both modes.
+
+The external-outcome policy and telemetry policy both put all six decisions
+within 2% regret.  Outcome-only selected a mean 262.5-second source measurement
+and cost 3.750 replay H20-hours across the six replayed decisions; telemetry
+selected 275 seconds and cost 3.833 H20-hours.  Telemetry therefore increased
+the replay lower-bound cost by 2.22%, with no regret reduction.  This is a
+negative result.  It does not settle the question because the dataset has only
+two source regimes, one source is at the offered ceiling, and there is no joint
+MNS+MBBT action.
+
+Sanity: n=6 decisions; regret min=0, max=0.009412, distinct=3; source cutoff
+min=150s, max=300s, distinct=3 across the two policies; all costs are
+non-negative, regrets are in `[0,1]`, target results are not all identical, and
+the six decisions are complete exact-workload pairs.
+
+## Frozen prospective setup
+
+- Host: `dash0`, 8 NVIDIA H20 GPUs available; each TP4 server runs alone on
+  GPUs 0-3.  Co-location is prohibited for SLO verdicts.
+- Engine: patched vLLM `0.24.1.dev3+g668cfb7e2` from clean source commit
+  `4b253fd8619764b6971a7f2e3a3aa7545f6ace05` at
+  `/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0`, using
+  `/tmp/wjh-opprof-phase2-dash0-20260711/.venv`.
+- Model: `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`, BF16.
+- Workload: unseen `chat_w20260313_1000`; input 0-8192; output exactly 128;
+  replay scale 0.5; 300-second arrival window.
+- Three disjoint repetitions: source rows are assigned by a deterministic
+  SHA-256 modulo-3 partition before input filtering.  Each repetition selects
+  approximately 3300 requests, or 2.75 requests/s/GPU at TP4.
+- SLO: at least 95% pass; stepped TTFT 2/4/6 seconds; TPOT at most 50 ms.
+- Checkpoints: 75, 150, 225, and 300 seconds.
+- Full 2x2 surface:
+  - source: `MNS=32, MBBT=4096`;
+  - MNS action: `64,4096`;
+  - MBBT action: `32,8192`;
+  - joint action: `64,8192`;
+  - `noop` retains the source.
+- Four config sessions are serialized.  Each session uses a fresh server,
+  warm-up, burn-in, and counter-rotated repetition order.
+- Expected campaign cost: 4.6-5.5 H20-hours; hard cap: 6.0 H20-hours;
+  expected wall time: 75-100 minutes.
+
+The source is executed first.  The frozen telemetry policy selects the next
+real config session; all remaining cells are then measured only to construct
+the exact finite-surface oracle.  Oracle annotation after the selected action
+is reported separately from tuner cost.
+
+## Frozen policies
+
+Both policies fit the paired treatment effect
+
+```text
+target normalized SLO-goodput - source normalized SLO-goodput
+```
+
+from source config, full config delta, offered load, and external prefix
+outcomes.  The telemetry policy additionally receives fixed direct Layer-1
+summaries and their interactions with `delta_log2(MNS)` and
+`delta_log2(MBBT)`.  It does not receive a bottleneck label or a
+diagnosis-to-knob rule.
+
+At each checkpoint, jackknife models produce an effect distribution for
+`noop`, MNS, MBBT, and joint actions.  Measurement stops at the earliest second
+consecutive checkpoint with the same confident best action; otherwise it uses
+the full 300 seconds.  Confidence requires a predicted margin of at least 0.02
+and the best lower bound to exceed the second-best upper bound.  If the final
+choice is not confident, the next run is the positive-UCB action, explicitly
+marked as a diagnostic intervention.  The exact same rule is used for the
+outcome-only baseline.
+
+## Hypotheses and gates
+
+### H1: action value
+
+Engine state must change the selected intervention or its ranking and reduce
+real action regret.  Prediction error or bottleneck-label accuracy is not a
+success metric.
+
+### H2: measurement value
+
+Engine state must select a shorter stable source measurement without increasing
+action regret.  A shorter reconstructed prefix is only a trigger; it is not an
+actual GPU-cost claim until an early-terminated confirmation run measures
+startup, warm-up, drain, and cleanup.
+
+### H3: end-to-end cost
+
+Primary development metric is H20-hours to first reach a configuration within
+2% of the exact median-goodput oracle.  The outcome-only and telemetry policies
+use the same measured config costs and differ only in source information.
+
+- At least 10% prospective replay cost reduction, telemetry regret at most 2%,
+  and no outcome-only-to-telemetry harm triggers an actual early-stop
+  confirmation.
+- At least 20% measured all-in H20-hour reduction is required for a contribution
+  claim.  This one task can only establish development feasibility; a paper
+  claim additionally requires task-held-out replication.
+- Source median normalized goodput at or above 0.98 stops the surface before
+  target runs because the workload has no material improvement headroom.
+- Any hash mismatch, missing/censored result, telemetry drop, non-monotonic
+  phase, negative cost, ratio outside `[0,1]`, or all-identical config outcomes
+  is a red flag and stops analysis.
+
+If the 10% trigger fails, this route is closed for the current engine-state
+representation.  The experimental control plane is not retained as a fallback
+research contribution.
+
+Pre-run provenance amendment: the first controller dry-run on 2026-07-15
+rejected two stale engine paths before starting a server.  The paths and exact
+runtime version above were recovered from the accepted trace-12 campaign and
+corrected before any trace-13 GPU work.  No scientific treatment or gate was
+changed.

docs/active-intervention-v0-results-20260715.md

@@ -0,0 +1,106 @@
+# Active intervention v0: held-out trace-13 result
+
+Date: 2026-07-15 (Asia/Singapore)
+
+Decision: **close the passive-telemetry treatment-effect route**.  The held-out
+campaign produced no telemetry-induced action change, measurement reduction, or
+GPU-cost reduction.  It did show that the engine state contained the correct
+action-specific mechanism; the current feature model failed to use it.
+
+## Headline result
+
+The outcome-only and telemetry policies both measured the source for 300
+seconds, selected `joint=(MNS64,MBBT8192)`, and produced the same complete
+acquisition order.  Both reached the exact finite-surface oracle after the
+first intervention at a reconstructed all-in lower-bound cost of 2.4284
+H20-hours.  Telemetry GPU-cost reduction was therefore exactly 0%, below the
+10% confirmation trigger and 20% contribution gate.  No actual early-stop
+confirmation was launched.
+
+The complete annotation campaign cost 5.0379 H20-hours, below the 6.0 H20-hour
+hard cap.  It ran 12 uncensored real-GPU outcomes: four configs, three disjoint
+request partitions, and a fresh server per config.
+
+## Exact response surface
+
+Median normalized SLO-goodput was:
+
+| Config | Rep values | Median |
+|---|---|---:|
+| `MNS32,MBBT4096` source | 0.40091 / 0.39788 / 0.42061 | 0.40091 |
+| `MNS64,MBBT4096` | 1.00000 / 0.99970 / 1.00000 | 1.00000 |
+| `MNS32,MBBT8192` | 0.44394 / 0.41515 / 0.42606 | 0.42606 |
+| `MNS64,MBBT8192` joint | 1.00000 / 1.00000 / 1.00000 | 1.00000 |
+
+Increasing MNS alone was sufficient and joint was redundant.  Increasing MBBT
+alone improved the median by only 0.02515, versus 0.59909 for MNS.  This is a
+strong non-additive action response, not a setting where independently tuning
+the knobs and merging their improvements is valid.
+
+## What the telemetry actually said
+
+Across 41,086 source scheduler records, 93.12% of steps had waiting work,
+85.36% were MNS-exclusive binding, 1.11% were MBBT-exclusive, mean running-slot
+utilization was 97.39%, mean token-budget utilization was 15.69%, mean KV usage
+was 2.75%, and there were no preemptions.
+
+The intervention transition agreed with that state:
+
+| Config | Waiting | MNS-exclusive | MBBT-exclusive | Median goodput |
+|---|---:|---:|---:|---:|
+| source | 93.12% | 85.36% | 1.11% | 0.40091 |
+| MNS only | 5.38% | 0% | 5.38% | 1.00000 |
+| MBBT only | 91.19% | 91.09% | 0.04% | 0.42606 |
+| joint | 0.89% | 0% | 0.89% | 1.00000 |
+
+Thus this experiment does **not** support the claim that engine telemetry lacks
+tuning information.  It rejects the narrower claim that adding passive state
+summaries to the current small-data ridge policy converts that information into
+lower tuning cost.
+
+## Why the learned policy failed
+
+At 300 seconds, the telemetry model predicted joint, MNS, and MBBT effects of
+0.35190, 0.26118, and 0.09686.  The actual median effects were 0.59909,
+0.59909, and 0.02515.  Telemetry therefore made the nonexistent joint-over-MNS
+gap larger: 0.09072 predicted versus 0 actual; the outcome-only model predicted
+0.03188.
+
+The failure has three concrete causes:
+
+1. The six training decisions contain no joint intervention.  The
+   `delta_product` feature has no support, so joint ranking is extrapolation.
+2. Passive raw summaries do not represent the counterfactual scheduler work
+   unlocked by each action.  Capacity-normalized MNS pressure was visible, but
+   the model was not structurally required to map it to MNS marginal value.
+3. The policy maximizes predicted effect.  It does not identify the smallest
+   epsilon-optimal intervention or price unsupported action complexity.
+
+## Research implication
+
+Do not retain the harness or the passive telemetry model as a contribution.
+The next defensible route is engine-native, action-conditional counterfactual
+instrumentation: at a real scheduling state, shadow-replay the exact scheduler
+decision under an MNS relaxation, MBBT relaxation, and their joint relaxation,
+then expose the incremental queued work admitted by each action.  Real paired
+interventions calibrate how those one-step shadow effects map to E2E SLO
+goodput.  This is distinct from a hand-written cap-to-knob rule and from a
+full-system simulator: it reuses the exact live queue, scheduler, and cache
+state while simulating only the local decision boundary.
+
+That route should be evaluated against outcome-only search, the present passive
+telemetry model, a cap-hit expert rule, and a full simulator.  The paper-level
+gate remains at least 20% measured H20-hour reduction to a 2%-oracle config on
+task-held-out workloads with at most 2% regret.
+
+## Sanity
+
+Surface outcomes: n=12, min=0.39788, max=1.0, distinct=8.  Session costs: n=4,
+min=1.1702, max=1.3566 H20-hours, distinct=4.  Scheduler-record counts: n=4,
+min=37,001, max=41,348, distinct=4.  All counters and costs were non-negative;
+all ratios were in `[0,1]`; request hashes matched; all 12 runs were uncensored;
+the controller and four sessions completed; and config outcomes were not all
+identical.  No red flags were found.
+
+Machine-readable summary: `runs/active-intervention-v0/trace13-results.json`.
+Raw immutable root: `/home/admin/cpfs/wjh/active-intervention-prospective-20260715`.

docs/aituner-harness-design-contract.md

@@ -0,0 +1,333 @@
+# AITuner Harness Design Contract
+
+本文总结当前 AITuner harness 的设计语义。它不是实验流水账，也不是最终论文文字；
+它的作用是把我们能 claim 的系统贡献、各模块做法、隐含假设和限制说清楚。
+
+核心结论：
+
+```text
+AITuner harness 的贡献不是“LLM 会调参”，也不是“写了一组专家 if/else 规则”。
+
+Harness 的目标是把 black-box knob search 转成：
+measurement-grounded, mechanism-guided, validator-controlled experiments。
+```
+
+换句话说，planner 可以是 LLM、BO、bandit、deterministic heuristic 或人工选择。
+Harness 负责把观测转换成可审计的机制假设，生成合法候选，并用真实测量验证或否定这些假设。
+
+## 核心状态机
+
+```mermaid
+flowchart LR
+    S[State<br/>workload, constraints, history] --> E[Evidence<br/>SLO symptoms to mechanism signals]
+    E --> C[CandidateSet<br/>typed interventions]
+    C --> V[Validator<br/>legal, novel, covered?]
+    V -->|run trial| M[Measurement<br/>verdict]
+    M --> S
+    V -->|no justified candidate| X[Stop / report]
+```
+
+Harness 的核心循环只有五步：
+
+1. **State**：维护 workload、SLO、engine/hardware constraints 和历史 trial measurement。
+2. **Evidence**：把 probe 结果从 raw logs 转成 serving-stage symptom signals。
+3. **CandidateSet**：在 mechanism space 中生成有限个 typed interventions。
+4. **Validator**：检查 legality、full-config novelty、failure memory 和 coverage。
+5. **Measurement**：执行被验证过的 intervention，用真实 SLO verdict 更新状态；若没有
+   justified candidate，则 stop 或报告 measurement/coverage gap。
+
+这个状态机表达的是 harness 的最小设计，不依赖具体 planner。LLM、BO、bandit 或
+deterministic heuristic 都只能在 `CandidateSet` 上排序或选择，不能绕过 `Validator`
+直接构造 config，也不能单方面决定 stop。
+
+## 核心设计不变量
+
+后续所有低层模块都服务于三个不变量：
+
+| 不变量 | 含义 | 为什么重要 |
+| --- | --- | --- |
+| Measurement-grounded | 每个状态转移都由真实 probe/SLO verdict 更新 | 防止 planner 把自然语言猜测当成事实 |
+| Mechanism-typed | 候选不是裸 knob vector，而是 topology/scheduler/admission/cache 等 intervention | 降低搜索维度，并让每个 trial 有可解释假设 |
+| Validator-controlled | candidate 和 stop 必须通过 legality、no-repeat、coverage 和 failure guards | 防止重复实验、非法配置和 premature stop |
+
+## 从 High Level 到 Low Level 的展开
+
+下面各节按实现层次展开：
+
+1. Observation schema 定义 harness 能看到什么；
+2. Evidence compiler 说明 symptom 如何变成机制证据；
+3. Mechanism space 说明候选空间从哪里来；
+4. CandidateSet 说明如何构造 intervention；
+5. Planner interface 说明 LLM/BO/heuristic 的边界；
+6. Validator 说明什么能执行、什么能停止。
+
+每一层都区分两件事：当前 prototype 的具体做法，以及这些做法的假设和限制。
+
+## 详细模块语义
+
+### 1. Observation Schema
+
+Harness 先把一次 trial/probe 的结果转成结构化 observation：
+
+```text
+O_t = {
+  workload summary,
+  SLO rules,
+  effective engine config,
+  best feasible probe,
+  limiting probe,
+  failed_reason_counts,
+  early_stop_reason,
+  pass_rate,
+  request_rate_per_gpu,
+  launch / OOM status
+}
+```
+
+其中 `failed_reason_counts` 的定义是 request-level SLO violation reason 的 multiset 计数：
+
+```text
+ttft_ms>threshold       request 的 TTFT 超 SLO
+tpot_ms>threshold       request 的 TPOT 超 SLO
+arrival_lag_s>limit     synthetic arrivals 已经追不上
+probe_elapsed_s>limit   probe 总耗时超过上限
+slo_pass_rate_unrecoverable
+                        已失败过多，数学上无法达到 target pass rate
+request_failed / timeout / completion mismatch
+                        请求级失败
+```
+
+重要限制：
+
+- 一个 request 可以同时贡献 `ttft_ms>...` 和 `tpot_ms>...`；
+- `failed_reason_counts` 是 symptom evidence，不是 root-cause ground truth；
+- queueing/admission 主要来自 probe 调度层 early stop，而不是单个 request latency 的精确分解。
+
+因此文档和论文里必须避免说“failed count 证明 root cause 是 TTFT/TPOT”。更准确的说法是：
+
+```text
+failed_reason_counts gives SLO violation symptoms.
+Harness infers serving-stage hypotheses from these symptoms.
+```
+
+### 2. Evidence Compiler / Bottleneck Hypotheses
+
+当前 prototype 做两层聚合。
+
+第一层是单个 probe 的 active bottleneck。当前实现用 count-majority：
+
+```text
+ttft_count = sum(count(reason startswith "ttft"))
+tpot_count = sum(count(reason startswith "tpot"))
+other_request_failed_count = non-TTFT, non-TPOT request failures
+
+if ttft_count >= max(tpot_count, other_request_failed_count):
+    active = ttft_prefill
+elif tpot_count >= max(ttft_count, other_request_failed_count):
+    active = decode_tpot
+else:
+    active = admission_or_queueing
+```
+
+这一步是 heuristic。它的语义基础是 TTFT/TPOT/arrival-lag 对应不同 serving stages，
+但 “majority label = root cause” 并不成立。
+
+第二层是跨 trial 的 ranked hypotheses。它把以下证据合成 score：
+
+- workload prior：decode-only + TPOT SLO 更支持 decode hypothesis；长 prompt tail 更支持 prefill hypothesis；
+- latest probe active label；
+- historical probe evidence；
+- `failed_reason_counts` 中 TTFT/TPOT/queueing symptom ratio；
+- launch failure / OOM。
+
+更稳健的目标设计应该把第一层 hard label 改成 soft evidence vector：
+
+```text
+e_prefill   = normalized count of TTFT symptoms
+e_decode    = normalized count of TPOT symptoms
+e_admission = normalized count of arrival lag / elapsed / unrecoverable / request failures
+e_memory    = launch or OOM evidence
+```
+
+Candidate generator 应该基于 evidence distribution 生成 top mechanism probes，而不是只相信一个
+hard dominant bottleneck。当前 prototype 的 hard label 是工程近似，不是最终 contribution。
+
+### 3. Mechanism Space
+
+Harness 不在 raw knob Cartesian product 中盲搜：
+
+```text
+raw space = {TP, DP, EP, GMU, MNS, MBT, chunked-prefill, ...}
+```
+
+它先把 knobs 映射到 serving pipeline 上的可控 mechanism：
+
+| Mechanism family | Example knobs | 机制含义 | 典型 evidence |
+| --- | --- | --- | --- |
+| Topology / resource partition | TP, DP, EP, visible GPUs | 改变 compute/memory 分布、replica 数、per-GPU efficiency | TTFT/TPOT pressure, topology frontier 未覆盖 |
+| Prefill scheduler | chunked prefill, MBT | 改变 prefill quantum 和 head-of-line blocking | TTFT symptoms, long prompt tail, low prefix reuse |
+| Admission / concurrency | MNS | 改变活跃 sequence 数和 batch/admission pressure | arrival lag, pass-rate unrecoverable, concurrency underuse |
+| KV/cache headroom | GMU, block/cache knobs | 改变 KV cache blocks 和 memory feasibility | cache pressure, launch/memory, topology settled 后仍有 SLO pressure |
+| Launch/memory feasibility | env, memory-affecting flags | 确认 engine 是否能启动、是否 OOM | launch failure, OOM |
+| Frontier delta transfer | measured runtime delta applied to other Pareto anchors | 将已测 runtime 改动投影到未测 frontier anchor | 同 topology 上 runtime delta 为正，且存在其他 Pareto anchor |
+
+这些 family 的依据不是某个 case 的 winning config，而是 LLM serving pipeline：
+
+```text
+arrival/admission -> prefill -> decode -> memory/launch feasibility
+```
+
+每个 family 必须满足三条约束：
+
+1. 它对应一个可解释的 serving mechanism；
+2. 它只生成 engine schema 和 hardware constraints 下合法的 candidate；
+3. 它的 confirm/reject condition 由真实 measurement 决定。
+
+限制：
+
+- Mechanism family 是 domain-specific，不是 engine-agnostic magic；
+- vLLM/SGLang 等 engine 的 knob 名称不同，需要 adapter 把 engine knobs 映射到同一 mechanism vocabulary；
+- family 本身有系统依据，但当前 score 常数和部分 gate 仍是 heuristic。
+
+### 4. CandidateSet / Intervention Generation
+
+Candidate 不是“一个 patch”这么简单。一个合法 candidate 应包含：
+
+```text
+candidate = {
+  mechanism_family,
+  config_patch,
+  hypothesis,
+  expected_effects,
+  confirm_condition,
+  reject_condition,
+  effective_full_config_signature
+}
+```
+
+当前 prototype 的候选顺序大致是：
+
+1. topology candidates；
+2. frontier-delta projection candidates；
+3. runtime candidates。
+
+其中 runtime candidates 又包含 prefill scheduler、MBT、MNS、GMU 等 family。
+
+设计假设：
+
+- topology/resource partition 通常改变较大的 capacity frontier；
+- runtime knobs 通常是同一 topology 下的 local refinement；
+- 当 topology frontier 未覆盖时，过早 runtime hill-climbing 可能把搜索困在坏 topology；
+- 当一个 runtime delta 已在某个 topology 上测得正收益时，把这个 delta 投影到其他 Pareto
+  anchor 是比完整 factorial grid 更便宜的 interaction test。
+
+限制：
+
+- `topology-before-runtime` 是强 prior，不是定理；需要 ablation；
+- frontier delta transfer 依赖已测 history，如果 history 太少就不能工作；
+- 当前 prototype 中一些 target step 和 score 常数仍然是人工 heuristic。
+
+### 5. Planner Interface
+
+Planner 的职责应该被限制为：
+
+```text
+rank/select candidate from CandidateSet
+explain why this candidate is worth the next trial
+```
+
+Planner 不应该：
+
+- 构造 schema 外的 knob；
+- 绕过 topology / memory constraints；
+- 重复已经测试过的 effective full config；
+- 单方面决定 stop；
+- 把自然语言猜测当成 measurement verdict。
+
+这也是 no-LLM harness 能工作的原因：只要 `CandidateSet` 和 `Validator` 足够有信息，
+一个 deterministic planner 也可以完成 tuning。LLM 的价值在于组合 evidence、解释 tradeoff、
+在候选较多时排序，而不是提供 tuning correctness 的唯一来源。
+
+### 6. Validator / Stop Authority
+
+Validator 是 harness 防止 prompt engineering 化的关键。它负责：
+
+- canonicalize effective full config；
+- 拒绝 no-op 或 repeat；
+- 检查 legal topology / visible GPU / tunable schema；
+- 记录 failure memory；
+- 判断 measurement ceiling，例如 `search.high` 是否不足；
+- 在 candidate coverage 不足时禁止 premature stop；
+- 只有在覆盖和 measurement guards 都满足时授权 stop。
+
+重要设计修正：
+
+```text
+no-repeat must use normalized effective full-config signature,
+not patch signature.
+```
+
+因为 runtime-only patch 在 materialization 时会继承 incumbent topology。
+如果只看 patch signature，可能把 `{"gmu": 0.9}` 误认为新 config，
+但真实执行时它可能 materialize 成已测过的 full config。
+
+限制：
+
+- Validator 只能保证相对于声明的 grammar/operator set 的 coverage；
+- 它不能证明全 raw knob space 没有更优点；
+- measurement ceiling 不足时应报告并请求人类确认，而不是静默合成 arrivals 或重复窗口。
+
+## 精确贡献表述
+
+我们应该 claim：
+
+```text
+AITuner introduces a planner-agnostic harness that converts LLM serving
+configuration tuning from black-box knob search into typed, measurement-grounded
+counterfactual experiments over serving mechanisms.
+```
+
+可拆成三点贡献：
+
+1. **Serving-stage evidence compiler**
+   将 workload profile、SLO violation symptoms、probe early stop 和 launch failure
+   转换为 prefill/decode/admission/memory/launch 的机制证据，而不是只给 planner 一个 scalar score。
+
+2. **Typed mechanism action space**
+   将 raw knobs 组织为 topology、prefill scheduler、admission/concurrency、cache headroom、
+   frontier transfer 等 intervention families，使搜索发生在 mechanism space 而不是任意 knob vector space。
+
+3. **Validator-controlled experimental loop**
+   用 full-config signature、constraints、failure memory、coverage 和 measurement guards
+   控制 proposal 与 stop，使 LLM/BO/heuristic 都只能在合法、可审计的 candidate set 上工作。
+
+我们不应该 claim：
+
+- bottleneck classifier 永远正确；
+- `failed_reason_counts` 是 root cause label；
+- 当前 heuristic score 常数有理论最优性；
+- harness 覆盖完整 raw knob space；
+- stop 证明全局最优；
+- 某个 case 的 winning config 被系统“证明”出来。
+
+## 必须补的证据
+
+为了证明贡献不是 rule accumulation，后续实验必须 ablate family 和 authority，而不是只报最终性能：
+
+| Ablation | 证明什么 |
+| --- | --- |
+| classifier off / shuffled evidence | evidence attribution 是否真的影响正确方向 |
+| mechanism space off，改用 raw random/BO | mechanism action space 是否压缩搜索并提升收敛 |
+| topology-before-runtime off | 大 frontier intervention prior 是否必要 |
+| frontier-delta projection off | cross-topology runtime transfer 是否解决 bad-start/local trap |
+| validator off / patch signature only | full-config validator 是否避免重复和 false progress |
+| no-LLM deterministic planner | harness 是否是 planner-agnostic substrate |
+| weak planner + harness vs strong planner naive | harness 是否能补偿 planner 能力差距 |
+
+最终论文表达应保持这个边界：
+
+```text
+Harness makes the search more structured, auditable, and measurement-efficient.
+It does not replace measurement, does not prove global optimality, and does not
+turn symptom labels into perfect causal diagnosis.
+```

docs/aituner-harness-summary.md

@@ -1,5 +1,23 @@
 # AITuner Harness Summary
 
+## No-LLM Deterministic Planner
+
+当前 harness 不只是给 LLM 的 prompt hints。它已经可以在没有 LLM endpoint 的情况下，
+作为 deterministic planner 完成一整轮 tuning：
+
+1. 先运行 baseline，得到真实 probe/SLO evidence。
+2. 从 probe history 构造 trial profile 和 bottleneck hypotheses。
+3. 从 topology/runtime intervention grammar 中生成合法 candidate actions。
+4. 用 expected relief、information gain、launch safety 和 regression risk 给候选打分。
+5. 若高分候选存在，直接写出 `harness-proposal-XXXX`。
+6. 若候选耗尽，且 validator 证明 post-incumbent validation 已充分，写出
+   `harness-stop-XXXX`。
+7. 只有 harness 既不能 propose 也不能 stop 时，才调用 LLM；如果没有 LLM endpoint，
+   tune loop 会显式失败。
+
+完整机制和 Qwen30B no-LLM 真实轨迹见：
+[No-LLM harness mechanism](harness-ablation/no-llm-harness-mechanism-20260625.md)。
+
 ## What The Harness Adds
 
 The harness turns each LLM proposal from open-ended config search into a bottleneck-directed decision.

docs/aituner-maas-collab-overview-20260703.md

@@ -0,0 +1,221 @@
+# AITuner：MaaS Serving Config 自动调优 — 合作概述
+
+> 面向：配置调优团队
+> 目的：介绍 AITuner 是什么、为什么它比纯 LLM loop 可靠、以及我们建议的 pilot 合作方式。
+> 日期：2026-07-03
+
+## 一句话总结
+
+AITuner 把 LLM serving engine 的 config tuning 从"人工试错 / LLM 黑盒瞎猜"变成
+**基于真实测量、按 bottleneck 机制分类、由 validator 把关的自动实验循环**。
+我们希望在贵团队的真实环境上跑通 1-2 个 case，验证它能否成为你们的日常工具。
+
+## 1. 问题背景
+
+MaaS 场景下 tuning 的现状：
+
+- 平台上有**数百个模型**，模型本身还在持续迭代；
+- 硬件平台在更新，同一个模型在不同硬件上的最优 config 不同；
+- 每个 (model, hardware, workload, SLO) 组合都是一个独立的 tuning case；
+- 人力远远覆盖不了所有 case，大量 case 只能用默认或粗调的 config 上线，
+  留下吞吐和成本上的浪费。
+
+AITuner 的目标：**自动化 tune 这些没人力覆盖的 case**，输出满足 SLO 的
+engine config，并附带可审计的实验证据。
+
+## 2. 为什么"纯 LLM 自动调参"不够
+
+直接让 LLM 在循环里提 config、跑一轮、再提 config，实践中有两个硬伤：
+
+**(a) 缺 domain-specific 知识和内部 context。**
+
+- LLM 会误读 engine knob 的语义，典型例子是 vLLM 的 DP：LLM 常把它当成
+  "免费加吞吐"的开关，忽略它改变的是 replica 数和 per-GPU 效率，
+  在 per-GPU 指标下盲目 scale-out 反而变差；
+- 内部平台（如 dash）有大量内部环境变量和 launch 约束，LLM 完全没有这部分
+  context，提出的 config 经常直接 launch failure 或 OOM。
+
+**(b) 缺 bottleneck breakdown，不会像专家一样理解系统。**
+
+- LLM 拿到的往往只是"pass rate 低了"这样的 scalar 结果，
+  它无法区分瓶颈在 prefill、decode、admission/queueing 还是 memory；
+- 没有瓶颈归因，proposal 就退化成 knob space 里的随机游走：
+  重复已试过的配置、在错误的 knob family 上反复消耗 GPU trial。
+
+AITuner 的设计就是补上这两块：把系统知识和瓶颈分析放进一个 **harness**，
+LLM 只负责在 harness 给出的合法候选里做排序和取舍。
+
+## 3. AITuner 是怎么工作的（概述）
+
+一句话：AITuner 是一个自动实验循环——对目标 case 做真实压测，
+从结果做瓶颈归因，按系统机制生成下一个 config 候选，
+经 validator 把关后执行下一轮测量，直到证据表明继续实验不再值得。
+
+```text
+真实压测 ──> 瓶颈归因 ──> 机制化候选 ──> Validator 把关 ──> 下一轮压测 / 停止
+```
+
+对使用方来说，需要知道的只有四点：
+
+1. **每一步决策都来自真实测量**，不是 LLM 的自然语言猜测——每轮以
+   SLO verdict、pass rate、`request_rate_per_gpu`、launch/OOM 状态为准；
+2. **像专家一样先归因再动手**：把 SLO 违约症状聚合成 prefill / decode /
+   admission / memory 瓶颈假设，候选 config 只从对应的机制
+   （拓扑切分、prefill 调度、并发准入、KV cache 余量等）中生成，
+   不在 knob 空间里盲搜；
+3. **Validator 挡住不合法和重复的实验**：engine 参数合法性、硬件/拓扑约束、
+   内部平台 launch 约束、已测配置查重、失败记忆——包括 LLM 在内的任何
+   proposal 来源都必须过这一关；
+4. **知道什么时候该停**：验证充分或触及测量上限时确定性停止，
+   不多烧 GPU，也不会静默宣称"已经最优"。
+
+架构与模块细节见 `docs/aituner-harness-design-contract.md`，
+pilot 阶段可按需深入，这里不展开。
+
+## 4. 关键性质：不被单一 LLM 绑死
+
+Harness 是 **planner-agnostic** 的：LLM、确定性 heuristic 甚至 BO/bandit
+都只是在同一个 CandidateSet 上做排序。目前已经验证：
+
+- **No-LLM 模式**：在没有任何 LLM endpoint 的情况下，harness 可以作为
+  deterministic planner 完成整轮 tuning（baseline → 假设 → 候选 → 打分 →
+  proposal/stop），已有 Qwen30B 真实轨迹；
+- 高分确定性候选存在时根本不调 LLM，LLM 只在候选需要复杂 tradeoff
+  排序时介入。
+
+这意味着**换 LLM 供应商的风险是可控的**：tuning 的正确性来自
+harness 的证据编译和 validator，而不是某个特定模型的能力。
+
+## 5. 已有证据（内部实验，3 个 case）
+
+对照组均为"纯 LLM loop"（同一 LLM、同一压测框架，只关闭 harness），
+指标为满足 SLO 的 `request_rate_per_gpu`（每 GPU 可承载请求率，越高越好）。
+
+### Case 1：qwen27b chat 0-8k，dash0 内部 vLLM，H20
+
+真实 trace 窗口回放（`chat_w20260311_1000`），SLO：95% pass rate、
+TTFT 2s/4s/6s 分档、TPOT ≤ 50ms。
+详见 `docs/qwen27b-chat-0-8k-current-config-fig18-20260506.md`。
+
+| | 纯 LLM loop | AITuner |
+| --- | --- | --- |
+| 最终最优 config | TP2/DP1，**0.2025** req/s/GPU | TP4，**0.4429** req/s/GPU（**约 2.2x**） |
+| baseline（起点相同） | TP1/DP1，0.0350 | TP1/DP1，0.0350 |
+| 搜索路径 | 第 2/3 轮先选 DP2、DP4，per-GPU 吞吐反而回落；第 4 轮才到 TP2 | 瓶颈归因判定 TTFT/prefill 主导，第 2 轮直接 TP2（0.2142），第 4 轮 TP4（0.4429） |
+| tuning 开销 | 跑满 12 轮 GPU trial，其中第 5-12 轮全部是无可行点的 runtime probe（纯浪费） | 第 8 轮确定性 stop，实际执行 4 次 GPU trial，全程约 2.5 小时 |
+
+两个值得注意的点：
+
+- 纯 LLM loop 的前几轮正是第 2 节所说的 DP 误读实例——LLM 把 DP scale-out
+  当成免费吞吐，per-GPU 效率被稀释，绕了 3 轮弯路；
+- 单轮真实 trial（engine launch + 多个二分 probe）约 1 小时，跑满 12 轮
+  意味着 10 小时以上的 GPU 占用；AITuner 在拿到约 2.2x 的 config 的同时，
+  把整个 tuning 过程压到约 2.5 小时。
+
+### Case 2：qwen235b thinking prefill（大模型，TP4 baseline）
+
+详见 `docs/qwen235b-thinking-prefill-harness-20260427.md`。
+
+| | 纯 LLM loop | AITuner |
+| --- | --- | --- |
+| 最优 config | TP8，**0.3794** req/s/GPU，第 **10** 轮才找到 | TP8，**0.3863** req/s/GPU，第 **2** 轮即超过对照组 12 轮的最优值 |
+| 搜索路径 | 中途浪费在 DP2、EP4 等失败探索上 | 从 baseline 直接跳到 TP8/DP1，跳过对照组踩过的失败方向 |
+| tuning 开销 | 12 轮预算 | 到达最优的迭代数从 10 降到 2（**5x**） |
+
+大模型 case 上单轮 trial 更贵，少跑 8 轮的绝对 GPU 成本节省也更大。
+
+### Case 3：Qwen3-30B-A3B，社区 vLLM 0.20（非内部环境同样适用）
+
+详见 `docs/qwen30b-community-vllm020/harness-early-stop-ablation-20260502.md`。
+两个子实验：
+
+- **测量上限识别**（search 上限较低时）：默认 config 已触及压测搜索上限。
+  AITuner 只执行 1 次 GPU trial 就识别出"当前压测范围测不出更优配置"，
+  停止并明确报告原因；纯 LLM loop 把 12 轮预算烧完——第 2 轮 DP2 使
+  per-GPU 吞吐减半，第 3-12 轮**连续 10 轮 launch failure**。
+  净节省 11 轮 GPU 占用。
+- **放宽上限后的真实搜索**：AITuner 第 4 轮到达最优 config family
+  （TP2 + runtime 精调）并在第 5 轮 stop；纯 LLM loop 经历 EP launch
+  failure、不可行 DP probe 等弯路后第 7 轮才到同一 family（单次测值差
+  约 1.5%，在重复压测噪声内）。
+
+### 汇总
+
+| Case | config 性能（vs 纯 LLM loop 最优） | 到达最优迭代数 | 省下的无效 GPU trial |
+| --- | --- | --- | --- |
+| qwen27b 内部 vLLM | **约 2.2x**（0.4429 vs 0.2025） | 4 vs 4，但对照组随后 8 轮全部无效 | 8 轮 infeasible probe |
+| qwen235b thinking | 持平略优（0.3863 vs 0.3794） | **2 vs 10（5x）** | 8 轮失败/弱探索 |
+| qwen30b 社区 vLLM | 同一 config family（±1.5%） | 4 vs 7；上限场景 1 vs 12 | 最多 11 轮（含 10 轮 launch failure） |
+
+诚实说明：以上是有限 case 上的证据，qwen30b 子实验使用了有界压缩回放
+（固定输出长度）做收敛性测试，不等同生产 benchmark；结论是"更快收敛到
+同等或更好的 config、大幅减少无效 GPU trial"，不是全局最优性证明。
+这正是我们想通过 pilot 在你们的真实 case 上进一步验证的。
+
+## 6. 你们会得到什么
+
+对每个 case，AITuner 的产出不只是一个 config：
+
+1. **满足 SLO 的 engine config**（以 `request_rate_per_gpu` 为主要
+   跨拓扑指标）；
+2. **完整可审计的实验轨迹**：每个 trial 的假设、预期效果、真实测量、
+   confirm/reject 结论，以及 probe 级别的明细（`probe_details.jsonl`）；
+3. **瓶颈归因报告**：这个 case 的限制因素是 prefill、decode、admission
+   还是 memory，为什么；
+4. **明确的 stop 理由**：是候选耗尽、验证充分，还是 measurement 上限
+   （比如 search 范围）不够——不会静默糊弄。
+
+这些轨迹本身对你们的人工 tuning 经验沉淀也有价值。
+
+## 7. 建议的合作方式（pilot）
+
+我们目前缺少的是贵团队每个 case 的真实环境和硬件。建议直接在你们的
+环境上做，分四步：
+
+**Phase 0 — 选 case、对齐输入（约 1 周）**
+- 双方选定 1-2 个有代表性的 case；
+- 每个 case 需要：模型 + 硬件规格、SLO 定义（TTFT/TPOT/pass rate）、
+  workload trace 或可复现的流量描述、当前人工 config（作对照基线）；
+- 我们把 dash 平台的内部环境变量 / launch 约束接入 harness 的
+  launch-feasibility 层和 validator。
+
+**Phase 1 — 在你们环境跑通（1-2 周）**
+- 部署 AITuner，先跑 baseline 建立测量基线，再跑完整 tune loop；
+- 我们负责跑通和调试，你们提供环境访问和平台侧支持。
+
+**Phase 2 — 对比评估**
+- AITuner 结果 vs 你们的人工 config：SLO 达标情况、`request_rate_per_gpu`、
+  消耗的 GPU trial 数、时间成本；
+- 全部结果附实验轨迹，可复查。
+
+**Phase 3 — 决策**
+- 若 pilot 达标，讨论扩展到更多 case 的方式（接入流程、权限、
+  运行成本、维护分工）。
+
+## 8. 当前依赖与风险（如实说明）
+
+1. **LLM 依赖**：当前 planner 使用 gpt-5.5。计划切换到百炼
+   qwen3.7-max 的 dog-fooding API，并做同 case 的效果对比。
+   风险缓冲：harness 的 no-LLM deterministic 路径已经能独立完成
+   相当一部分 tuning（见第 4 节），planner 模型的能力差距被 harness
+   部分补偿，切换成本预计可控——但对比数据出来之前这是一个待验证项。
+2. **Engine 适配**：当前 mechanism families 主要针对 vLLM 的 knob
+   语义；SGLang 等其他 engine 需要一层 adapter 把 knobs 映射到相同的
+   mechanism vocabulary（架构上已预留，工作量取决于目标 engine）。
+3. **平台适配**：dash 内部环境变量和 launch 约束需要在 Phase 0 一次性
+   接入，之后由 failure memory 持续积累。
+4. **边界**：AITuner 保证的是结构化、可审计、测量高效的搜索；
+   它不证明全局最优，瓶颈分类是 symptom-based 的启发式归因而非
+   完美因果诊断。对生产决策来说，可审计比"号称最优"更重要。
+
+## 9. 我们需要贵团队提供的
+
+- 1-2 个 case 的测试环境和硬件访问（或由你们的同学代跑，我们远程支持）；
+- 每个 case 的 SLO 定义和 workload trace；
+- dash 平台内部环境变量 / launch 约束的文档或对接人；
+- 百炼 qwen3.7-max dog-fooding API 的配额（用于 LLM 切换对比）。
+
+---
+
+附：更完整的设计语义见 `docs/aituner-harness-design-contract.md`，
+harness 各机制与实验证据见 `docs/aituner-harness-summary.md`。

docs/aituner-roadmap.md

@@ -0,0 +1,283 @@
+# AITuner roadmap
+
+本文只维护最小 roadmap：paper framing、claim 树、已有证据、最高优先级实验。
+详细实验流水账放到对应专题文档里。
+
+## Paper thesis
+
+AITuner 的核心不是“用 LLM 调参”。更准确的 framing 是：
+
+```text
+black-box knob optimization
+  -> grey-box / mechanism-guided experimental optimization
+```
+
+也就是说，AITuner 仍然通过真实实验测量目标函数，但它不再把 serving engine 当成
+完全黑盒的 `config vector -> scalar score`。Harness 将 workload、SLO failure、
+probe trace、topology constraints 和 failure memory 转换成结构化的 serving
+mechanism state，并把下一步搜索限制在可解释、可验证的 intervention 上。
+
+因此 LLM 不是不可替代的核心。LLM 是 planner backend / copilot；核心系统贡献是
+planner-agnostic 的 tuning substrate：
+
+```text
+Harness H = (O, R, G, V, M)
+
+O: observation schema
+   workload L/C/A profile + probe trace + latency/SLO failure + launch status
+
+R: regime attribution
+   SLO violation -> prefill-bound / decode-bound / admission-bound / memory-bound / launch-bound
+
+G: serving intervention grammar
+   regime -> legal intervention families, not raw arbitrary knobs
+
+V: validator
+   tunable schema + topology constraints + no-repeat + failure memory + stop authority
+
+M: measurement/scoring protocol
+   SLO-constrained feasible frontier, req/s/GPU, latency quantiles, pass-rate guard
+```
+
+当前 `src/aituner/harness.py` 是 prototype：它已经展示 no-LLM loop 和 mechanism-guided
+proposal 的可行性，但仍然包含大量 rule-based heuristics，不能作为最终 harness
+contribution。新的目标设计见 [Declarative intervention harness design](harness-ablation/declarative-intervention-harness-design-20260626.md)。
+
+Planner 是可替换的：
+
+```text
+pi in {LLM, BO, bandit, deterministic heuristic, tree search}
+```
+
+AITuner 的强 claim 应该是：同一个 planner 放在 harness-shaped space 里，比放在
+raw knob space 里更快、更稳、更接近最优；弱模型或非 LLM planner 也能从这个 substrate
+中获益。
+
+## Why not pure white-box
+
+我们不应 claim 完整 white-box optimization。AITuner 没有解析 vLLM scheduler、
+kernel、KV cache、通信和排队的闭式性能模型。更稳妥也更强的表述是 grey-box：
+
+- objective 仍然由真实测量决定；
+- action space、constraints、failure attribution 和 intervention semantics 是系统知识驱动；
+- 每个 trial 是一个 counterfactual experiment，而不是盲目采样一个 knob vector。
+
+## 关键设计点
+
+当前 harness 设计语义、模块假设和限制见
+[AITuner Harness Design Contract](aituner-harness-design-contract.md)。Roadmap 只维护
+claim 和实验优先级；design contract 负责精确定义我们能说什么、不能说什么。
+
+| 设计点 | 更强表述 | 作用 | 需要证明 |
+| --- | --- | --- | --- |
+| Observation | mechanism state | 将 workload shape、probe trace、SLO failure、launch/memory failure 结构化 | agent 看到的是可计算状态，不是自然语言日志 |
+| Bottleneck classifier | SLO violation attribution | 把失败归因到 serving regime，而不是只看哪个指标超阈值 | attribution 和后续有效 intervention 有因果关联 |
+| Candidate family | serving intervention grammar | 把 raw knobs 提升为 topology / batching / admission / memory interventions | 搜索空间被压缩，但不写死某个 case |
+| Scoring | counterfactual verdict | 用 SLO frontier 和 req/s/GPU 判断 intervention 是否支持假设 | 最终 winner 由测量决定，不由 LLM 决定 |
+| Validator / stop | fail-safe control | 禁止非法、重复、已知失败 family；只有 validator 授权 stop | 错误 attribution 最多浪费 trial，不污染 incumbent |
+
+## Claim roadmap
+
+| Claim | 当前状态 | 证据文档 | 关键缺口 |
+| --- | --- | --- | --- |
+| C1. Harness 将 raw knob search 转成 mechanism-guided intervention search，提升固定预算优化效果 | 已有强信号 | [No-LLM harness mechanism](harness-ablation/no-llm-harness-mechanism-20260625.md), [Qwen27B 2x2](harness-ablation/qwen27b-tight-2x2-model-ablation-20260623.md), [Qwen30B SLO robustness](harness-ablation/qwen30b-slo-robustness-20260624.md) | 补 Qwen235B decode 2x2 aggregate；补 mechanism ablation |
+| C2. 收益来自 harness-defined substrate，不依赖某个强 LLM | Qwen30B no-LLM 已完整闭环；Qwen27B 弱/强模型已有 | [No-LLM harness mechanism](harness-ablation/no-llm-harness-mechanism-20260625.md), [Qwen27B 2x2](harness-ablation/qwen27b-tight-2x2-model-ablation-20260623.md) | 做 `BO/heuristic + harness` vs `BO/heuristic + raw knobs` |
+| C3. Weak planner + harness 可以匹配或超过 strong LLM naive | Qwen27B 已支持；Qwen235B 正在补 | [Qwen27B 2x2](harness-ablation/qwen27b-tight-2x2-model-ablation-20260623.md), [Qwen235B prefill progress](harness-ablation/qwen235b-prefill-2x2-progress-20260623.md) | 完成 Qwen235B decode 2x2；更新 prefill final doc |
+| C4. Attribution 和 intervention grammar 有机制贡献，不只是 prompt 信息更多 | 设计和 no-LLM case 已整理；严格 ablation 不足 | [No-LLM harness mechanism](harness-ablation/no-llm-harness-mechanism-20260625.md), [AITuner summary](aituner-harness-summary.md) | 做 shuffled attribution / no attribution / no grammar / no topology-first / no validator ablation |
+| C5. AITuner 找到 near-optimal region，而不是只找到一个可行 config | Qwen30B 有解释性信号 | [Qwen30B SLO robustness](harness-ablation/qwen30b-slo-robustness-20260624.md) | 选 1-2 个 case 做局部 grid 或专家配置对照 |
+| C6. AITuner 能随 SLO tightness 移动到合适 frontier | Qwen30B 已完成 | [Qwen30B SLO robustness](harness-ablation/qwen30b-slo-robustness-20260624.md) | 再选一个非同质 case 做 SLO sweep；同时画 SLO tightness -> frontier/regime transition |
+| C7. Engine adapter 让 intervention grammar 可迁移到其他 serving engine | 设计上可行，暂不作为主实验 claim | `EngineLaunchSpec` / launch recipe / tunable schema | vLLM 主线完成后，再做 SGLang adapter 和一个低成本验证 case |
+| C8. Harness 对坏初始点有恢复能力，不只依赖可信 base config | 单个 adversarial bad-start 已通过 first repair；分布级 robustness 不能 claim | [Declarative intervention harness design](harness-ablation/declarative-intervention-harness-design-20260626.md), [Bad-start stop counterexample](harness-ablation/bad-start-stop-counterexample-20260626.md), [Bad-start robustness suite](harness-ablation/bad-start-robustness-suite-20260626.md), [No-LLM harness mechanism](harness-ablation/no-llm-harness-mechanism-20260625.md) | 按 pre-registered 20-case suite 跑 random/adversarial start distribution |
+
+## 最高优先级实验
+
+### P0a. Declarative harness redesign gate
+
+目的：停止继续向 `harness.py` 添加 testcase-specific rules，把 harness 重构成
+declarative intervention grammar + coverage-relative validator。
+
+最低验收：
+
+- CandidateSet 完整枚举并持久化 snapshot；
+- CandidateSet v1 先限定为当前 harness generator 实际构造出的 concrete candidates，
+  不 claim 全 Cartesian knob space 枚举；`candidate_set_hash`、eligible/blocked
+  records 和 blocked reason summary 已在 harness context 与 `harness/candidate-set-*.json`
+  sidecar 中实现；
+- `harness_priority` 与 backend ranking 分离；
+- CoverageUnit 结构化，stop 不能只依赖 exact signature；
+- `search_high_saturated_by_incumbent` 不能绕过 CandidateSet coverage；对 `req/s/GPU`
+  目标，未覆盖 topology/resource-efficiency contrast 时必须继续；
+- 加入 `auto_search_high` measurement policy：可在已有 trace 内自动提高 ceiling；若
+  `search.high=1.0` 仍然不足，必须报告 `measurement_ceiling_insufficient` 并等待人类
+  确认，不得静默重复窗口或合成 arrivals；
+- normalized full-config signature：no-repeat 不能只看 patch signature；base config 与
+  no-op patch 必须被识别为同一 full config；`48911b6` 已实现并在 dash1 bad-start
+  validation 中通过；
+- materialized effective signature：runtime-only proposal 必须先按真实执行路径继承
+  incumbent topology，再做 no-repeat；已加入 shared signature/canonicalization，并在
+  CLI 进入 trial 前 hard-veto 重复 LLM/manual/harness proposal；
+- Failure invalidation 有保守 region predicate 和 retry/unblock 条件；
+- grammar/policy/capability 都有 version 和 anti-overfitting static checks；
+- LLM/BO 只能选择合法 candidate，不能绕过 validator。
+
+优先级原因：如果不先完成这个 gate，继续扩展 bad-start/SLO/2x2 实验只是在证明一个
+rule-based prototype。
+
+### P0b. Bad-start recovery confirmation after redesign
+
+目的：回答 harness 是否只能从可信 base config 起步，还是能从明显不合理的初始 config
+恢复到正确方向。
+
+Pre-registered suite 见 [Bad-start robustness suite](harness-ablation/bad-start-robustness-suite-20260626.md)。
+
+最小实验矩阵：
+
+| Case | 初始配置 | 证明点 |
+| --- | --- | --- |
+| bad-topology | `TP=8, DP=1` | 高 TP 起点会先做相邻低 TP bracket |
+| bad-runtime | `TP=2, gmu=0.5, max-num-seqs=8` | 低 runtime headroom 会跳回 nominal floor |
+| combined-bad | `TP=8, gmu=0.5, max-num-seqs=8` | topology recovery 和 runtime recovery 能串联 |
+
+注意：这不是先跑一条手工 bad case。必须在 declarative harness 之后跑 random/adversarial
+start distribution，并报告分布结果。
+
+预期图：
+
+- x-axis: trial index；
+- y-axis: best-so-far SLO-constrained req/s/GPU；
+- line groups: trusted-start vs bad-start cases；
+- annotation: proposal family sequence，例如 `TP downshift`, `gmu floor jump`, `gmu climb`。
+
+启动条件：先完成 P0a；再确认 dash fleet 有空闲 8xH20 机器；用户确认后再开跑。
+
+### P0c. 完成 Qwen235B decode 2x2 并整理 aggregate
+
+目的：补齐最核心的 `harness on/off x strong/weak planner` 证据，回答：
+
+```text
+weak LLM + harness >= strong LLM naive ?
+```
+
+预期产出：
+
+- 2x2 表格：每个 arm 在相同 iter budget 下的 best-so-far req/s/GPU；
+- convergence curve / normalized AUC；
+- 每个 arm 的 trial path 和主要 config patches；
+- 解释 naive 为什么走错，harness 如何通过 regime attribution 走到正确 intervention。
+
+优先级原因：实验已经在跑，增量成本最低，而且直接支撑 C1/C3。
+
+### P1. Planner-agnostic substrate 实验
+
+目的：证明 AITuner 不是 LLM tuner，而是 harness-defined optimization substrate。
+
+最小实验矩阵：
+
+| Planner | Raw knob space | Harness intervention space |
+| --- | --- | --- |
+| deterministic heuristic | raw heuristic | harness policy |
+| BO 或 lightweight bandit | raw BO | harness-guided BO |
+| weak LLM | naive weak LLM | weak LLM + harness |
+| strong LLM | naive strong LLM | strong LLM + harness |
+
+如果 BO 实现成本高，先用 deterministic harness policy 做 non-LLM planner baseline：
+它已经能证明“没有 LLM 也能 work”。随后再补 BO，使论证更强。
+
+预期图：
+
+- x-axis: trial budget；
+- y-axis: best-so-far SLO-constrained req/s/GPU；
+- line groups: raw knob space vs harness intervention space；
+- 单独 bar：invalid launch rate、repeated config rate、wasted trial rate。
+
+优先级原因：这是新 framing 的关键实验。没有它，paper 仍然容易被读成“LLM prompt
+engineering”。
+
+### P2. Mechanism ablation
+
+目的：证明 harness 内部不是普通信息堆叠，而是 attribution、intervention grammar、
+validator 分别贡献有效机制。
+
+建议 ablation：
+
+| Variant | 删除/破坏什么 | 预期证明 |
+| --- | --- | --- |
+| full AITuner | 无 | 最好 |
+| no attribution | 不提供 regime attribution，只给 scalar score 和历史结果 | attribution 对方向选择有贡献 |
+| shuffled attribution | 故意打乱 regime label，但保留文本长度 | 收益来自语义正确性，不是更多 prompt tokens |
+| no intervention grammar | 允许任意 tunable knobs，移除 family guidance | action-space shaping 有贡献 |
+| no topology-first | runtime knobs 可以优先于 topology intervention | topology 是 LLM serving 的一阶决策 |
+| no validator/failure memory | 允许重复、已知 launch failure family | fail-safe control 减少 GPU burn |
+
+预期图：
+
+- mechanism ablation bar：final best、AUC、TTT；
+- waste breakdown：invalid launch、repeat config、wrong-family trial；
+- case study trace：每个 variant 前 3-5 个 proposal 对比。
+
+优先级原因：这是回应 novelty 质疑的核心证据。
+
+### P3. Near-optimum / expert baseline 证据
+
+目的：证明 AITuner 不是只找到“能收敛但性能差”的 config。
+
+优先选择一个成本可控 case 做局部 grid：
+
+```text
+topology: TP/DP frontier
+runtime: max-num-seqs, max-num-batched-tokens, gpu-memory-utilization 的小邻域
+objective: max feasible req/s/GPU under pass_rate >= 0.95
+```
+
+预期图：
+
+- local grid heatmap；
+- AITuner trial path overlay；
+- AITuner best vs grid best vs expert config；
+- near-optimum gap，例如 `AITuner >= 95% of local grid optimum`。
+
+优先级原因：这是 claim “tune 出最好的 config，而不是差的收敛 config” 的必要证据。
+
+### P4. 第二个 SLO robustness case
+
+目的：证明 Qwen30B 的 SLO robustness 不是单 case 现象。
+
+不要先大规模铺 sweep。先选一个和 Qwen30B 机制不同的 case：
+
+- 一个 decode-heavy case，观察 TP/DP redistribution 和 concurrency/memory intervention；
+- 或一个 long-prefill / tight-TTFT case，观察 TP 和 prefill batching intervention。
+
+预期图：
+
+- x-axis: SLO tightness；
+- y-axis: best feasible req/s/GPU；
+- marker/color: selected intervention regime；
+- annotation: final TP/DP/MNS/MBT；
+- 展示 SLO 放宽时 frontier/right shift 或 regime transition。
+
+优先级原因：重要，但应排在 planner-agnostic 和 mechanism ablation 之后。
+
+### P5. SGLang / multi-engine adapter validation
+
+目的：证明 intervention grammar 可以通过 adapter lowering 到不同 serving engine。
+
+当前暂缓，不作为 vLLM 主线之前的高优先级实验。等 C1-C5 稳定后再做一个低成本 case：
+
+```text
+same workload profile
+same SLO objective
+same intervention grammar
+different engine adapter
+```
+
+优先级原因：它能扩展 generality，但不能替代 vLLM 主线的机制证明。
+
+## 暂不做
+
+- 暂不把主 claim 写成“LLM 比 BO 更聪明”。新 claim 是 harness substrate 对多种 planner
+  都有用。
+- 暂不 claim full white-box 或全局最优。当前更稳妥的是 grey-box、near-optimum、
+  fixed-budget utility。
+- 暂不横向铺大量 SLO sweep。先补机制 ablation、planner-agnostic 和 near-optimum。
+- 暂不把 multi-engine support 放进主实验 claim。先写成 adapter-based design，等 vLLM
+  证据链完整后再补一个 SGLang validation。

docs/collectivespec-p2-gate-20260713.md

@@ -0,0 +1,154 @@
+# CollectiveSpec P2：logical-plan 对照的审计与停止门槛
+
+## 决策
+
+**不启动正式的 P1/P2 SLO-goodput sweep，也不把 `compact-vs-padded` 作为
+CollectiveSpec 的研究主线。**
+
+原因不是这条机制一定没有工程收益，而是它的核心研究主张已经无法排除公开工作的
+覆盖：
+
+- [DSpark](https://arxiv.org/html/2607.05147) 已明确采用每请求的动态 verification
+  length，并把逻辑 sequence tracking 与物理 execution 解耦、flatten variable-length
+  token；
+- SGLang 的 [DSpark 集成说明](https://www.lmsys.org/blog/2026-07-06-dspark-sglang/)
+  已公开 `static`、`compact` 与 `cap-accept` 三种 verify mode。其中 `cap-accept`
+  执行完整 block、但只提交 compact window，且说明其输出与 `compact` 相同。这正是
+  “同一语义下 full/padded 与 compact”的 counterfactual；
+- 该实现还公开了 DP attention 下各 rank 使用最大 graph tier 的处理。因此，仅在
+  vLLM/H20 上再复现 compact 比 padded 快，只是环境复现，不是新的系统贡献。
+
+P0 还独立否定了原来的 liveness 动机：目标 runtime 已经以 scalar DP metadata 协调
+不同 DP replica 的物理 shape；异构 verifier candidate 没有引起运行期 collective
+错误。故不能再把“必须新增 canonical header 才能避免死锁”作为论文 premise。
+
+## 术语：什么必须相同
+
+### Logical plan（也称 semantic plan）
+
+logical plan 是一次 speculative verification **应当计算和提交什么**的不可变记录；它
+不包含 padding、CUDA graph tier、物理 rank 行数、worker PID 或耗时。每一个 verifier
+epoch 的最小条目为：
+
+```text
+(global_epoch, dp_rank, ordered request id,
+ logical_output_offset_before, scheduled_seq_len,
+ available_candidate_token_ids, requested_k, effective_k,
+ visible_candidate_token_ids_hash)
+```
+
+请求层还必须固定 `client_request_id`、server request id、prompt/body hash、arrival、DP
+assignment、提交顺序、temperature/seed 与预期 completion length。最终还要逐请求验证
+output token-id hash、completion length、finish reason、usage，以及 endpoint semantic
+transcript hash（content/reasoning/tool-call 的 canonical JSON）。
+
+这里的关键是 `k_i` 的 key 必须是
+`(server_request_id, logical_output_offset_before)`，而不是只有 request id：同一请求会
+经历多个 verification epoch。只有两个 cell 的这些事实都相同，才称为 *same logical
+plan*。
+
+### Compact vs. padded：只是同一 plan 的两个 lowering
+
+给定同一组 non-dummy logical entries：
+
+- **PaddedSync-semantic**：保留这些 entries，但为同步域插入 masked dummy rows，使各
+  DP peer 的 physical shape 对齐；
+- **CompactSync**：保留完全相同的 entries、candidate 和 commit semantics，用 ragged
+  packing / split vector 执行真实 rows，不计算 dummy rows。
+
+因此 `static K=3` 不能当作 padded 对照：它改变了每个请求可见 candidate prefix，改变了
+logical algorithm，而不是只改变 physical lowering。真实 physical-row 公式也不能简单
+写成 `N * (1 + max k_i)`；普通 decode、TP alignment 和 CUDA-graph alignment 都要从
+runner 的 row map 分开计数。
+
+## 当前 P0 对 P2 的限制
+
+P0 heterogeneous policy 按 vLLM **随机生成的 server request id** 哈希，而 client 没有
+发送 `X-Request-Id`。所以即使 trace 和 seed 相同，两个 cell 的每个 request/epoch 的
+`k_i` 也不可保证相同；日志只有 aggregate digest/histogram，也没有 per-row candidate
+token、assignment 或最终 token-id hash。P0 因而不能充当 same-logical-plan 的 P2 A/B。
+
+P0 的 padding 上界也已校正。66 个 target epoch 中 62 个 raw DP counts 不等：
+
+```text
+raw logical rows                 6,276
+local non-DP-aligned rows        6,536
+PaddedSync physical rows         7,024
+DP-global-max attributable rows    488  (= 6.95% of physical rows)
+```
+
+此前的 748 / 10.65% 将 260 行本地 TP/CUDA-graph alignment 混入 DP max padding。即使
+488 行都可回收，它仍只是 **target verifier row-count 的上界**：EAGLE3 仍按 Kmax=3
+完成 drafter 工作，也没有测得 EP bytes、collective critical path 或 E2E SLO-goodput。
+
+## 若未来重新打开，先补齐的测量契约
+
+不应先实现 compact lowering。先添加只用于 audit 的 telemetry：
+
+1. client 对每个请求发送固定 `X-Request-Id`、`X-Data-Parallel-Rank` 和
+   `return_token_ids=true`；记录 response id、prompt/output token-id hash、semantic
+   transcript hash 和 finish reason；
+2. scheduler 作为 semantic ledger 的唯一 writer，记录 per-epoch ordered entries、候选
+   token、requested/effective K、logical cursor 与 sampled token hash；
+3. worker 只记录物理事实：每 rank physical rows、DP/TP/graph padding 的原因、packed row
+   map；若要主张 EP 收益，额外记录 DeepEP all-to-all split vector、bytes、duration 和
+   rank wait；
+4. 汇总器先给出第一个 semantic/output diff，任一 mismatch 即标为 invalid，禁止读取
+   性能数字。
+
+最小 reproducibility smoke（仅在发现 topology gap 后执行）是 fresh engine 上的 16 个
+decode-only requests、每个 64 output tokens、temperature=0、DP0/DP1 各 8 个、显式
+`{0,3}` alternating manifest。先连续跑两次 **同一个** padded cell；只有 ledger 和逐请求
+token hash 全等，才允许运行 padded/compact mechanism probe。`return_token_ids` 会改变 SSE
+负担，故最终 latency cell 必须关掉该字段、改以 scheduler-side hash 审计。
+
+## 唯一尚可证伪的拓扑假设
+
+公开材料没有证明、也没有否定下列特殊情形：**独立 standard-DP scheduler 共享同一个 EP
+all-to-all domain** 时，DP global-max graph tier 之外仍有 EP split-vector / collective
+ordinal 的关键路径浪费。这不能从“论文没有写”推断为新颖性。
+
+只有一次短的 topology reconnaissance 观测到该额外瓶颈，才重新进行文献审计并考虑下列
+顺序严格的 gates：
+
+1. 与 SGLang-style DP global-max tier / current runtime 相比，compact plan 降低实际 EP
+   bytes、split imbalance 或 collective critical-path time；仅少几个 rows 不够；
+2. 在相同 semantic plan、token-exact 输出和无 tail-latency 退化下，至少三次 fresh-engine
+   paired runs 显示 E2E SLO-goodput 增益 >=10%；
+3. topology ablation 支持因果归因：DP=1 或 EP 不跨 DP 时收益消失或显著缩小，而
+   DP×shared-EP 时出现；
+4. 重新完成与 DSpark/SGLang 的逐项差异审计，证明贡献是 topology-aware collective
+   scheduling，而不是已有 ragged packing。
+
+任一 gate 不成立即结束 CollectiveSpec；不以 controller/K/queue knob 调优替代证据。
+
+## 如果 gate 重开时的固定环境与 setup
+
+下列是 P0 实际使用、后续必须 provenance-pin 的环境，而不是当前已启动的实验：
+
+| 项目 | 固定值 |
+|---|---|
+| host / accelerator | `dash0`，8 × NVIDIA H20 |
+| target / draft | Qwen3-235B-A22B FP8；EAGLE3，Kmax=3 |
+| parallelism | TP=4，DP=2，EP=8；`VLLM_MOE_USE_DEEPEP=1` |
+| engine | dash0 live installed vLLM wheel；记录 wheel metadata、import path、launch command 与 commit；不能以本地 checkout API 代替 |
+| execution | `FULL_DECODE_ONLY` CUDA graphs、FP8 KV cache、block size 64、`max-num-batched-tokens=1024`、`max-num-seqs=192`、max model len 262144 |
+| workload | immutable materialized `thinking_w20260327_1000` 的 decode-only window；机制 smoke 使用固定 burst，E2E 使用完整、session-closure 状态明确的 trace |
+| reproducibility | fresh engine per cell、temperature=0、固定 seed、固定 request ids/DP assignment、prefix-cache state 从空开始、ABBA cell order |
+| SLO（若进入 E2E） | 预注册 TPOT <= 40 ms、pass rate >= 0.95；同时报告 completion success、p50/p95/p99、deadline failures 与 output equivalence |
+
+remote source 必须从 Git 同步到
+`/home/admin/cpfs/wjh/collectivespec-pilot/20260713T054328Z/source`，并记录运行时实际
+source revision；任何远端 job 启动前在 artifact 中写明 resolved command、模型/trace path、
+预计 GPU 时间和结果目录。
+
+## 审计数据健全性
+
+- 新增实验数：n=0；本文件不报告任何新的性能数字。
+- 已复核的 P0 target epochs：n=66；两个 DP rank 的 raw row values 共 n=132，
+  min=1、max=77、distinct=35；physical rows 则 n=66，min=4、max=80、distinct=14。
+  原始 JSONL 可复核，不以 aggregate 值伪造每 epoch 分布。
+- 已用 aggregate row totals：n=4，min=488，max=7,024，distinct=4；均为非负。校正后的
+  关系 `6,276 <= 6,536 <= 7,024` 成立，且 `7,024 - 6,536 = 488`。
+- 外部材料覆盖判断区分为“论文明确描述”“官方公开实现明确描述”和“未公开拓扑细节”；
+  未从缺失的 EP 细节推导新颖性或性能收益。

docs/collectivespec-pilot-design-20260713.md

@@ -0,0 +1,319 @@
+# CollectiveSpec：先做机会判定的实验设计
+
+## 结论先行
+
+当前不应先实现一个“按请求动态 K、再做一次 DP collective 同步”的原型。该路径的
+工程修复很薄，而且相邻公开工作已经覆盖了 request-level dynamic speculation 与
+ragged verification 的大量空间。CollectiveSpec 只有在
+一个更强、可证伪的事实成立时才值得继续：**在 wide-EP MoE、DP>1 的生产负载中，
+不同 DP rank / request 所需的投机深度确实不同，并且全局 static K 明显浪费了 SLO
+可行 goodput。**
+
+本文件把第一轮实验定义为一个机会判定（opportunity gate），不是最终性能主张。
+
+## 固定条件
+
+- Host: `dash0`, 8x NVIDIA H20。
+- Model: Qwen3-235B-A22B FP8；draft: EAGLE3。
+- Deployment: TP=4, DP=2, EP=8，`VLLM_MOE_USE_DEEPEP=1`。
+- Trace: `thinking_w20260327_1000`，600 秒 decode-only 窗口。
+- SLO: TPOT <= 40 ms，pass rate >= 0.95。
+- 同一 engine revision、同一模型/trace 路径、同一环境变量；实验串行执行，避免 GPU
+  互相干扰。
+
+这里的 resolved topology 来自远端实际 StudySpec，而不是仓库 README 中可能已过期的
+配置描述。
+
+## 假设与可证伪指标
+
+### G0：static-K 是否有足够可利用的空间？
+
+- H0：在该固定拓扑和负载下，NoSpec/K=1/2/3 的 SLO-goodput 差异很小；最佳固定 K 已经
+  足够好。此时停止 CollectiveSpec。
+- H1：不同 static K 的可行前沿存在实质差异，且最优 K 对负载区间敏感。只有 H1 才
+  说明 dynamic policy 可能有直接性能价值。
+
+主要指标：每个 K 在相同 SLO 下可达到的最大 `sampling_u`，以及对应请求率、TPOT
+pass rate、p50/p95 TPOT、成功/失败原因。`sampling_u` 是现有 replay 使用的一致 trace
+抽样旋钮，因此只能作为此 trace 的 SLO-goodput proxy，不能直接外推为线上 QPS。
+
+本 trace 的 output-length 分布很重尾；replayer 的 drain deadline 可能在长输出尚未完成
+时终止 probe。故每个结果必须同时报告 completion-success count 和 deadline failure；不能
+只因 TPOT pass rate 达标就把截断 request 当作“无成本”。本轮的原始 static screen 仍沿用
+现有 SLO 以便和项目 baseline 可比，但它不能替代完整 completion 的确认实验。
+
+判定门槛（预注册）：
+
+1. 在 K=1/2/3 之间，最优 K 相对次优 K 的最大可行 `sampling_u` 小于 5%，或
+   置信区间/重复实验重叠很大：
+   **停止**。
+2. 最优 speculative K 比次优 speculative K 至少高 10%，且在两个独立重复中方向一致：
+   进入 G1。NoSpec 仅作为“是否值得用 draft model”的部署对照，不能替代这条判据。
+3. 如果 K=3 不受当前 engine 支持、任一配置启动失败，记录为兼容性结果，不把它误作
+   性能差。
+
+## 第一阶段：static-K screening
+
+配置为 `{NoSpec, K=1, K=2, K=3}`。NoSpec 会删除 `--speculative-config`，而不是传
+非法的 `num_speculative_tokens=0`。它释放 draft model 相关资源，因此不等于
+same-stack 的 logical K=0；后者在 EAGLE 类实现中仍可能需要一次 draft forward 来保持
+KV 同步。当前 dash0 binary 的 MLA indexer 明确限制 `num_speculative_tokens <= 3`，
+故这已穷尽该 binary 的合法 static horizon。原计划每个配置：
+
+- 搜索范围 `sampling_u in [0.005, 0.020]`；
+- 最多 3 次 probe、tolerance=0.003；
+- 每个 probe 使用完整 600 秒 trace replay（不会使用 `max_requests_per_probe` 的
+  截断模式）；
+- 启动顺序 `2,1,3,0`，降低冷启动或时间漂移与 K 单调对应的风险；
+- 每个 K 都使用独立 Store、不可变派生 StudySpec、完整 stdout/stderr log。
+
+这是一轮筛选而非 final frontier。它一旦显示值得继续，才对 top-2 K 做交叉顺序的完整
+搜索与至少两次重复。
+
+### 2026-07-13 数据质量修正：controlled screen
+
+原始 trace 第一个 K=2 probe 暴露了一个不应隐藏的 measurement red flag：worker 的
+drain deadline 按 selected set 的 p99 output length 计算，而该 set 含一个 36,034-token
+completion。该 request 因 deadline 被裁掉；尽管 TPOT-only pass rate 仍可达标，censoring
+会随 `sampling_u` 改变 selected set，不能用于比较 static-K frontier。
+
+因此原始 run 只保留为诊断 artifact，停止后改跑一个明确标为 **controlled** 的 screen：
+保持相同 arrival、prompt、sampling seed 和 topology，但把每个 request 的
+`min_tokens=max_tokens=4096`。4096 接近原始 output mean 3,924.6，且使 p99 deadline 覆盖
+每个 request 的完整 completion。它回答的是“在相同输入/到达条件下，static K 是否留下
+可利用空间”，不是 production trace 的最终 goodput 结论。最终论文实验必须同时有：
+
+1. 该受控 curve（机制和可重复性）；
+2. 原始长度 trace 的完整-completion 版本（不能使用 p99 censoring）；
+3. 至少一个 held-out window。
+
+### 2026-07-13 数据质量修正 #2：fresh-engine fixed grid
+
+受控 screen 的第一条 K=2、`u=0.0125` probe 本身通过了完整性检查（152/152 success、
+usage 返回的 completion token 均精确为 4096、无 early stop）。但随后发现原二分搜索会
+在同一 engine 内连续执行多个 probe；第二、三 probe 继承前一 probe 的 prefix/KV cache，
+也继承全局 RNG 的已消耗状态。不同 K 的二分分支/中止路径不同，因此不能把该搜索输出的
+`best_sampling_u` 差异直接归因于 K。该 run 在第二 probe 中主动停止，**不作为 G0 结果**。
+
+替代协议是每个 `(offered-load, K)` 只运行一次、每次均启动一个 fresh engine。两个固定
+负载由原始 immutable trace 的统一 `sampling_u` 阈值物化：`u=0.0125`（152 requests,
+0.2533 req/s）和 `u=0.0200`（263 requests, 0.4383 req/s）。物化后的 request 仍保留原
+prompt、arrival 和 sampling provenance，但强制 `temperature=0`、显式 engine `seed=0`，
+并用统一的 4096 completion override。每个 K 只有一个 probe，所以 accelerator KV/prefix
+cache 为空且 RNG 从相同 seed 开始；每次都从 `probe_details.jsonl` 验证：
+
+1. `early_stopped=false`；
+2. outcome count = selected count，且每个 request success；
+3. TTFT/TPOT 均非空；
+4. completion token 的 source 为 usage，且实际/预期均为 4096；
+5. result 无 partial-probe failure，且只包含一个 primary probe。
+
+运行顺序在两个负载间反向（ABBA），避免 K 与时间漂移完全共线。这个 grid 仍然只回答
+static-K 是否存在足够大的机会；它不估计 production sampling goodput，也不证明 rank-local
+K 的上界。当前 vLLM deployment 的 `reasoning_parser=''`；其 SSE 实现将生成文本放在
+`delta.content`，所以本协议中的 token-time 定义覆盖当前 `<think>` 输出。若以后启用
+reasoning parser，客户端必须同时记录 `reasoning_content` 后才可复用此指标。
+
+## G1：只有在 G0 通过后才做的直接验证
+
+目标不是“不同请求有不同 K”这种已经很常见的说法，而是验证下面的系统命题：
+
+> 在 DP+EP MoE 下，局部独立的 K 决策会让 collective 序列分歧；把它们编译为
+> rank-agreement 的 ragged execution plan，可保留异质请求的计算节省，同时不改变
+> collective order。
+
+当前 dash0 vLLM 已经有一个很好的切入点：每个 DP step 会 all-reduce 一段 metadata，
+并把各 rank 的 total token count padding 到最大值；CUDA graph mode 也会取跨 rank 的
+共同模式。这说明论文的最小机制不应另造一个 scheduler，而应把现有 scalar
+`(num_tokens, num_reqs, graph_mode)` agreement 扩展为 canonical speculative-plan header。
+关键增量是让 header 描述真实 active frontiers，并保证后续 verifier/EP split vector 的
+collective ordinal 相同；若最后仍 padding 到 global max，就没有可主张的性能机制。
+
+需要实现/测量：
+
+1. **oracle trace replayer**：利用 G0 的 per-K service curve，为每个到达时刻选择
+   SLO-feasible K，比较 best-static K 与 oracle 的 upper bound。若 oracle gain <10%，
+   停止，避免把噪声当论文方向。
+2. **collective trace**：按 DP rank 记录每个 decode step 的 collective 序列、token
+   shape、active-sequence mask、MoE all-to-all bytes 和 rank idle time。验证“local K
+   不同”是否真的导致 sequence divergence，而不是仅是一个 API 限制。
+3. **CollectiveSpec prototype**：固定 collective order，用全局 agreement header 和
+   ragged/padded verification plan；对比 `best static K`、global-max-K、oracle 和当前
+   upstream dynamic-spec baseline（包括 DSpark/FASER 能实现的部分）。
+4. **ablation**：去掉 agreement、去掉 ragged packing、去掉 queue/SLO policy；报告
+   goodput、p50/p95/p99 TPOT、acceptance、MoE communication bytes、GPU SM/HBM util、
+   rank skew。
+
+## 主要风险
+
+- 最新 upstream 动态投机对 DP>1 的处理可能本身只需一个 global-K broadcast；那是
+  feature patch，不构成研究贡献。
+- 当前 dash0 runtime 已验证 DP=2 + static EAGLE 可以工作；尚未在这个 binary 上证明
+  “local dynamic K 会 deadlock”。因此研究动机必须写成固定 EAGLE horizon 的执行限制，
+  不能把未运行的 dynamic-K 路径当作既成故障。
+- FASER/DSpark 等相邻工作会把“dynamic K + ragged verify”作为强 baseline；必须在
+  做任何大实现前进行逐项复现/排除。
+- trace 的 `sampling_u` 是 proxy；最终结论必须在固定 arrival trace、真实请求长度和
+  至少一个不同 workload 上复现。
+
+## 2026-07-13 系统重审：收紧主张和后续 gate
+
+本轮查阅 [vLLM Dynamic SD 限制](https://docs.vllm.ai/en/latest/features/speculative_decoding/dynamic_speculative_decoding/)
+与 [DSpark](https://arxiv.org/abs/2607.05147) 后，原先“dynamic K 会使 collective
+diverge，因此做 ragged verifier”这一表述太宽，不能作为实现前提：
+
+- DSpark 已公开主张按请求动态 verification length、跨请求 token flatten 与 ragged
+  physical execution；这些本身不再构成新颖性。
+- dash0 当前实际 import 的 vLLM wheel 与本地 checkout 不同。实验中的运行日志称
+  `v0.11.1`，而 wheel metadata 为 `0.13.0rc2.dev2111+gb44b43f43.d20260309`；后续任何
+  hook 必须对这个 live source 做 provenance pin，不能把本地 v0.24 API 当作证据。
+- 这个 live runtime 已原生使用 per-request `list[list[int]]` draft token IDs，并将真实
+  长度送入 scheduler/metadata；EP all-to-all 也具有 variable-split data path。因此
+  **per-request horizon 不等于 collective divergence**。必须先观测到不同 DP replica
+  的 collective call count、phase 或 branch trace 的真实不一致。
+
+修订后的唯一可能研究命题是更窄的：
+
+> 对共享一个 EP collective domain 的独立 DP scheduler，如何将异构 verification plan
+> 编译为可证明 liveness 的 canonical execution trace，同时在同一逻辑 plan 下回收
+> global physical-max padding 的关键路径成本。
+
+它有四个顺序严格的停止门槛：
+
+1. **P0 / 真实 premise**：以预先给定的 `k_i ∈ {0,1,2,3}` replay 表截断已生成的
+   EAGLE candidates，在每个 DP/TP rank 记录 target、EAGLE 和 EP phase signature。若
+   没有 trace mismatch、强制 global padding 或 liveness 问题，就停止把 plan header 当作
+   研究贡献。
+2. **P1 / 机会量**：即使存在 mismatch，也必须证明 rank-local oracle 相对 best static
+   或 globally synchronized K 有至少 10% 的 SLO-goodput headroom；全局同步若恢复 90%
+   以上 gap，则只值得做 upstream patch。
+3. **P2 / 因果物理对照**：同一个 `k_i` replay plan 必须对比
+   `PaddedSync-semantic`（物理 `N × (1 + max k_i)` rows）与 `CompactSync`
+   （物理 `Σ_i(1+k_i)` rows）。static K=3 不是 PaddedSync，因为二者 logical algorithm
+   不同。若实际 target/EP work 与关键路径未减少，停止。
+4. **P3 / 正确性与部署**：所有 TP peers 的 plan digest 相同、所有 EP ranks 观察到同一
+   header vector、每 epoch 的 target/EAGLE/MoE signature 一致；greedy output token-exact，
+   并通过 empty-rank、`{0,3}` 交替、长时间 stress。只有随后在两个 session-coherent
+   workload 上重现 SLO-goodput 才能讨论论文。
+
+若 P0--P3 任一项失败，合理结论是停止 CollectiveSpec，而不是继续调 controller、K 或
+queue policy。若全部通过，最小原型的边界也只应是 verifier-side compaction：当前
+EAGLE 仍按 Kmax 产生 candidate，短 `k_i` 不会自动消除 draft-side work，不能把 verifier
+节省误报为完整 draft+verify speedup。
+
+### Trace 数据可用性与 window-closure fallback
+
+本轮发现 `prepare_trace_windows.py` 依赖的 2026-03-27 原始格式化 trace span 在 dash0
+已不存在，不能把它的 streaming full-session root 当作已经复验。现有的完整 600 秒
+materialized window 仍含 prompt、`chat_id` 与 `parent_chat_id`，因此新增的 fallback 仅将
+窗口内图的 connected component 作为选择原子：保留 384/384 条窗口内 parent edge，且把
+同一个窗口外 parent 的 sibling 归为同一 component。该窗口有 15,479 requests、15,095
+components；414 条 non-root edge 中 30 条（7.25%）父节点落在窗口外。
+
+这只能称为 **window-session-closed**，不等于 full-session coherent：任何结果都必须报告
+这 30 条 boundary-parent residual，且不能据此声称跨窗口 KV reuse。若原始 span 恢复，必须
+重新从完整 source resolve root 与重新采样，不能沿用 fallback 的 score/threshold。
+
+## 2026-07-13 P0 v2：header/liveness premise 的实际结果
+
+### 先报异常
+
+两个 cell 都在 probe/result 已落盘、64 个请求都已完成后出现 teardown 异常。heterogeneous
+cell 有 `free(): corrupted unsorted chunks` 与共享资源泄漏；control 还出现 SIGTERM/SIG11
+和 TCPStore broken pipe。这些不是运行期 request/collective failure，但也意味着本实验**不
+证明干净退出或部署鲁棒性**。本节只使用完成前的 request、worker phase 与 DP metadata
+作为 P0 evidence；不报告任何 TPOT/QPS 比较。
+
+### 设计、判定与修正后的观测范围
+
+在 dash0 的 Qwen3-235B-A22B FP8 + EAGLE3、TP=4/DP=2/EP=8、DeepEP 配置上，P0 将 EAGLE
+已生成的 Kmax=3 candidates 按预先给定的 request-static 表截断为 `k_i ∈ {0,1,2,3}`。它只
+改变 verifier 可见 candidate，不消除 EAGLE 的 Kmax drafter 工作。
+
+原始 worker hook 还会记录 vLLM 的 profile/DP dummy run；空 `SchedulerOutput` 仍可能有
+physical rows。因此真实 target batch 的判据固定为：
+
+```text
+event == batch_execution_plan
+AND request_count > 0
+AND total_scheduled_rows > 0
+```
+
+最初 summary 将 676/640 条 dummy/profile record 混入 target phase，错误地把 control 的
+145/146 internal-call 差异解释为 rank mismatch。修正后的汇总只比较 target event，并将
+真实 DP pair 识别为 `[0,4]`、`[1,5]`、`[2,6]`、`[3,7]`；同一 logical DP replica 内的 TP
+group 则为 `[0,1,2,3]` 与 `[4,5,6,7]`。
+
+### 结果
+
+远端可复核 artifact：
+`/home/admin/cpfs/wjh/collectivespec-pilot/20260713T054328Z/p0_phase_v2_20260713T0944Z`
+（run source `bb698b5`；`summary.json`、`summary.md`、`driver_result.json` 和原始
+`p0_logs/*.jsonl` 均在该目录）。
+
+| cell | completion | 实际 candidate K | target worker records | target plan / DP coordination |
+|---|---:|---|---:|---|
+| control K=3 | 64/64，usage 均为 64 | `{3}` | 488；DP0 每 TP peer 65，DP1 每 peer 57 | 每个 logical DP replica 内序列完全一致；四个真实 DP pair 的 57 个 shared target epoch 的 scalar coordination signature 一致 |
+| heterogeneous | 64/64，usage 均为 64 | `{0,1,2,3}` | 528；8 个 peer 各 66 | 两个 logical DP replica 内序列完全一致；四个真实 DP pair 的 66/66 shared target epoch signature 一致 |
+
+heterogeneous 的 `candidate_truncate` 直方图为 `{0: 1408, 1: 670, 2: 672, 3: 398}`，而截断前
+全部为 K=3（共 3,148 个 candidate）。所以它不是只改变 log 的“伪异构”实验。两 cell 的
+target record 都满足：
+
+```text
+num_tokens_per_rank[dp_rank] == total_scheduled_rows
+physical_batch_rows == rows_across_dp[dp_rank]
+rows_across_dp[i] >= num_tokens_per_rank[i]
+len(rows_across_dp) == len(num_tokens_per_rank) == 2
+```
+
+这说明两个独立 scheduler 的 logical plan 可以不同，但 live runtime 已用 scalar DP metadata
+协调共同 physical shape，并让共享 EP domain 的真实请求完成；没有观察到运行期 deadlock 或
+collective error。它反驳的是“异构 verifier-side K 必须新增 canonical header 才能先保证
+liveness”的必要性，而不是一般性的形式化证明。
+
+### 留下的物理现象，以及为什么它仍不足以继续造系统
+
+heterogeneous 的 66 个 shared target epoch 中有 62 个的 raw DP token counts 不相等；现有
+runtime 将 `rows_across_dp` 同步为共同 shape。按每个 DP replica 的一个 TP anchor 计，
+target-only raw logical rows 为 6,276；逐 epoch 保留当前 TP/CUDA-graph local alignment 后为
+6,536；最终 physical rows 为 7,024。因此可单独归因给跨 DP global-max padding 的只有
+488 rows（6.95% physical rows）。此前用 7,024-6,276 得到的 748（10.65%）还混入了 260
+行本地 alignment，不能当作 compact 对照可回收的 DP work。control 的 row totals 也不与
+heterogeneous 直接比较，因为 logical plan 和 scheduler trajectory 不同。
+
+这只能看作 **P2 的 row-count upper bound**，绝不能把 control 与 heterogeneous 相减当作
+速度收益：两者 logical plan、scheduler trajectory 都不同。更关键的是 EAGLE drafter 仍完成
+Kmax 工作；即使理想 compact verifier 回收全部 6.95% 的 DP-only target rows，端到端
+SLO-goodput 增益也只会更小。
+
+因此决策为：
+
+1. **停止**把 canonical plan header / deadlock avoidance 当作 CollectiveSpec 的研究主线；
+   P0 已在目标部署上否定其必要前提。
+2. **停止**把“dynamic verification length + flattened ragged execution”本身当贡献；DSpark
+   已覆盖该组合，且它也指出固定长度 drafter 的前置工作不会因 verifier 截断自动消失。
+3. 仅保留一个很窄的、默认 no-go 的机会：同一 logical plan 下的 compact-vs-padded physical
+   execution。只有先以 P1 证明相对 best-static/global-sync 至少 10% E2E SLO-goodput，再以
+   P2 的因果对照证明关键路径 rows/bytes 真正下降，并完成 DSpark topology gap 审计，才值得
+   再投入实现。当前 P0 不满足这些条件。
+
+P0 未验证 greedy token-exact 输出、真实 DeepEP dispatch ordinal/split digest、取消/empty-rank
+stress，或其他模型/后端/更大 K 的泛化；这些都不能从本结果外推。
+
+### P0 data sanity
+
+- **teardown red flag 已单列**：control/heterogeneous 都在 completion 后发生 allocator/资源
+  清理异常；因此没有使用延迟、吞吐或 clean-shutdown 指标作结论。
+- n=2 cells，8 worker/cell，64 usage-verified completions/cell；completion count 的
+  min=max=64，distinct=1。
+- target worker record count：control=488、heterogeneous=528（min=488，max=528，distinct=2）；
+  dummy/profile records 分别为 676/640，已排除。
+- heterogeneous K：min=0，max=3，distinct=4；control K distinct=1。所有计数、rows 和
+  padding 非负；JSON parse errors=0。
+- heterogeneous 的 66 target epochs：两个 DP rank 的 raw row values 共 n=132，min=1、
+  max=77、distinct=35；physical rows n=66，min=4、max=80、distinct=14；分解
+  `6,276 <= 6,536 <= 7,024` 与 `7,024 - 6,536 = 488` 均成立。
+- 修正后的不变量均为 true：probe integrity、8 workers observed、每个 DP replica 内 target
+  phase/sequence 一致、target DP metadata 合法、DP coordination record 存在、四个真实 DP pair
+  的 shared scalar coordination signature 一致。

docs/fidelity-aware-harness-headroom-20260714.md

@@ -0,0 +1,135 @@
+# Fidelity-aware harness headroom audit
+
+Status: **HISTORICAL PREMISE DID NOT PASS PROSPECTIVE P1; NO CONTRIBUTION CLAIM**.
+
+The audit answers whether engine instrumentation has enough incremental signal
+to justify a prospective experiment.  It does not establish generalization.
+
+Post-run update: the exact-timestamp held-out P1 completed and failed the
+registered gate.  Under the stronger simulator-aware `k=2` end-to-end replay,
+telemetry preserved zero regret but saved only 1.426% online H20-hours versus
+sim top-k + real final.  The current route is closed; see
+`docs/fidelity-aware-harness-p1-report-20260714.md`.
+
+## Simulator shortlist lower bound
+
+On the frozen 12-cell SimFid task, the strongest calibrated SLO simulator
+reading places TP2/MNS32 and TP2/MNS64 in the same first tie bucket.  Real-final
+evaluation of that two-cell bucket selects TP2/MNS32 and has zero real regret.
+A method requiring a real calibration probe plus final verification cannot beat
+two real cell evaluations on this task.  Therefore “better initial selection”
+is not a viable claim here; the remaining headroom is shorter real verification
+inside the same shortlist.
+
+## Five-second prefix result
+
+The retrospective Phase-6 dataset contains 37 primary anchors across 12 cells.
+Stable labels use the frozen same-placement 2-of-3 adjudication: 28 feasible and
+9 infeasible.  Three TP4 primary measurements disagree with their repeated
+labels, so single-run feasibility is not treated as ground truth.
+
+Using leave-one-cell-out folds, identical L2 logistic models, and a 5-second
+prefix:
+
+| Metric | Outcome-only | Instrumentation-aware | Delta |
+|---|---:|---:|---:|
+| Accuracy | 78.38% | 89.19% | +10.81 pp |
+| Balanced accuracy | 70.63% | 81.55% | +10.92 pp |
+| Brier score | 0.1297 | 0.0901 | -0.0396 |
+| Correct only in this model | 0 | 4 | +4 |
+| McNemar exact two-sided p | — | 0.125 | not significant |
+
+At the frozen conservative threshold 0.95, both policies make zero false
+accepts and zero false rejects on this retrospective set.  Outcome-only safely
+cuts 36.35% of measured primary-trial cost; instrumentation-aware safely cuts
+61.10%, an additional 24.75 percentage points.  Regularization sensitivity for
+accuracy delta is `[0.00, +10.81]` percentage points, so the sign is
+non-negative but the magnitude is not stable.
+
+Longer prefixes do not strengthen the case monotonically.  At 10 seconds,
+headline accuracy is 91.89% outcome-only versus 89.19% instrumentation-aware;
+at 15 seconds it is 88.89% versus 91.67%; at 20 seconds it is 86.11% versus
+91.67%, but both 0.95 policies make one false reject.  Five seconds is therefore
+a training-selected operating point, not a test result.
+
+## Strong simulator-aware calibration baseline
+
+The original nested comparison used the same simulator shortlist but did not
+put Frontier's per-anchor prediction in either model.  A stronger retrospective
+audit now gives both models frozen-calibrated simulated throughput, simulated
+SLO pass rate, and simulated feasibility.  Under the same leave-one-cell-out
+folds, 5-second cutoff, L2 logistic family, regularization 1.0, and threshold
+0.95:
+
+| Metric | Sim + outcome | Sim + outcome + instrumentation | Delta |
+|---|---:|---:|---:|
+| Accuracy | 81.08% | 89.19% | +8.11 pp |
+| Balanced accuracy | 72.42% | 81.55% | +9.13 pp |
+| Brier score | 0.1058 | 0.0957 | -0.0101 |
+| Safe early decisions | 20/37 | 25/37 | +5 |
+| Valid full-trial cost reduction | 50.89% | 68.98% | +18.09 pp |
+| Residual verification H20-hours | 0.5240 | 0.3310 | -36.84% |
+
+Both 0.95 policies have zero false accept and zero false reject on this
+retrospective task.  Only three 0.5-threshold classifications differ in favor
+of instrumentation and none in favor of the strong baseline; McNemar's exact
+two-sided p-value is 0.25.  The cell-bootstrap accuracy-delta interval is
+`[0.00,+18.18]` percentage points.  The result is not robust to regularization:
+at 0.1 the strong baseline is more accurate and the instrumentation policy
+makes two unsafe decisions; at 10.0 the strong baseline is also more accurate.
+Thus the stronger comparison still has enough point-estimate headroom for a
+held-out test, but it materially weakens the evidence and makes a prospective
+task-level result mandatory.
+
+## Interpretation
+
+There is enough headroom to run a held-out pilot, but not enough evidence to
+claim the harness contribution:
+
+- the 5-second cost gap is operationally large;
+- only four paired classifications differ, so significance is absent;
+- all examples share one workload/SLO/engine task;
+- completion timestamps are reconstructed from arrival + TTFT + TPOT rather
+  than recorded directly;
+- three adjudication disagreements are concentrated in transient TP4 runs;
+- outcome-only already recovers the simulator shortlist oracle with very few
+  real cells.
+
+The next experiment must therefore freeze the 5-second model and threshold,
+record exact monotonic completions, use a held-out trace, and label each anchor
+with three full repetitions.  The registered protocol is
+`docs/fidelity-aware-harness-protocol-20260714.md`.
+
+## Artifacts
+
+- `runs/fidelity-headroom/analyze_existing.py`
+- `runs/fidelity-headroom/metrics.json`
+- `runs/fidelity-headroom/analyze_prefixes.py`
+- `runs/fidelity-headroom/prefix-metrics.json`
+- `runs/fidelity-headroom/test_analysis.py`
+- `runs/fidelity-headroom/test_prefix_analysis.py`
+- `runs/fidelity-headroom/analyze_strong_baseline.py`
+- `runs/fidelity-headroom/strong-baseline-metrics.json`
+- `runs/fidelity-headroom/test_strong_baseline.py`
+
+## Sanity block
+
+| Family | n | Min | Max | Distinct | Invariant/result |
+|---|---:|---:|---:|---:|---|
+| Real SimFid cell scores | 12 | 1.2833 | 3.2833 | 7 | Non-negative; not identical |
+| Prefix examples at 5 s | 37 | 5 s | 5 s | 1 expected | All 12 cells represented |
+| Adjudicated labels | 37 | 0 | 1 | 2 | 28 positive / 9 negative |
+| Primary/adjudicated disagreement | 37 | 0 | 1 | 2 | 3 TP4 disagreements retained |
+| Full primary elapsed time | 37 | 14.566 s | 62.064 s | 37 | Every 5 s prefix is in range |
+| Outcome probability | 37 | in `[0,1]` | in `[0,1]` | >1 | Checked before metrics |
+| Instrumentation probability | 37 | in `[0,1]` | in `[0,1]` | >1 | Checked before metrics |
+| Layer-1 streams | 12 | 14,174 records | 58,725 records | 12 | Contiguous, zero drops |
+| Matched frozen simulator anchors | 37 | pass rate 0.0688 | pass rate 1.0 | 12 pass-rate values | Every prefix matched exactly once |
+| Frozen simulator anchor corpus | 92 | positive throughput | positive throughput | >1 | No duplicate cell/anchor run |
+
+Checked invariants: same folds/model family and cutoff; no full verdict in a
+feature; prefix-only Layer-1 slicing; non-negative costs/counters; bounded
+ratios/probabilities; both labels present; per-config results not identical;
+tie expansion before top-k; no imputation of non-monotonic frontiers.  The main
+limitation is reconstructed request completion time, explicitly marked on all
+37 five-second examples.

docs/fidelity-aware-harness-p1-report-20260714.md

@@ -0,0 +1,242 @@
+# Fidelity-aware harness P1 result
+
+Status: **REGISTERED ROUTE REJECTED; DO NOT OPEN P2/P3 FOR THE CURRENT METHOD**.
+
+Date: 2026-07-14 (Asia/Singapore).
+
+## Outcome
+
+The registered five-second instrumentation-aware verifier did not pass P1.
+The stronger simulator-aware comparison also failed the independent
+contribution bar.  On the frozen `k=2` end-to-end replay:
+
+- `sim top-k + real final` selected the real oracle with zero regret;
+- instrumentation-aware also selected the oracle, but reduced online H20-hours
+  by only **1.426%** (1.329% when the prior failed attempt is added to both);
+- the required reduction was 30% versus full real final and 20% versus a safe
+  outcome-only calibrator;
+- the outcome-only calibrator was not safe: it rejected the true best cell, so
+  its apparent cost saving is not a deployable comparison.
+
+This rejects the claim that the **current joint logistic verifier**, trained on
+one historical workload, gives the harness an independent tuning contribution.
+It does not prove that engine telemetry contains no useful signal.  Telemetry
+improved held-out classification and removed unsafe decisions, but did not turn
+that signal into meaningful end-to-end tuning-cost reduction.
+
+## Frozen setup
+
+- Host: `dash0`, 8 NVIDIA H20 GPUs; cells were serialized and used TP1, TP2,
+  or TP4 without co-resident serving jobs.
+- Engine/model: patched vLLM 0.24.1.dev3, Qwen3-30B-A3B BF16.
+- Workload: held-out `chat_w20260312_1000`, seven disjoint repeat bands,
+  60-second replay after 0.1 time scaling, input `[0,8192]`, exactly 128 output
+  tokens.
+- SLO: stepped TTFT 2/4/6 seconds, TPOT 50 ms, request pass rate at least 0.95.
+- Cells: TP1/MNS8, TP1/MNS64, TP2/MNS8, TP2/MNS64, TP4/MNS16, TP4/MNS64.
+- Per cell: burn-in, three low-rate repeats, and three high-rate repeats.  The
+  first repeat supplied the five-second prefix; 2-of-3 supplied its label.
+- Models: the registered pair used config/workload/outcome versus the same
+  vector plus Layer-1 engine telemetry.  The strengthened pair additionally
+  gave both models identical frozen Frontier throughput, SLO pass-rate, and
+  feasibility predictions.
+- Policy: accept at `p>=0.95`, reject at `p<=0.05`, otherwise continue the same
+  trial.  Model, cutoff, threshold, role order, request hashes, and cap were
+  frozen before their applicable evaluation.
+
+The first launch failed its warm-up input-count validation before a measured
+anchor.  It cost 0.020552 H20-hours.  The corrected primary attempt cost
+1.722112 H20-hours, so aggregate campaign cost was **1.742664 H20-hours**, below
+the 3.5 cap.  The fix changed only warm-up validation; formal request counts and
+hash checks were unchanged.
+
+## P1 labels are not an artificial easy split
+
+The 12 adjudicated anchor labels contain 7 feasible and 5 infeasible examples.
+They are not simply “low feasible, high infeasible”:
+
+- TP2/MNS64 high was feasible in all three repeats;
+- TP4/MNS64 low and high were feasible in all six repeats;
+- TP4/MNS16 low and high were infeasible in all six repeats.
+
+That last pair creates a large real MNS interaction under an otherwise matched
+TP4 configuration.  Frontier correctly predicted TP4/MNS64 high as feasible,
+but incorrectly predicted TP4/MNS16 low as feasible.  It also incorrectly
+predicted TP1/MNS64 high as feasible.  Overall simulator-only feasibility was
+10/12 correct: 83.33% accuracy, with two false-feasible predictions and no
+false-infeasible prediction.
+
+The two false-feasible cases expose the intended latent-state problem.  At five
+seconds, all 26 completed TP4/MNS16-low requests and all 9 completed
+TP1/MNS64-high requests still passed their SLO, although both full anchors were
+infeasible.  External outcomes had not yet exposed the future failure; queue,
+running-batch, and scheduler state existed before the tail outcome.  This is
+mechanistic evidence that instrumentation can be useful, not evidence that the
+current learned policy uses it well enough.
+
+## Registered and strengthened prefix results
+
+At the frozen 0.95 policy threshold:
+
+| Comparison | Accuracy | Balanced acc. | Early decisions | False accept | False reject | Valid primary-trial saving |
+|---|---:|---:|---:|---:|---:|---:|
+| Registered outcome-only | 41.67% | 50.00% | 6/12 | 0 | 2 | invalid |
+| Registered + telemetry | 66.67% | 71.43% | 4/12 | 0 | 0 | 11.44% |
+| Strong sim + outcome | 66.67% | 68.57% | 5/12 | 0 | 1 | invalid |
+| Strong sim + outcome + telemetry | 83.33% | 85.71% | 4/12 | 0 | 0 | 11.44% |
+
+For the strong pair, telemetry was correct on two examples where the baseline
+was wrong and lost none; McNemar's exact two-sided p-value is 0.5 at `n=12`.
+This is a safety/classification improvement, not a cost contribution.  The
+registered instrumentation policy made two fewer early decisions than its
+baseline, so it failed the registered `+3 decisions or +15 percentage points`
+incremental gate.
+
+The result is not robust to the frozen regularization sensitivity:
+
+| L2 lambda | Sim+outcome acc. | +telemetry acc. | Base policy errors | Telemetry policy errors | Base saving | Telemetry saving |
+|---:|---:|---:|---:|---:|---:|---:|
+| 0.1 | 41.67% | 75.00% | 4 | 2 | invalid | invalid |
+| 1.0 | 66.67% | 83.33% | 1 | 0 | invalid | 11.44% |
+| 10.0 | 83.33% | 83.33% | 0 | 0 | 0.00% | 5.98% |
+
+Consequently the positive classification delta is neither statistically nor
+hyperparameter robust.
+
+## End-to-end shortlist result
+
+Frontier's simulator-feasible ranking on the tested P1 surface was:
+
+| Rank | Cell / anchor | Sim throughput/GPU | Real feasible | Real offered goodput/GPU |
+|---:|---|---:|---:|---:|
+| 1 | TP4/MNS64 high | 3.0718 | yes | 3.1250 |
+| 2 | TP1/MNS64 high | 2.8823 | no | 2.9833 |
+| 3 | TP2/MNS64 high | 2.8096 | yes | 2.8750 |
+| 4 | TP4/MNS16 low | 2.0866 | no | 2.1250 |
+| 5 | TP1/MNS8 low | 1.9806 | yes | 2.0333 |
+| 6 | TP2/MNS8 low | 1.8637 | yes | 1.9083 |
+
+The real oracle was TP4/MNS64 high at 3.125 req/s/GPU.  Cost includes an
+inferred per-cell startup/warm-up/burn-in/cleanup component and the selected
+anchor; benchmark-only 2-of-3 annotation intervals are removed.  Gaps around
+annotation runs remain in the shared setup term, making this a conservative
+method-cost estimate.
+
+| Frozen policy (`k=2`) | Online H20-hours | + prior failure | Real regret | Safety | Saving vs full |
+|---|---:|---:|---:|---|---:|
+| Sim top-2 + real final | 0.281383 | 0.301935 | 0.00% | valid | — |
+| Sim + outcome prefix | 0.214664 | 0.235216 | no selected cell | 1 false reject | invalid |
+| Sim + outcome + telemetry | 0.277370 | 0.297922 | 0.00% | valid | **1.426%** |
+
+For `k=1`, the simulator already picked the oracle and telemetry abstained, so
+its saving was exactly 0%.  For `k=3`, telemetry remained correct but saved
+1.007%.  The result therefore does not depend on an unfavorable choice of `k`.
+
+Across all six cells, a generous replay that removes benchmark-only repeat
+intervals estimates 0.944244 H20-hours for the full online workflow.  The
+instrumentation policy saved 0.043467 H20-hours: 11.44% of primary trial time,
+but only **4.603%** of end-to-end online cost.  Even a post-hoc oracle symmetric
+threshold cannot make the current telemetry model reach the contribution bar;
+its best zero-error envelope saves at most 5.69% of inferred online cost.  A
+strong outcome-only model at another post-hoc regularization/threshold can save
+16.13%, which further prevents attributing a unique advantage to telemetry.
+These oracle-threshold numbers are diagnostics only and are not test evidence.
+
+## Why the learned verifier did not generalize
+
+The training corpus has only 37 anchors from one workload/SLO task.  P1 shows
+large covariate shift:
+
+- sim+outcome: 12/192 feature values exceed 3 training standard deviations and
+  4 exceed 5; maximum absolute z-score is 10.36;
+- sim+outcome+telemetry: 19/396 exceed 3 and 9 exceed 5;
+- the largest shifts include admitted input-length mean (10.36), waiting state
+  (7.77), running maximum (6.38), and decode-batch maximum (6.08).
+
+Coefficient attribution shows that the input-length feature dominates several
+wrong feasible-anchor logits.  Because all training examples share one task,
+the joint classifier can learn incidental within-task correlation and override
+a correct simulator prior on TP2/MNS64-high and TP4/MNS64-high.  This is a
+supported diagnosis of model/data insufficiency; it is not a causal proof that
+one feature alone caused the P1 failure.
+
+More importantly, retuning lambda, threshold, features, or cutoff on P1 and
+then calling P1 a held-out result would violate calibration/evaluation
+separation.  P1 may now be used only as development data.
+
+## Decision and the only defensible reopening condition
+
+Do not run registered P2/P3 with the current model.  It failed the predeclared
+gate on the favorable primary-trial denominator and is even farther from the
+bar under end-to-end cost.  Spending six-task headline GPU budget on the same
+method would be metric shopping, not replication.
+
+A new route may be opened only as a new hypothesis:
+
+1. Replace the joint classifier with a **simulator-residual verifier**.  The
+   simulator prediction remains an explicit prior; nested outcome-only and
+   telemetry models learn when that prior is wrong, rather than freely
+   relearning feasibility and overriding it under workload shift.
+2. Train on multiple complete workload/SLO tasks.  SLO thresholds and target
+   pass rate must be explicit inputs; splits are by complete task.
+3. Calibrate abstention with task-level risk control.  No threshold is selected
+   on a headline task, and “never early decide” is included as the safe
+   outcome-only baseline.
+4. Treat Phase 6 and P1 as development only, freeze the residual architecture,
+   features, cutoff, threshold, simulator reading, and `k`, then use entirely
+   new trace windows for a new gate.
+
+This reopening is justified only if development data show both (a) the
+simulator's errors are predictable from pre-outcome engine state and (b) a
+simulator-preserving residual model does not corrupt correct simulator
+predictions.  It is a new project decision, not a continuation automatically
+authorized by P1.
+
+## Benchmark audit
+
+| Audit item | Verdict | Severity | Evidence / disposition |
+|---|---|---|---|
+| Calibration set separate from P1 | PASS | — | Phase 6/0311 trained; P1/0312 tested |
+| Strong simulator-aware baseline | PASS | — | Identical Frontier features in both nested models |
+| Sim top-k + real-final E2E baseline | PASS | — | Frozen `k=2`, tie expansion, measured setup/continuation cost |
+| Multiple independent headline tasks | NEEDS EVIDENCE | Blocking for a positive claim | P1 gate failed; P2 correctly not opened |
+| Statistical significance | NEEDS EVIDENCE | Blocking for a positive claim | n=12 anchors from one task; McNemar p=0.5 |
+| Hyperparameter robustness | FAIL | Blocking | Lambda sensitivity changes safety and relative result |
+| Full resource accounting | PASS for P1 | — | Failures, startup/warm-up/burn-in, continuation and annotation separated |
+| Avoid post-test retuning | PASS only if route stops | Blocking if violated | P1 is now development-only |
+| Selective winning-workload reporting | PASS | — | Negative P1 and TP/MNS losing cases retained |
+
+Overall recommendation: **Block the current independent harness contribution
+claim.**
+
+## Artifacts
+
+- Registered protocol: `docs/fidelity-aware-harness-protocol-20260714.md`
+- Historical headroom: `docs/fidelity-aware-harness-headroom-20260714.md`
+- Registered P1 analysis: `runs/fidelity-headroom/analyze_pilot.py`
+- Strong P1 analysis: `runs/fidelity-headroom/analyze_strong_pilot.py`
+- E2E shortlist replay: `runs/fidelity-headroom/analyze_pilot_e2e.py`
+- External immutable result root:
+  `/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714`
+
+## Data sanity block
+
+| Data | n | Min | Max | Distinct | Invariant |
+|---|---:|---:|---:|---:|---|
+| P1 labels | 12 | 0 | 1 | 2 | 7 feasible / 5 infeasible |
+| Primary elapsed seconds | 12 | 19.448 | 61.435 | 12 | Every five-second prefix is in range |
+| Prefix Layer-1 records | 12 | 332 | 557 | 12 | Contiguous; zero drops |
+| Exact timestamped outcomes | 12 anchors | 54 | 750 | 11 | Monotonic completion timestamps |
+| Simulator pass rate | 12 | 0.1548 | 1.0 | 7 | Ratios in `[0,1]` |
+| Strong nested probabilities | 24 | 0.000208 | 0.809422 | 24 | Ratios in `[0,1]` |
+| E2E cost components | 36 | 0.001389 | 0.169653 H20-h | 21 | Non-negative |
+| GPU attempts | 2 | 0.020552 | 1.722112 H20-h | 2 | Aggregate 1.742664 < 3.5 |
+| Copied raw files | 191 | — | 153,093,348 bytes total | — | Remote/local aggregate SHA identical |
+
+Checked invariants: six cells and twelve anchors; exact request count and
+request-ID/arrival/length hashes; all cell validation flags true; both labels
+present; probabilities bounded; costs and counters non-negative; simulator
+results not all identical; committed simulator rerun 12/12 numerically
+identical to the exploratory run; no prompt text in public simulator fixtures;
+no co-resident serving process; final eight GPUs at 0 MiB and 0% utilization.
+No red flag remains.

docs/fidelity-aware-harness-protocol-20260714.md

@@ -0,0 +1,231 @@
+# Fidelity-aware real-verification harness protocol
+
+Status: **P1 FAILED; P2/P3 CLOSED FOR THE REGISTERED METHOD; CONTRIBUTION NOT ESTABLISHED**.
+
+Date frozen: 2026-07-14 (Asia/Singapore).
+
+Post-run disposition (2026-07-14): P1 completed with valid data but failed its
+registered incremental gate.  The strengthened simulator-aware comparison and
+end-to-end `k=2` replay also failed: instrumentation was safe and retained zero
+regret, but reduced online H20-hours by only 1.426% versus sim top-k + real
+final, against the 30% bar.  Outcome-only was unsafe.  P2/P3 are therefore not
+opened for this model.  Full results and the permitted reopening condition are
+in `docs/fidelity-aware-harness-p1-report-20260714.md`; the protocol below is
+retained unchanged as the pre-run record.
+
+## Research question and contribution bar
+
+The harness has an independent systems contribution only if engine-internal
+instrumentation improves a tuning decision beyond what is already achievable
+with a simulator shortlist and external benchmark outcomes.  The intended
+claim is therefore deliberately stronger than “telemetry explains a run”:
+
+> Given the same simulator ranking, the same candidate order, and the same
+> short real-GPU probe, a learned instrumentation-aware verifier reaches a
+> configuration with at most 5% real SLO-goodput regret using materially fewer
+> H20-hours than both (a) simulator top-k followed by full real evaluation and
+> (b) an outcome-only verifier given exactly the same probe.
+
+The paper-facing gate is:
+
+- at least 20% lower real-verification H20-hours than outcome-only calibration;
+- at least 30% lower real-verification H20-hours than simulator top-k plus full
+  real final evaluation;
+- paired 95% task-bootstrap confidence interval for the outcome-only cost
+  reduction strictly above zero;
+- selected-configuration SLO-goodput regret at most 5% on every headline task;
+- no false-safe early accept in the pilot and at most 1% in the expanded suite;
+- profiling, warm-up, confirmation, instrumentation, and failed-run costs are
+  included rather than amortized away.  An amortized profile-cost view may be
+  reported only as a secondary result.
+
+If these conditions fail, instrumentation remains a debugging facility.  It is
+not an independent tuning-harness contribution.
+
+## What is learned, and what is not a rule
+
+The decision target is a stable, repeated real verdict, not a hand-authored
+diagnosis such as “queue length above N means reject.”  Each anchor receives
+three full real repetitions and a frozen 2-of-3 feasibility label.  A nested
+pair of regularized models predicts that label from a fixed prefix:
+
+- **Outcome-only input X:** configuration, offered rate, admitted/completed
+  progress, observed TTFT/TPOT margins, failures, and known workload lengths.
+- **Instrumentation input Z:** the same X plus generic engine state: running and
+  waiting queues, decode-batch shape, KV usage, graph mode and padding, prefill
+  share, preemptions, and model-step rate.
+
+Both models use the same L2 logistic family, train split, standardization,
+regularization, cutoff, and probability threshold.  The only experimental
+difference is Z.  The initial family is intentionally simple: a positive result
+then demonstrates value in the engine signal rather than capacity in a larger
+learner.  A sequence model is admissible only as a later, paired ablation.
+
+### Amendment A1: strengthen the calibration baseline before P2
+
+Frozen 2026-07-14 13:08 Asia/Singapore, after P1 launch but before P1
+completion or analysis.  A baseline audit found that the first frozen P1
+models use the simulator only to define candidate order; their feature vectors
+do not contain the simulator's per-anchor prediction.  This is insufficient
+for the stronger term **outcome-only calibration**.  P1 therefore remains a
+prospective test of the originally frozen cross-workload predictor, but cannot
+by itself open a contribution claim.
+
+For P2/P3, both nested models must additionally receive the identical frozen
+simulator outputs available at that decision: predicted completed throughput
+per GPU, predicted SLO pass rate, and predicted feasibility.  The comparison
+is consequently `sim + config + workload + real outcome prefix` versus that
+exact vector plus real engine state.  Simulator features, regularization,
+cutoff, and thresholds are frozen before any P2 task.  If telemetry does not
+improve this stronger baseline, the harness has no independent contribution.
+
+The same audit also separates algorithm cost from benchmark-oracle cost.
+Headline method cost includes every action the method would execute online:
+simulator profiling/calibration, model onboarding, server startup, warm-up,
+real prefix, continuation after abstention, method-requested confirmation,
+logging overhead, failures, and cleanup.  Exhaustive real-oracle runs and the
+extra repetitions used only to construct 2-of-3 evaluation labels are common
+benchmark annotation cost; they are reported separately and charged to no
+method.  A second, deliberately conservative table adds that common cost to
+all methods.  This prevents both hiding real method cost and making the
+percentage gate mathematically depend on offline ground-truth annotation.
+
+The frozen first policy uses a 5-second prefix, L2 regularization 1.0, and a
+two-sided abstaining threshold of 0.95: accept at `p(feasible)>=0.95`, reject at
+`p(feasible)<=0.05`, otherwise continue the exact same trial to completion.
+Threshold and cutoff were selected on the historical training task and are
+therefore not evidence; all claims come from subsequent held-out tasks.
+
+## Fair baselines
+
+| Method | Simulator | 5-second real prefix | External outcomes | Engine state | Full real continuation |
+|---|---:|---:|---:|---:|---:|
+| Real-only oracle | no | no | full | optional diagnostic | every candidate/anchor |
+| Sim top-k + real final | yes | included in full run | full | no decision use | every shortlisted candidate/anchor |
+| Outcome-only calibration | yes, including its prediction features | yes | yes | no | only on abstention |
+| Instrumentation-aware | same prediction features | yes | yes | yes | only on abstention |
+
+Tie buckets are expanded before top-k.  `k` is selected on training tasks and
+is fixed on held-out tasks; an oracle per-task k is forbidden.  Outcome-only
+receives all information available outside the engine, including config,
+workload, and frozen simulator-prediction features.  Instrumentation cannot use
+any record submitted after the cutoff.  The full label, confirmation votes,
+realized simulator error, and later requests are never model features.
+
+## Staged experiment
+
+### R0: historical premise and headroom audit
+
+The frozen SimFid surface has 12 cells.  The strongest calibrated SLO simulator
+reading has a top tie bucket `{TP2/MNS32, TP2/MNS64}`; full real evaluation of
+those two cells already finds the oracle with zero regret.  Consequently this
+single task cannot demonstrate a selection-count advantage: any method needing
+one real calibration probe and one real final verification has a lower bound of
+two real cells.
+
+The viable estimand is instead the duration and number of full real frontier
+evaluations inside a fixed shortlist.  Historical Phase-6 prefixes are analyzed
+only as training/premise data.  Their request completion times are reconstructed
+from arrival, TTFT, TPOT, and token count, so they cannot support a final claim.
+
+### P1: exact-timestamp prospective pilot
+
+- Engine/model/hardware: patched vLLM 0.24.1.dev3, Qwen3-30B-A3B, one solo
+  server/client on dash0, NVIDIA H20, `TP in {1,2,4}`.
+- Held-out workload: `chat_w20260312_1000`, 60-second replay after the frozen
+  0.1 time scale, raw input `[0,8192]`, exactly 128 output tokens.
+- SLO: stepped TTFT 2/4/6 seconds, TPOT 50 ms, 95% request pass rate.
+- Cells: TP1/MNS8, TP1/MNS64, TP2/MNS8, TP2/MNS64, TP4/MNS16, TP4/MNS64.
+- Per cell: one attainable low offered rate near 0.85x the historical v0.24
+  frontier and one high rate near 1.25x.  The exact threshold and selected
+  request hashes are frozen by a CPU preflight before launch.
+- Each cell uses a fresh server, the accepted long-request warm-up, one
+  unmeasured full-window burn-in, then three repetitions per rate.  Rate order
+  alternates and reverses across cells to prevent a fixed warm-state/order
+  confound.
+- The first repetition supplies the exact prefix.  All three repetitions supply
+  the 2-of-3 label.  Every request records a monotonic completion timestamp;
+  Layer-1 records are cut at the same monotonic boundary.
+- Placement is serialized.  Co-location is forbidden because Phase 6 observed
+  up to 92.86 percentage-point pass-rate shifts under co-location.
+- Hard cap: 3.5 H20-hours, including startup, warm-up, burn-in, all repetitions,
+  failures, and cleanup.  Projected cap violation stops before the next cell.
+
+P1 opens P2 only if all data invariants pass and instrumentation-aware has zero
+false accept/reject, is no worse than outcome-only, and either makes at least
+three additional correct early decisions or improves total valid trial-cost
+reduction by at least 15 absolute percentage points.  The pilot is a gate, not
+paper evidence.
+
+### P2: held-out task replication
+
+If P1 passes, freeze the model and run at least six independent task groups:
+three trace windows spanning distinct date/slot combinations and two SLO
+regimes.  No task used for threshold/model selection enters the headline test.
+The candidate surface is the full 12-cell `TP={1,2,4} x MNS={8,16,32,64}`
+surface.  Splits are by complete task, never by anchor or request.  A task-level
+paired bootstrap (10,000 repetitions, fixed seed) estimates cost and regret
+intervals.  Non-monotonic or split 2-of-3 anchors remain explicit; no frontier
+is imputed.
+
+### P3: end-to-end shortlist and search replay
+
+For each P2 task, run the same frozen simulator and tie-expanded top-k policy.
+Replay the real binary/frontier search under all three verification policies:
+full real, outcome-only, and instrumentation-aware.  The policy consumes only
+prefixes that would have been available at that decision point.  Report:
+
+- selected cell and real SLO-goodput regret;
+- number of real cells, anchors, and confirmations;
+- measured H20-hours and wall time;
+- false accept, false reject, and abstention counts;
+- profile, startup/warm-up, probe, full-continuation, confirmation, logging, and
+  failure cost breakdowns.
+
+### P4: simulator-rank-error attribution
+
+This phase distinguishes an outdated implementation/profile from a structural
+simulator limitation.  For each held-out task compare:
+
+1. the original simulator/profile;
+2. a version-matched re-profiled simulator;
+3. a trajectory-conditioned run supplied with the realized arrival and request
+   length sequence;
+4. outcome-only residual calibration;
+5. instrumentation-aware residual calibration.
+
+The engine trace is extended only as needed with a worker-level step UID and
+CUDA-event duration, because current async submit-to-complete spans overlap and
+are not GPU step time.  Residuals are decomposed into operator-profile error,
+scheduler/state error, and run-to-run noise.  If re-profiling alone restores the
+ranking, the old 30% loss was an implementation/profile defect.  If exact
+profiles and realized trajectories still mis-rank cells, and the residual is
+systematically explained by queue/KV/graph/batch state unavailable to the
+simulator, that is evidence of a structural state-abstraction gap.  Correlation
+alone is not called causal.
+
+## Failure modes that reject the route
+
+- Outcome-only matches or beats instrumentation-aware under the same cutoff.
+- Instrumentation gains average accuracy but introduces false-safe decisions.
+- Gains disappear under task-level rather than request/anchor-level splitting.
+- Savings come only from excluding startup, warm-up, profiling, confirmations,
+  or failed trials.
+- A different cutoff/threshold must be selected after seeing each test task.
+- The simulator top-k baseline already reaches the target with equal or lower
+  total H20-hours.
+- Exact instrumentation overhead exceeds 1% throughput or materially changes
+  p95/p99 latency.
+- Results depend on TP4 transient/non-monotonic trials and do not replicate on
+  held-out tasks.
+
+## Data sanity contract
+
+Every analysis ends with n, min/max, distinct count, label balance, and these
+invariants: non-negative counters/costs; probabilities and ratios in `[0,1]`;
+per-config results not all identical; timestamps monotonic; every prefix record
+at or before its cutoff; selected request ID/arrival/length hashes stable across
+repetitions; exact 128-token completion or counted failure; no dropped Layer-1
+records; 2-of-3 labels reproducible; no co-resident GPU process; total H20-hours
+below the hard cap; final GPUs idle.  A red flag is reported first and blocks
+the contribution claim.

docs/harness-ablation/bad-start-robustness-suite-20260626.md

@@ -0,0 +1,122 @@
+# Bad-start robustness suite - 2026-06-26
+
+本文定义 P0 bad-start robustness 的分布级验证。它不是新的 claim 结果，而是下一轮实验的
+pre-registration：先固定 starts、指标和 pass/fail，再运行，避免根据单个 case 调规则。
+
+## 当前前提
+
+已完成的代码 gate：
+
+- normalized full-config signature；
+- materialized effective signature：runtime-only proposal 先继承 incumbent topology 再签名；
+- CLI hard-veto：LLM/manual/harness proposal 在进入 trial 前禁止重复 effective config；
+- CandidateSet audit：`candidate_set_hash`、eligible/blocked candidates、blocked reason summary；
+- sidecar persistence：`harness/candidate-set-*.json`。
+
+已通过的单 case：
+
+```text
+TP8, DP1, gmu0.5, max-num-seqs8
+  -> TP4
+  -> TP4 + gmu0.9
+```
+
+这个 case 只能证明 sentinel recovery，不能证明分布级 robustness。
+
+## 实验矩阵
+
+使用同一 Qwen30B-A3B community vLLM 0.20 bounded replay setup、no-LLM harness、
+`search.auto_high.enabled=true`。先跑 fresh trusted-start control，得到同 commit 下的
+参考值 `R_ref`。
+
+| Group | N | Initial starts | 证明点 |
+| --- | ---: | --- | --- |
+| trusted control | 1 | 可信/default start | 定义 `R_ref` |
+| topology-only | 4 | `(TP,DP)=(8,1),(4,2),(1,4),(2,4)`，runtime nominal | 证明不是只会 `TP8 -> TP4` |
+| runtime-only | 4 | `TP2/DP1` with `gmu={0.50,0.70}` and `max-num-seqs={8,16}` | 证明 runtime floor/admission recovery |
+| combined | 4 | `TP8/gmu0.70/mns16`, `TP4/DP2/gmu0.50/mns8`, `TP1/DP8/gmu0.50/mns16`, `TP2/DP4/gmu0.70/mns8` | 证明 operators 可串联 |
+| held-out random | 8 | fixed-seed stratified samples over legal topology x low/nominal `gmu` x low/normal `mns`，排除已通过 sentinel | overfit denominator |
+
+总计：1 control + 20 novel bad starts。
+
+## Primary metrics
+
+- best-so-far SLO-feasible `req/s/GPU / R_ref`；
+- time-to-95%-reference；
+- normalized AUC over trial budget；
+- final pass rate；
+- executed normalized full-config repeat count；
+- no-op blocked count；
+- candidate family / operator sequence；
+- stop reason and `candidate_set_hash`。
+
+每个 run 必须保留：
+
+```text
+state.json
+proposals/*.json
+harness/candidate-set-*.json
+trials/trial-*/trial_spec.json
+trials/trial-*/result.json
+```
+
+## Pass/fail
+
+Run-level pass：
+
+```text
+best_so_far_req_per_gpu >= 0.95 * R_ref within 12 measured trials
+pass_rate >= 0.95
+executed_effective_config_repeat_count == 0
+no harness stop while high-priority eligible candidates remain
+```
+
+Suite-level pass：
+
+```text
+20 / 20 novel bad starts pass.
+```
+
+如果任一 novel start 失败，不能 claim distribution-level bad-start robustness。修复后必须
+冻结失败分析，并重新抽 held-out random set。
+
+## Overfit guards
+
+- pre-register all starts and random seed；
+- 不把已通过的 exact `TP8,gmu0.5,mns8` sentinel 计入 20-case denominator；
+- 不在 starts 之间调 threshold；
+- 报告 operator names，例如 `topology_bracket`, `topology_redistribute`,
+  `runtime_floor_jump`, `admission_recovery`，而不是 case-specific action；
+- 每次 stop 必须引用 `candidate_set_hash` 和 no high-priority eligible candidate evidence。
+
+## GPU cost
+
+Expected：
+
+```text
+21 runs * 6-8 measured trials = 126-168 trials
+```
+
+Hard cap：
+
+```text
+21 runs * 12 measured trials = 252 trials
+```
+
+按当前 Qwen30B bounded replay 粗估：
+
+```text
+15-35 min / measured trial
+expected = 250-780 H20 GPU-hours
+cap      = 500-1175 H20 GPU-hours
+```
+
+因此建议先跑 3-case pilot：
+
+| Pilot case | 起点 | 目的 |
+| --- | --- | --- |
+| topology-only | `TP=4, DP=2, gmu=0.9, mns=64` | 检查不是只会处理 TP8 |
+| runtime-only | `TP=2, DP=1, gmu=0.5, mns=8` | 检查 runtime floor/admission recovery |
+| combined | `TP=1, DP=8, gmu=0.5, mns=16` | 检查 topology + runtime 串联 |
+
+Pilot 通过后再启动完整 20-case suite。

docs/harness-ablation/bad-start-stop-counterexample-20260626.md

@@ -0,0 +1,372 @@
+# Bad-start stop counterexample - 2026-06-26
+
+本文记录一次有意构造的 adversarial bad-start 测试。它的目的不是证明 harness 已经
+robust，而是攻击当前实现，确认它是否会从明显不合理的初始配置中恢复。
+
+结论：
+
+```text
+当前 production/prototype harness 还不能支持 bad-start robustness claim。
+
+它会在高 GPU、高 TP 的坏起点上被 search_high_saturated_by_incumbent 提前 stop，
+没有测试 topology/resource-efficiency contrast。
+```
+
+这不是一个需要补 `TP=8 -> TP=4` 特例规则的问题。它暴露的是更基础的 stop authority
+问题：measurement saturation 不能绕过 coverage-relative candidate set。
+
+同时，这个反例也暴露了 measurement policy 的缺口：`search.high` 太小时，tuning 会被
+offered-load ceiling 右截断。后续应该加入 `auto_search_high`，但它只能在已有 trace
+sampling space 内自动校准；如果 `search.high=1.0` 仍然不能压到真实 capacity frontier，
+系统必须主动报告 measurement ceiling 不足，并等待人类确认是否更换 trace、提高 trace
+density 或启用额外负载生成方式。
+
+## 实验设置
+
+机器：`dash1`，8x H20。
+
+目标：从一个故意不合理的初始配置开始：
+
+```text
+tensor-parallel-size = 8
+data-parallel-size = 1
+gpu-memory-utilization = 0.5
+max-num-seqs = 8
+LLM endpoint disabled
+```
+
+期望行为：
+
+- harness 不应只因为 baseline feasible 就停止；
+- 它至少应生成 topology/resource-efficiency contrast candidate；
+- 对 `req/s/GPU` 目标，8 GPU incumbent 需要被低 GPU 或邻域 topology probe 验证。
+
+## Run A: 低 search.high
+
+第一轮保留原始 `search.high=0.125`。
+
+结果：
+
+```text
+trial-0001 completed
+harness-stop-0002
+tuning_stop_reason = harness_stop
+validator reason = search_high_saturated_by_incumbent
+best request_rate = 1.0333 total
+best request_rate_per_gpu = 0.1292
+pass_rate = 1.0
+```
+
+解释：这个 run 的 offered-load ceiling 太低，baseline 很容易 saturate `search.high`。
+因此它不能区分“配置真的足够好”和“测量上限太低”。
+
+## Run B: corrected high search ceiling
+
+第二轮把 `search.high` 提到 `1.0`，保留同一个 bad-start 配置，`max_trials=3`。
+
+远端产物：
+
+```text
+session = adv_badcase_corr_casea_20260626T095356Z
+store = /home/admin/cpfs/wjh/aituner/aituner/.aituner/adversarial-badcase-corrected-casea-20260626T095356Z
+spec = /home/admin/cpfs/wjh/aituner/aituner/.aituner-run-configs/adversarial-badcase-corrected-casea-20260626T095356Z/casea-combined-bad-highsearch.json
+log = /home/admin/cpfs/wjh/aituner/aituner/.aituner/adversarial-badcase-corrected-casea-20260626T095356Z.log
+```
+
+结果仍然是在 baseline 后 stop：
+
+```text
+trial-0001 completed
+harness-stop-0002
+no harness-proposal-0002.json
+tuning_stop_reason = harness_stop
+validator reason = search_high_saturated_by_incumbent
+best sampling_u = 0.9375
+best request_rate = 8.033333333333333
+best request_rate_per_gpu = 1.0041666666666667
+pass_rate = 1.0
+```
+
+Probe trace：
+
+| sampling_u | request_rate | feasible |
+| --- | ---: | --- |
+| 0.5 | 4.6000 | true |
+| 0.75 | 6.5167 | true |
+| 0.875 | 7.5000 | true |
+| 0.9375 | 8.0333 | true |
+
+它触发 stop 的原因是当前 guard 计算：
+
+```text
+binary_probe_resolution = max(tolerance, (high - low) / 2**max_probes)
+                        = 0.0625
+threshold_gap_to_high = 1.0 - 0.9375
+                      = 0.0625
+```
+
+因此当前实现认为 incumbent 已经 saturate `search.high`。
+
+## 为什么这是反例
+
+当前 objective 是 SLO-constrained `req/s/GPU`，不是固定 8 GPU 的 total throughput。
+一个 8-GPU incumbent saturate offered-load ceiling，并不能证明：
+
+- 低 TP / 低 GPU 配置没有更高 `req/s/GPU`；
+- 当前 topology 是资源效率最优；
+- runtime knobs 已经进入合适 trust region；
+- no-LLM harness 能从 bad start 中恢复。
+
+所以这个 stop 是 unsound 的，至少相对于 bad-start robustness claim 是 unsound。
+
+更形式化地说：
+
+```text
+search_high_saturated_by_incumbent
+  does not imply
+incumbent_validated(topology/resource-efficiency)
+```
+
+当目标包含 resource efficiency，并且 parallel-size/topology 仍然 tunable 时，
+`search_high_saturated_by_incumbent` 只能作为 measurement evidence，不能单独作为 stop
+authority。
+
+## 对新 harness 设计的约束
+
+这个反例直接约束 declarative harness：
+
+1. Stop 前必须生成并持久化完整 `CandidateSet`。
+2. Stop proof 必须引用 `candidate_set_hash`。
+3. 如果存在未覆盖的 high-priority topology/resource-efficiency candidate，validator
+   必须返回 `eligible_candidates_remain`，即使 incumbent saturate `search.high`。
+4. `search.high` saturation 只能更新 measurement coverage，不能替代
+   `incumbent_validated`。
+5. 对 `req/s/GPU` objective，required coverage 必须包含至少一个 topology 或
+   resource-efficiency contrast，除非 StudySpec 明确固定 GPU budget 和 topology。
+
+Measurement policy 约束：
+
+1. `auto_search_high` 可以根据 trace 的 sampling threshold 和目标 GPU 规模自动提高
+   `search.high`，避免低 ceiling 让所有 config 过早 saturate。
+2. 自动校准不能越过 trace 原生上限。当前 `sampling_u` 语义下，`search.high=1.0`
+   表示完整 trace。
+3. 如果完整 trace 仍然被 incumbent 轻松 saturate，validator 不能假装搜索完成；它应该
+   输出 `measurement_ceiling_insufficient` 或把该事实作为 stop proof 的阻塞项。
+4. 系统不得自动使用重复窗口、合成 arrivals 或 replay scaling 来扩大 workload，除非
+   StudySpec 显式启用，或人类确认该实验要测 synthetic/offline stress regime。
+5. `measurement_ceiling_insufficient` 和 `eligible_candidates_remain` 是不同问题：前者说
+   load ceiling 不足，后者说 mechanism coverage 未完成。二者任一存在，都不能把结果
+   写成 bad-start robustness 成功。
+
+这也说明当前 repair 方向不能是：
+
+```text
+if tp == 8 and gmu == 0.5: try tp = 4
+```
+
+正确方向应该是：
+
+```text
+ordered topology lattice + resource-efficiency objective
+  -> candidate set includes lower/redistributed topology contrast
+  -> stop is blocked until that coverage unit is measured or invalidated
+```
+
+## 当前 verdict
+
+当前 production harness：
+
+```text
+prototype, not yet fundamental
+```
+
+新的 declarative prototype：
+
+```text
+promising substrate, but not production-proven
+```
+
+它已经把 `CandidateSet`、`CoverageUnit`、failure region 和 coverage-relative stop 的最小
+接口跑通，但还没接入真实 tuning loop，也还没证明 bad-start distribution 的收敛。
+
+因此接下来的 P0 gate 是：
+
+```text
+先实现 coverage-relative stop authority，再重跑 bad-start distribution。
+```
+
+## 2026-06-26 implementation validation
+
+Commit `c8a0f98` 实现了第一片 production 修复：
+
+- `search.auto_high` schema，默认关闭，旧配置兼容；
+- trial materialization 时在已有 trace sampling space 内 resolve effective `search.high`；
+- `trial_spec.json` 和 `result.json` 写入 auto-high / measurement evidence；
+- `search_high_saturated_by_incumbent` 降级为 measurement evidence；
+- 对 `req/s/GPU` 且 topology 可变的 study，high saturation 不能直接授权 stop；
+- 固定 GPU product 但 TP/DP redistribution 可调时，仍视为 topology 可变；
+- auto-high ceiling 低于 `search.low` 时不生成非法 search interval。
+
+本地验证：
+
+```text
+PYTHONPATH=src python3 -m unittest discover -s tests
+Ran 143 tests OK
+```
+
+dash1 validation：
+
+```text
+run label = adversarial-badstart-autohigh-c8a0f98-20260626T122622Z
+git sha   = c8a0f9870eac5438fb19be8edf1534a893723ab9
+machine   = dash1, 8x H20
+```
+
+Spec 仍使用 bad-start：
+
+```text
+tensor-parallel-size = 8
+data-parallel-size = 1
+gpu-memory-utilization = 0.5
+max-num-seqs = 8
+search.auto_high.enabled = true
+```
+
+Auto-high resolution：
+
+```text
+original_high       = 1.0
+effective_high      = 0.9979913161468553
+trace_max_sampling_u = 0.9979913161468553
+reason              = search_high_lowered_to_trace_ceiling
+```
+
+结果：
+
+| trial | config patch | best sampling_u | request_rate | req/s/GPU | pass |
+| --- | --- | ---: | ---: | ---: | ---: |
+| trial-0001 | baseline TP8, DP1, gmu0.5, mns8 | 0.935616858887 | 8.00 | 1.0000 | 1.0000 |
+| trial-0002 | `tensor-parallel-size=4` | 0.810867944369 | 6.95 | 1.7375 | 0.9784 |
+| trial-0003 | `tensor-parallel-size=8` | 0.935616858887 | 8.00 | 1.0000 | 1.0000 |
+
+关键结论：
+
+```text
+旧 failure 已被修复：
+baseline 后不再产生 harness-stop-0002/search_high_saturated_by_incumbent。
+
+新实现产生 harness-proposal-0002，并测试 TP4 topology contrast。
+TP4 将 best req/s/GPU 从 1.0000 提高到 1.7375。
+```
+
+这证明第一片修复解决了“measurement saturation 绕过 topology coverage”的问题。
+
+但是 trial-0003 暴露了新 blocker：
+
+```text
+当前 no-repeat 仍基于 patch signature，而不是 normalized full-config signature。
+```
+
+`tensor-parallel-size=8` 对这个 study 的 base config 是 no-op，等价于 baseline TP8，
+但系统仍把它当成一个新 proposal 执行。这说明下一片 P0 必须实现：
+
+1. normalized full-config signature；
+2. CandidateSet snapshot，包含 eligible 和 blocked candidates；
+3. blocked reason，例如 `blocked_noop_equivalent_to_tested_full_config`；
+4. Stop/report 中同时呈现 `measurement_ceiling_*` 和 `eligible_candidates_remain`。
+
+因此当前 verdict 更新为：
+
+```text
+P0 measurement/stop-order slice: passed.
+P0 full coverage-relative harness: not yet passed.
+```
+
+## 2026-06-26 normalized full-config validation
+
+Commit `48911b6` 修复了上一节暴露的新 blocker：no-repeat 不再只比较 patch
+signature，而是比较 normalized effective full config。
+
+实现语义：
+
+```text
+effective_config =
+  normalize(base_envs + env_patch,
+            base_flags + flag_patch)
+
+no_repeat_signature = stable_json(effective_config)
+```
+
+因此下面两个 proposal 在 validator 看来是同一个 full config：
+
+```text
+baseline patch: {}
+noop patch:    {"tensor-parallel-size": 8}
+```
+
+本地验证：
+
+```text
+PYTHONPATH=src python3 -m unittest discover -s tests
+Ran 145 tests OK
+```
+
+dash1 validation：
+
+```text
+run label = adversarial-badstart-fullsig-48911b6-20260626T133112Z
+git sha   = 48911b658bbf052d70d952d1cdf55ad6b50ba7a5
+machine   = dash1, 8x H20
+```
+
+Spec 仍使用同一个 adversarial bad-start：
+
+```text
+tensor-parallel-size = 8
+data-parallel-size = 1
+gpu-memory-utilization = 0.5
+max-num-seqs = 8
+search.auto_high.enabled = true
+LLM endpoint disabled
+```
+
+结果：
+
+| trial | proposal | best sampling_u | request_rate | req/s/GPU | pass |
+| --- | --- | ---: | ---: | ---: | ---: |
+| trial-0001 | baseline TP8, DP1, gmu0.5, mns8 | 0.935616858887 | 8.00 | 1.0000 | 1.0000 |
+| trial-0002 | `tensor-parallel-size=4` | 0.810867944369 | 6.95 | 1.7375 | 0.9832 |
+| trial-0003 | `tensor-parallel-size=4`, `gpu-memory-utilization=0.9` | 0.935616858887 | 8.00 | 2.0000 | 1.0000 |
+
+关键 observation：
+
+```text
+旧 trial-0003:
+  {"tensor-parallel-size": 8}
+  -> 等价于 baseline，但仍被执行
+
+新 trial-0003:
+  {"tensor-parallel-size": 4, "gpu-memory-utilization": 0.9}
+  -> 在已验证 TP4 topology 上继续测试 KV/cache headroom
+```
+
+这证明 normalized full-config signature 已经阻止了 patch-level no-op 重测。
+
+机制解释：
+
+1. baseline TP8 saturate search ceiling 只被记录为 measurement evidence；
+2. 因为 objective 是 `req/s/GPU`，topology/resource-efficiency contrast 仍未覆盖，所以
+   validator 不允许 stop；
+3. harness 先测试相邻低 TP topology，TP4 把 `req/s/GPU` 从 `1.0` 提高到 `1.7375`；
+4. no-repeat 用 full config signature block 掉等价 TP8 patch；
+5. harness 在 settled TP4 topology 上继续测试 runtime headroom，`gmu=0.9` 把
+   `req/s/GPU` 提高到 `2.0`。
+
+当前 verdict 更新为：
+
+```text
+P0 measurement/stop-order slice: passed.
+P0 normalized full-config no-repeat slice: passed.
+P0 single adversarial bad-start recovery: passed for this case.
+P0 distribution-level bad-start robustness: not yet proven.
+```

docs/harness-ablation/candidate-family-gap-log.md

@@ -0,0 +1,37 @@
+# Candidate Family Gap Review Log
+
+本文档维护 LLM 在 `advisory` 模式下提出 harness candidate set 之外配置、且该配置带来性能提升时的人工 review 入口。
+
+运行时系统不会自动修改 harness，也不会把 LLM 的 out-of-set proposal 直接提升为规则。每条提升先写入 study artifact：
+
+```text
+.aituner/<study>/candidate_family_gaps/<trial-id>.json
+```
+
+然后人工 review 决定是否需要修改：
+
+- `KnobDescriptor`
+- generic operator
+- acquisition / step-size policy
+- evidence estimator
+
+## Gap 分类
+
+| 类型 | 含义 | 默认处理 |
+|---|---|---|
+| `same_operator_new_step` | harness 已有同 knob / 同方向候选，但 LLM 给了更好的 step/value | 优先检查 trust-region、step-size、candidate budget 和 acquisition |
+| `missing_operator` | visible candidate set 中没有同 knob 或同 mechanism 的候选 | 检查是否缺 generic operator 或 descriptor 映射 |
+| `missing_descriptor` | knob 不在 adapter descriptor 中 | 需要 engine adapter review |
+| `missing_mechanism` | 现有机制词表无法表达该 proposal 的作用 | 需要 design review |
+| `llm_independent_discovery` | LLM 发现无法归入当前 harness abstraction 的新方向 | 只作为 observation，不直接进入 harness |
+
+## Review 原则
+
+1. 不接受 case-specific 数值表，例如“这个 case 试 `max-num-seqs=24`”。
+2. 若归类为 `same_operator_new_step`，只能修改通用 step policy，例如 grow/shrink factor、local grid budget、bracket 触发条件。
+3. 若归类为 `missing_descriptor`，descriptor 只能表达 knob 语义、约束、search geometry 和 directional effects，不能表达具体目标答案。
+4. 任何被接受的 gap 都需要新增 synthetic test，证明它不依赖 vLLM 常见取值或某个 bad-start case。
+
+## Pending
+
+当前 repo 内尚无已人工接受的 candidate family gap。实验产生的 JSON artifact 需要在这里补充 review 摘要后再进入代码设计。

docs/harness-ablation/harness-vs-naive-20260616.md

@@ -0,0 +1,99 @@
+# Harness vs naive (use_harness on/off) — convergence ablation — 2026-06-16/17
+
+Controlled ablation of the paper's "harness" (domain-knowledge knob-family steering):
+the same agentic loop with `llm.use_harness=true` vs `false` (= the paper's naive
+agentic tuner: free-form LLM proposals, no `Harnesses:` prompt section, no
+deterministic guided proposals, no Stop-B validator/veto). Same workload, model, SLO,
+substrate — the only difference is `use_harness` (configs
+`dash0_qwen27b_ablation_harness_on.json` / `..._naive_off.json`, verified to differ
+only in that flag + study_id).
+
+- Model: dense Qwen3.5-27B, vLLM 0.11.1, 8×H20 (dash0 and dash1 share the cpfs mount).
+- Workload: chat 0–8k, length-aware TTFT SLO (4s + L_in/8k) + TPOT ≤ 50 ms, pass ≥ 95%.
+- Substrate (process comparison, not absolute peak-rate): `replay_time_scale=0.5`,
+  `completion_tokens_override=128`, Stop-A on, `search.high=0.25`, 6 probes, max-trials 6,
+  **`--skip-baseline`** (the low-capacity TP1 auto-baseline is infeasible under this
+  SLO+compression and would trip `baseline_all_infeasible`; skipping it lets both loops
+  climb from their first proposal).
+- This measures the tuning *process* (which knob family, convergence speed, stop
+  discipline), not a validated peak-rate.
+
+## Harness ON — converged in 2 iterations, then stopped
+| iter | proposer | config | per_gpu | outcome |
+| --- | --- | --- | --- | --- |
+| 1 | LLM (harness-guided) | TP2 | 0.247 | feasible |
+| 2 | harness (deterministic) | **TP4** | **0.340** | feasible — incumbent |
+| 3 | harness | TP4 + chunked-prefill + mbt=16384 | 0.333 | worse → rejected |
+| (—) | LLM | `should_stop` | — | **VETOED** ("decode TPOT still the bottleneck; adjacent probes weak") |
+| 4 | LLM | TP2 + DP2 | 0.194 | worse → rejected |
+| (—) | LLM | `should_stop` | **STOP** | honored after veto budget |
+
+Best **TP4 @ 0.340**; iters-to-best = **2**; ran **4 trials then stopped** (Stop-B +
+one veto of a premature stop); no regression.
+
+## Naive OFF — nondeterministic; reaches the optimum slowly at best, fails at worst
+
+The naive (free-form) `gpt-5.4` loop behaved very differently across two runs — it has
+no harness steering and no stop logic:
+
+**Run A (dash0, interrupted by an outage at trial-5):** kept **TP=1** the whole time and
+cycled runtime knobs (`max-num-batched-tokens` 16k→65k, `max-num-seqs`, caching). All
+trials **infeasible** (same `tpot>50` + `ttft>budget`), trial-4 **crashed the engine**
+(OOM at mbt=65536). Found **no feasible config** in 5 trials — never tried raising TP.
+
+**Run B (dash1, full budget):**
+| iter | config | per_gpu | note |
+| --- | --- | --- | --- |
+| 1 | TP2 | 0.247 | feasible |
+| 2 | TP2 + max-num-seqs=32 | 0.218 | worse |
+| 3 | TP2 + mbt=12288 | 0.218 | worse |
+| 4 | TP2 (re-proposal) | 0.218 | no gain |
+| 5 | TP2 + gpu-mem-util=0.85 | 0.218 | worse |
+| 6 | **TP4** | **0.340** | reaches the optimum — at the last trial |
+
+Best **TP4 @ 0.340** — the *same* optimum as the harness — but iters-to-best = **6**,
+it used the **entire budget with no early stop**, and trials 2–5 were detours (TP2 +
+runtime tweaks, all worse than trial-1) before it stumbled onto TP4.
+
+## Comparison
+
+| | Harness ON | Naive OFF (B, dash1) | Naive OFF (A, dash0) |
+| --- | --- | --- | --- |
+| best per-GPU | 0.340 (TP4) | 0.340 (TP4) | none (failed) |
+| iters-to-best | **2** | 6 | — |
+| trials used | **4 (stopped)** | 6 (full budget, no stop) | 5 (interrupted) |
+| stopped early? | yes (Stop-B + veto) | no | — |
+| wasted trials | 2 (post-best refinements) | 4 (TP2+runtime detours) | 5 (runtime-only, infeasible) |
+| path to optimum | direct (TP2→TP4) | slow (TP2→runtime detour→TP4) | wrong family (runtime on TP1) |
+
+## Interpretation (honest)
+
+The bottleneck is **compute** (decode TPOT + prefill queueing), which only a
+compute-adding knob (**tensor parallelism**) fixes. Findings:
+
+1. **A strong frontier model can sometimes find the right knob unaided** — naive run B
+   eventually reached TP4 = 0.34, the same optimum as the harness. This matches the
+   paper's own caveat (§7.3): stronger models reduce, but do not remove, the need for
+   structured guidance. So the harness's value is **not** "naive always fails."
+2. **The harness's value is reliability, speed, and stop discipline.** With the harness:
+   converged in **2 iters** and **stopped at 4** (recognized convergence; vetoed a
+   premature stop). Naive: **3× slower** to the same answer (6 iters), **never stopped**
+   (burned the full budget on detours), and in run A **failed outright** — never tried
+   TP, found nothing, crashed the engine. Naive is **nondeterministic and unreliable**;
+   the harness is fast, monotone (no regression), and self-terminating.
+3. This reproduces the paper's Figure-18 story: the harness converges in a few
+   iterations and stops, while the naive agentic tuner wastes the budget (and can fail
+   to converge entirely).
+
+## Caveats
+
+- Compressed substrate (scale=0.5, out=128) → per-GPU numbers are *process* comparators,
+  not validated peak-rates; the convergence behavior is the result. (The TP4 optimum
+  reproduced at 0.340 across the harness run and naive run B, a useful consistency check.)
+- One run per arm per host; naive is nondeterministic (runs A and B differ markedly),
+  which is itself part of the finding. The harness arm's deterministic guided proposal
+  (TP4 at iter 2) and validator veto are reproducible.
+- Infra notes: dash0 (LLM-gateway reachable) went down mid-experiment; dash1 shares the
+  cpfs and ran the completion. The codex `config.toml` points at a dash0-local proxy
+  (`127.0.0.1:11235`); on dash1 the LLM endpoint must be reached directly (set empty
+  `*_proxy` env) — see `scripts/run_naive_d1.sh`.

docs/harness-ablation/knob-conditional-effects-20260705.md

@@ -0,0 +1,192 @@
+# Knob conditional effect 证据整理
+
+本文整理 2026-07-01 到 2026-07-03 在 `dash1` 上跑的 interaction screening 结果，用来支持一个具体论点：
+
+> Serving tuning knobs 不是彼此独立的。一个 knob 的收益方向和收益大小依赖当前 topology、admission/concurrency 和 scheduler context，因此不能假设“逐个 knob tune 到最好”一定可靠。
+
+## Presentation review: 应该怎么展示
+
+原来的 delta summary 能证明 `Delta_knob(context)` 不同，但它不够直观，因为它没有展示 tuning algorithm 会怎么失败。更适合作为 paper 主图的是：
+
+1. **主图：OAT path counterexample**  
+   在同一个 measured response surface 上画两条 one-knob-at-a-time 路径。读者能直接看到：同一个起点、不同单维 tuning 顺序，会停在不同点，而且其中一个是 coordinate-wise local optimum。
+2. **补充图：interaction residual**  
+   用 additive model residual 形式说明：如果 TP 和 MNS 是独立贡献，残差应接近 0；实际残差有结构性正负块。
+3. **补充图：delta/context summary**  
+   保留为形式化证据，但不作为主图，因为它不能直接展示 OAT 的路径依赖。
+
+因此本文推荐把 `knob-oat-counterexample-c1-qwen30b` 作为主文图，把 C3 crossed lines 和 residual/delta 放在 appendix 或机制分析图中。
+
+## 图 1：OAT path counterexample
+
+![OAT counterexample](figures/knob-oat-counterexample-c1-qwen30b.png)
+
+数据来源：
+
+- `interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z`
+- `interaction-mixed-qwen30b-tp4-mns-nocap-qps20-dash1-d8899c5-20260701T161900Z`
+
+这张图直接展示为什么“逐个维度独立 tune”不可靠。我们从同一个起点 `TP=1, MNS=8` 出发：
+
+| Strategy | Path | Final req/s/GPU |
+|---|---|---:|
+| tune MNS first, then TP | `TP1,MNS8 -> TP1,MNS16 -> TP4,MNS16` | `2.44` |
+| tune TP first, then MNS | `TP1,MNS8 -> TP2,MNS8 -> TP2,MNS32` | `3.28` |
+
+`TP4,MNS16` 是一个 measured coordinate-wise local optimum：
+
+- 固定 `TP=4` 调 `MNS`：`MNS16/32/64` 都是 `2.44`，没有 strictly improving move；
+- 固定 `MNS=16` 调 `TP`：`TP4=2.44` 高于 `TP1=2.35` 和 `TP2=2.27`；
+- 但全局最好点 `TP2,MNS32=3.28` 比它高 `25.6%`。
+
+这比单纯说 “MNS 的 delta 依赖 TP” 更有力：它展示了一个实际 tuning path 如何被独立维度假设带到次优点。要从 `TP4,MNS16` 逃到 `TP2,MNS32`，tuner 必须允许非独立的 context-aware move，或者至少维护 frontier/plateau 上的反事实 anchor；单维 greedy OAT 不够。
+
+这里的结论不是“所有 workload 都有强 interaction”，而是更严格地说：
+
+1. 在真实 case 中确实存在明显 conditional effect；
+2. 这个现象足以否定 naive one-knob-at-a-time/OAT 作为通用 tuning strategy；
+3. harness 需要维护 mechanism-aware context，而不是把 knobs 当作独立维度。
+
+## Formal definition
+
+记某个 engine config 的 SLO-feasible objective 为：
+
+```text
+f(config) = max request_rate_per_gpu subject to pass_rate >= target
+```
+
+对 knob `x` 的一个 intervention `x_low -> x_high`，在 context `c` 下的效果定义为：
+
+```text
+Delta_x(c) = f(x_high, c) - f(x_low, c)
+```
+
+如果存在两个 context `c1, c2`，使得：
+
+```text
+Delta_x(c1) != Delta_x(c2)
+```
+
+则说明 knob `x` 存在 conditional effect。若符号也变化，比如一个 context 下提升、另一个 context 下降，则是更强的 interaction。
+
+## 图 2：C1 Qwen30B mixed workload surface
+
+![C1 Qwen30B surface](figures/knob-conditional-c1-qwen30b-surface.png)
+
+数据来源：
+
+- `interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z`
+- `interaction-mixed-qwen30b-tp4-mns-nocap-qps20-dash1-d8899c5-20260701T161900Z`
+
+关键观察：
+
+| Context | `MNS=8 -> 32` 的 req/s/GPU 变化 |
+|---|---:|
+| `TP=1` | `2.10 -> 2.28`, `+8.7%` |
+| `TP=2` | `2.28 -> 3.28`, `+44.3%` |
+| `TP=4` | `1.28 -> 2.44`, `+90.3%` |
+
+这说明 `max-num-seqs` 的收益强烈依赖 `tensor-parallel-size`。同一个 `MNS` 调整在 `TP=1` 下只是小幅提升，在 `TP=2/4` 下变成决定性能上限的关键 knob。
+
+反过来看，`TP` 的收益也依赖 `MNS`：
+
+- 在 `MNS=8` 时，`TP=4` 是坏点，只有 `1.28 req/s/GPU`；
+- 在 `MNS=32` 时，`TP=2` 变成全局最优附近，达到 `3.28 req/s/GPU`。
+
+因此，如果 tuner 固定 `MNS=8` 去判断 topology，会错误低估 `TP=4`，也会无法看到 `TP=2 + MNS=32` 的最佳区域；如果固定 `TP=1` 去调 `MNS`，又会低估更高 TP 下 concurrency knob 的价值。这就是 OAT order sensitivity。
+
+## 图 3：C1 additive residual
+
+![C1 interaction residual](figures/knob-interaction-residual-c1-qwen30b.png)
+
+如果 `TP` 和 `MNS` 可以独立建模，一个简单 additive model：
+
+```text
+f(TP, MNS) ~= base + effect(TP) + effect(MNS)
+```
+
+应该留下接近 0 的 residual。实际 residual 最大达到约 `0.46 req/s/GPU`，而且呈现结构性模式：
+
+- `TP2,MNS32/64` 是正 residual，说明这个组合比独立效应相加更好；
+- `TP2,MNS16` 和 `TP4,MNS8` 是强负 residual，说明某些组合显著低于独立假设预测。
+
+这张图适合放在机制/appendix 中，用数学形式支持“不是独立 knob effect”。
+
+## 图 4：C3 Qwen235B decode workload
+
+![C3 Qwen235B decode lines](figures/knob-conditional-c3-qwen235b-decode-lines.png)
+
+数据来源：
+
+- `interaction-qwen235b-decode-c3-topo-mns-mbt-fixed-dash1-d8899c5-20260703T022514Z`
+
+完整 8 点结果：
+
+| Config | req/s/GPU | pass rate |
+|---|---:|---:|
+| `TP4 DP2 EP8 MNS64 MBT256` | `0.0535` | `1.0000` |
+| `TP4 DP2 EP8 MNS64 MBT384` | `0.0535` | `0.9922` |
+| `TP4 DP2 EP8 MNS128 MBT256` | `0.0590` | `0.9929` |
+| `TP4 DP2 EP8 MNS128 MBT384` | `0.0590` | `0.9929` |
+| `TP2 DP4 EP8 MNS64 MBT256` | `0.0590` | `0.9753` |
+| `TP2 DP4 EP8 MNS64 MBT384` | `0.0535` | `0.9961` |
+| `TP2 DP4 EP8 MNS128 MBT256` | `0.0590` | `0.9788` |
+| `TP2 DP4 EP8 MNS128 MBT384` | `0.0590` | `0.9823` |
+
+关键观察：
+
+- `MBT 256 -> 384` 在 `TP4/DP2 + MNS64` 下没有收益；
+- 同一个 `MBT 256 -> 384` 在 `TP2/DP4 + MNS64` 下反而下降约 `9.2%`；
+- `MNS 64 -> 128` 在 `TP4/DP2` 下提升约 `10.1%`；
+- 同一个 `MNS 64 -> 128` 在 `TP2/DP4 + MBT256` 下没有收益，但在 `TP2/DP4 + MBT384` 下恢复约 `10.1%`。
+
+这说明 runtime knobs 的作用不是单调独立的。`MBT` 是否有害取决于 topology 和 `MNS`；`MNS` 是否有用也取决于 topology 和 `MBT`。
+
+## 图 5：Delta 形式的直接证据
+
+![Delta summary](figures/knob-conditional-delta-summary.png)
+
+这张图把上面的论证直接转成 `Delta_x(context)`：
+
+- C1 中，同样是 `MNS 8 -> 32`，收益从 `+8.7%` 到 `+90.3%` 不等；
+- C3 中，同样是 `MBT 256 -> 384`，有的 context 是 `0%`，有的 context 是 `-9.2%`；
+- C3 中，同样是 `MNS 64 -> 128`，有的 context 是 `0%`，有的 context 是 `+10.1%`。
+
+这就是 conditional effect 的直接测量证据。
+
+## C2 是边界案例，不是反例
+
+C2 Qwen235B prefill tight SLO 的结果更弱：
+
+- `TP4` family: `0.1067~0.1175 req/s/GPU`；
+- `TP8` family: `0.1727 req/s/GPU`；
+- 在测过的 `MNS={64,128}`、`MBT={8192,16384}` 网格里 runtime knobs 基本平。
+
+这个 case 说明并不是每个 workload 都会在 runtime knobs 上表现出强 interaction。它的主要结论是 topology 主导：`TP8` 相比 `TP4` 约 `+47% req/s/GPU`。
+
+这对 paper framing 反而有用：我们的 claim 不应该是“所有 knobs 总是强耦合”，而应该是：
+
+> Tuning system 不能预设 knobs 独立；它必须通过 measured response 判断当前 case 是 topology-dominant、runtime-interaction-dominant，还是 flat/noisy。Harness 的作用是把这些 measured evidence 维护成 search context。
+
+## 对 harness 设计的含义
+
+这些图支持我们当前 framing：
+
+1. Harness 不应该只做单 knob local search。它需要保留 topology/runtime context，并允许 joint or projected interventions。
+2. Candidate generation 不能只说“把某个 knob 调大/调小”，而要说明这个 intervention 所依赖的 context。
+3. Validator 不能只比较 raw request rate；必须比较 SLO-feasible `request_rate_per_gpu`，并保存 negative evidence。
+4. LLM/planner 的价值不应被描述成“猜一个更好的 knob 值”，而是基于 harness 提供的 measured context 去提出 plausible joint moves。
+
+## 复现图
+
+```bash
+python3 scripts/plot_knob_conditional_effects.py
+```
+
+输出：
+
+- `docs/harness-ablation/figures/knob-oat-counterexample-c1-qwen30b.png`
+- `docs/harness-ablation/figures/knob-interaction-residual-c1-qwen30b.png`
+- `docs/harness-ablation/figures/knob-conditional-c1-qwen30b-surface.png`
+- `docs/harness-ablation/figures/knob-conditional-c3-qwen235b-decode-lines.png`
+- `docs/harness-ablation/figures/knob-conditional-delta-summary.png`

docs/harness-ablation/no-llm-harness-mechanism-20260625.md

@@ -0,0 +1,482 @@
+# No-LLM Harness Mechanism - 2026-06-25
+
+Status note, 2026-06-26:
+
+本文记录的是当前 rule-based prototype harness 的 no-LLM 机制和已有实验现象。它能证明
+AITuner 可以在没有 LLM endpoint 的情况下闭环运行，但不能证明 harness 的完备性、
+通用 robustness 或最终系统贡献。最终目标设计已经调整为 declarative intervention
+grammar + coverage-relative validator，见
+[`declarative-intervention-harness-design-20260626.md`](declarative-intervention-harness-design-20260626.md)。
+
+本文回答一个核心问题：如果不调用 LLM，harness 为什么还能自动找到配置？
+
+结论先说清楚：no-LLM 模式下并不是“没有 planner”。当前 harness 本身就是一个
+deterministic planner。LLM 在 AITuner 里只是一个可替换的 proposal backend；当
+harness 能从观测、瓶颈归因、候选 family 和 stop validator 中推出下一步时，tuning
+loop 会直接使用 harness proposal，而不会请求 LLM。
+
+## Tune loop 中 LLM 的位置
+
+`study tune` 每轮的决策顺序是：
+
+```text
+state + study spec + workload/probe results
+        |
+        v
+build_harness_context(...)
+        |
+        +--> build_harness_stop_proposal(context)
+        |       if true: write harness-stop and exit
+        |
+        +--> build_harness_guided_proposal(context)
+        |       if true: run this deterministic proposal
+        |
+        +--> call_llm_for_proposal(...)
+                only if no harness stop/proposal exists
+```
+
+因此在 `study.llm.endpoint = null` 的 no-LLM run 中，只要 harness 每轮都能给出
+一个 deterministic proposal 或 deterministic stop，整个实验就可以完全不调用 LLM。
+如果 harness 既不能 propose 也不能 stop，且没有 LLM endpoint，AITuner 会报错，而不是
+偷偷退化成随机搜索。
+
+当前 Qwen30B stopfix run 就是这种完整闭环：
+
+```text
+.aituner/qwen30b-harness-only-medium-stopfix-dash1-20260624T144701Z/
+```
+
+它没有 LLM endpoint，但仍完成了 9 个 measured trials，并最终由 validator 写出
+`harness_stop`。
+
+## Harness 做的不是 prompt engineering
+
+Harness 做的事情可以形式化成：
+
+```text
+H = (O, B, G, S, V)
+
+O: Observation schema
+   将 workload、trial probes、SLO failure、launch failure、topology constraints
+   转成结构化状态。
+
+B: Bottleneck attribution
+   将 SLO violation 归因到 serving regime，例如 ttft_prefill、decode_tpot、
+   admission_or_queueing、launch_or_memory。
+
+G: Intervention grammar
+   将 raw knobs 组织成有语义的 candidate families，例如 topology、batching、
+   sequence admission、KV memory headroom。
+
+S: Scoring policy
+   对候选 intervention 评分，选择最有信息量且最可能提升 SLO-constrained
+   req/s/GPU 的下一步。
+
+V: Validator / stop policy
+   阻止非法、重复、已知失败或无意义的 proposal；只有在剩余高价值候选被测完后
+   才允许 stop。
+```
+
+LLM 可以读取这些结构化信息并生成 proposal，但 no-LLM 时 `H` 自己就能生成
+proposal。换句话说，我们的核心是把：
+
+```text
+raw config vector search
+```
+
+转成：
+
+```text
+mechanism-guided intervention search
+```
+
+这就是为什么没有 LLM 也能工作。
+
+## Agent loop 流程图
+
+```mermaid
+flowchart TD
+  A[Baseline or latest measured trial] --> B[Load probe history and trial result]
+  B --> C[Build workload L-C-A profile]
+  B --> D[Build TrialProfile]
+  C --> E[Rank bottleneck hypotheses]
+  D --> E
+  E --> F[Generate legal candidate actions]
+  F --> G[Score candidates]
+  G --> H{High-value candidate?}
+  H -- yes --> I[Emit harness-proposal]
+  I --> J[Run real vLLM trial over search range]
+  J --> B
+  H -- no --> K{Validator stop allowed?}
+  K -- yes --> L[Emit harness-stop]
+  K -- no --> M{LLM endpoint exists?}
+  M -- yes --> N[Ask LLM backend]
+  M -- no --> O[Fail loudly: no proposal source]
+```
+
+## Observation: harness 看到什么
+
+每一轮 harness 不看自然语言日志做猜测，而是读结构化状态：
+
+- `StudySpec`
+  - hardware: GPU 数量、GPU 型号；
+  - engine: base flags/envs、tunable flags/envs、topology constraints；
+  - trace: request mode、window id、输入长度过滤、输出长度 override；
+  - SLO: TTFT/TPOT rule、target pass rate；
+  - search: load range、tolerance、probe budget。
+- `window_summary` / `WorkloadProfile`
+  - L: request length 分布、tail ratio；
+  - C: prefix/cache reuse；
+  - A: arrival rate、burstiness、interarrival variation。
+- 最近 trials
+  - config patch；
+  - best feasible request rate；
+  - request_rate_per_gpu；
+  - pass rate；
+  - probe history；
+  - latency p50/p95/p99；
+  - SLO failure reason counts；
+  - launch/runtime failure stage。
+
+这些数据会被压成 `recent_trial_diagnostics` 和 `trial_profiles`，后续步骤只使用这些结构化
+字段。
+
+## Bottleneck classifier: 怎么判断方向
+
+Harness 维护一组 ranked bottleneck hypotheses：
+
+```text
+ttft_prefill
+decode_tpot
+admission_or_queueing
+launch_or_memory
+```
+
+它的输入不是单一阈值，而是多类证据：
+
+- workload default：长 prompt tail 更偏向 `ttft_prefill`；
+- request mode：decode-only 且有 TPOT SLO 时更偏向 `decode_tpot`；
+- probe sequence：最近 trial 的 active bottleneck 权重大于旧 trial；
+- failed reason counts：
+  - `ttft_ms>...` 支持 `ttft_prefill`；
+  - `tpot_ms>...` 支持 `decode_tpot`；
+  - `arrival_lag_s>` / `probe_elapsed_s>` 支持 `admission_or_queueing`；
+- launch failure / OOM：支持 `launch_or_memory`。
+
+代码里这不是一个硬编码单标签，而是带 confidence 的 ranked list。例如最近 probe
+明确出现 TPOT failure，会提高 `decode_tpot` 分数；如果同时 workload 有长 prompt tail，
+`ttft_prefill` 仍会保留为次级 hypothesis。
+
+## Candidate family: raw knobs 如何变成 intervention
+
+Harness 不直接在所有 tunable flags 上盲采样。它先把 knobs 分成有系统含义的
+intervention family：
+
+| Family | 代表 knobs | 机制含义 |
+| --- | --- | --- |
+| topology | `tensor-parallel-size`, `data-parallel-size`, EP knobs | 改变每请求并行度、replica 数量、通信/效率 tradeoff |
+| batching | `max-num-batched-tokens`, `enable-chunked-prefill` | 改变 prefill/decode batching 与 HoL blocking |
+| admission | `max-num-seqs` | 改变并发 admission 与 TPOT/TTFT tail |
+| KV memory | `gpu-memory-utilization` | 改变 KV cache blocks 和可承载并发 |
+| failure memory | failed signatures | 阻止重复已知 launch/runtime 失败方向 |
+
+关键点是：candidate 来自当前 `StudySpec` 的 tunable schema 和 topology constraints。
+例如 topology candidate 只枚举合法 TP/DP/EP 组合；如果 EP 没有直接证据，generic
+topology search 不会主动引入 EP。
+
+## Scoring: 为什么会先走 topology，再走 gmu
+
+Candidate action 的评分大致是：
+
+```text
+score = expected_bottleneck_relief * bottleneck_confidence
+      + information_gain
+      + launch_safety
+      - regression_risk
+```
+
+然后 `experiment_plan.next_action` 选择最高分候选。分数超过阈值时，harness 直接生成
+proposal；否则进入 stop validator 或 LLM fallback。
+
+这套 scoring 体现了几个系统原则：
+
+1. Topology 是 serving 的一阶决策。
+   当 TP frontier 还没测完，`gpu-memory-utilization`、`max-num-seqs` 这类 runtime
+   微调不会抢在 topology 前面。
+
+2. Topology 不是“越大越好”。
+   评分和最终 winner 都看 `request_rate_per_gpu`，不是 raw request rate。TP4 可能总吞吐
+   更高，但如果使用更多 GPU 后 per-GPU 效率下降，就不会成为 incumbent。
+
+3. Runtime tuning 必须 anchored on incumbent topology。
+   当 topology 已经验证过，runtime proposal 会 preserve 当前 best topology，只在其上
+   调 `gpu-memory-utilization`、`max-num-seqs`、`max-num-batched-tokens`。
+
+4. Measurement 决定最终答案。
+   Candidate 只是一个 hypothesis；是否接受由真实 trial 的 SLO-constrained
+   `request_rate_per_gpu` 决定。
+
+5. Bad-start recovery 需要先 bracket，再微调。
+   如果 no-LLM run 从一个很高 TP 的初始点开始，且同 DP 下更高 TP frontier 已经不存在
+   或已测过，harness 会优先验证相邻低 TP，而不是把当前高 TP 当作 topology 已收敛。
+   这避免了 `TP=8` 这类坏初始点直接进入 `gpu-memory-utilization` 微调。
+
+6. Pathological runtime 起点需要跳回正常工作区间。
+   `gpu-memory-utilization` 的常规策略是在 settled topology 上小步 hill-climb；
+   但如果初始值明显低于正常工作区间，例如 `0.5`，harness 会先跳到 nominal floor
+   `0.9`，再按 `0.02` 步长向 safe ceiling `0.97` 验证。
+
+## Validator stop: 为什么不会过早停止
+
+Harness stop 不是“找到一个不错配置就停”。当前 stop validator 包含几个条件：
+
+- `search_high_saturated_by_incumbent`
+  - incumbent 的最高 feasible probe 已经贴近 configured search high；
+  - 说明当前测量范围已被打满。
+- `topology_frontier_requires_probe`
+  - 如果 active bottleneck 仍要求更高 TP frontier 且未测，禁止 stop。
+- `experiment_plan_has_high_value_candidate`
+  - 如果还有高分候选，禁止 stop。
+- `post_incumbent_validation_exhausted`
+  - strong incumbent 后至少要有 post-incumbent validation；
+  - validation 覆盖 topology/runtime family 或达到足够数量；
+  - 没有任何 validation trial 超过 incumbent；
+  - 才允许 clean stop。
+
+所以 validator 的作用是 fail-safe：
+
+```text
+wrong proposal 最多浪费一个 trial；
+wrong stop 会终止搜索，所以必须由 deterministic validator 授权。
+```
+
+## Qwen30B no-LLM run 中具体发生了什么
+
+Run:
+
+```text
+qwen30b-harness-only-medium-stopfix-dash1-20260624T144701Z
+```
+
+设置：
+
+- Model: `Qwen/Qwen3-30B-A3B`
+- Engine: community vLLM 0.20
+- Hardware: 8x H20, 允许 TP/DP/EP frontier
+- Trace: chat 0-8k, output 128, replay time scale 0.1
+- SLO: target pass rate 0.95, TTFT step rule, TPOT 50ms
+- LLM endpoint: `null`
+
+真实 trial path:
+
+| Trial | Source | Config patch | req/s/GPU | pass rate | Harness 解释 |
+| --- | --- | --- | ---: | ---: | --- |
+| 0001 | baseline | `{}` | 2.2000 | 1.0000 | 建立 baseline 和 probe evidence |
+| 0002 | harness | `TP=2` | 3.2583 | 1.0000 | latency/SLO pressure 下先测 adjacent TP |
+| 0003 | harness | `TP=4` | 2.0917 | 1.0000 | 验证更高 TP frontier；raw 总吞吐高但 per-GPU 低 |
+| 0004 | harness | `TP=2, gmu=0.92` | 3.2583 | 1.0000 | topology 已 settle，开始 incumbent topology 上的 KV headroom climb |
+| 0005 | harness | `TP=2, gmu=0.94` | 3.2583 | 1.0000 | 继续小步 hill-climb；未改善但未失败 |
+| 0006 | harness | `TP=2, gmu=0.96` | 3.3333 | 1.0000 | KV headroom 带来更高 feasible frontier |
+| 0007 | harness | `TP=2, gmu=0.97` | 3.4333 | 1.0000 | 达到 safe ceiling，成为 incumbent |
+| 0008 | harness | `TP=4, DP=2` | 1.0458 | 1.0000 | post-incumbent topology validation，没有超过 incumbent |
+| 0009 | harness | `TP=8` | 1.0458 | 1.0000 | 继续 frontier validation，没有超过 incumbent |
+| 0010 | harness stop | stop | - | - | validator: `post_incumbent_validation_exhausted` |
+
+这个过程里没有外部 LLM 决策。每一步 proposal 都来自 harness：
+
+1. baseline 观测到当前 engine 在 SLO 下的可行 frontier；
+2. bottleneck/机制模型认为 topology 是一阶干预；
+3. 测 TP2，接受，因为 per-GPU 从 2.2 提到 3.2583；
+4. 测 TP4，拒绝为 incumbent，因为 per-GPU 降到 2.0917；
+5. topology frontier settle 后，在 TP2 上小步提升 `gpu-memory-utilization`；
+6. `gmu=0.97` 达到 3.4333；
+7. 再测 nearby topology，确认没有更好；
+8. validator 授权 stop。
+
+## 为什么这不是写死 Qwen30B
+
+这条路径看起来像“harness 知道答案是 TP2+gmu0.97”，但代码机制不是这样写的。
+
+没有写死的部分：
+
+- 没有写死 model name；
+- 没有写死 Qwen30B；
+- 没有写死 `TP=2` 是最终答案；
+- 没有写死 `gmu=0.97` 一定最好；
+- 没有跳过真实测量；
+- 没有把 TP4/TP8 直接判负，而是实际运行并比较。
+
+真正写入 harness 的 domain knowledge 是：
+
+- TP/DP/EP 是 topology family，必须满足 topology constraints；
+- topology 通常是一阶 serving intervention，要先于 runtime 微调被验证；
+- raw throughput 不等于目标，跨 topology 比较要用 `request_rate_per_gpu`；
+- `gpu-memory-utilization` 是 KV memory headroom 微调，只应在 incumbent topology 上小步 hill-climb；
+- launch failure 和 tested signatures 是 hard negative evidence；
+- stop 必须由 validator 授权，不能由 proposer 自己说停就停。
+
+这是一种系统机制约束，不是 case-specific prompt。
+
+## 它和 BO / raw heuristic 的区别
+
+普通 BO 或 raw heuristic 的搜索空间通常是：
+
+```text
+config = {tp, dp, ep, gmu, max_num_seqs, max_num_batched_tokens, ...}
+score = measured req/s/GPU
+```
+
+这会产生几个问题：
+
+- 它不知道哪些 knobs 是 topology family，哪些是 runtime family；
+- 它可能在没测 TP frontier 前浪费大量 trial 调 runtime；
+- 它可能重复已知 launch failure；
+- 它可能把 raw throughput 高但 GPU efficiency 差的配置误当方向；
+- 它很难解释“这个 trial 试图证伪哪个瓶颈 hypothesis”。
+
+Harness-shaped search space 是：
+
+```text
+state -> bottleneck hypothesis -> legal intervention family -> measured verdict
+```
+
+因此 BO、bandit、LLM、deterministic heuristic 都可以接在 harness 后面。它们优化的不是
+raw knob vector，而是有 serving 语义的 intervention graph。
+
+这也是我们新 framing 的核心：
+
+```text
+black-box optimization
+  -> grey-box / mechanism-guided experimental optimization
+```
+
+## 当前还需要补的证据
+
+No-LLM Qwen30B run 证明了 deterministic harness 可以完整闭环，但 paper 还需要继续补：
+
+1. Planner-agnostic ablation
+   - `raw BO` vs `harness-guided BO`；
+   - `raw heuristic` vs `harness deterministic policy`；
+   - 证明收益来自 harness substrate，而不是某个 LLM。
+
+2. Mechanism ablation
+   - no attribution；
+   - shuffled attribution；
+   - no topology-first；
+   - no intervention grammar；
+   - no validator/failure memory。
+
+3. Near-optimum evidence
+   - 在 1-2 个 case 做局部 grid；
+   - 证明 harness path 找到的是 near-optimal region，不只是一个可行 config。
+
+4. Cross-case robustness
+   - 再选 decode-heavy 或 long-prefill case；
+   - 验证不同 workload/SLO 下 candidate family 会发生合理切换。
+
+5. Bad-start recovery
+   - 从非可信初始配置开始，例如 `TP=8, max-num-seqs=8, gmu=0.5`；
+   - 证明 harness 不是只能从“已经比较合理”的 base config 出发；
+   - 观察它是否能先恢复 topology，再恢复 runtime headroom，并最终回到同一 near-optimal
+     region。
+
+## Bad-start recovery 审计 - 2026-06-26
+
+用户提出的问题是：如果我们不是从可信 base config 开始，而是从一个恶意或不合理的
+配置开始，例如：
+
+```text
+TP=8, DP=1, max-num-seqs=8, gpu-memory-utilization=0.5
+```
+
+no-LLM harness 是否仍能自动找到正确方向？
+
+目前结论要分开说：
+
+1. **旧 planner 不能直接 claim 任意坏起点可恢复。**
+   本地合成审计显示，旧逻辑会把 `TP=8` 误当作 topology frontier 已收敛，并把下一步
+   proposal 设为 `gpu-memory-utilization=0.52`。这会在坏 topology 和坏 runtime 上
+   做很慢的小步爬坡，不能作为 robust evidence。
+
+2. **已补 planner 机制。**
+   当前 harness 增加了两个 no-LLM deterministic recovery rules：
+   - `bad_start_topology_bracket`：当当前 anchor 在高 TP，且没有未测的更高 TP frontier 时，
+     先测相邻低 TP，例如 `TP=8 -> TP=4`；
+   - `gmu_nominal_floor`：当 settled topology 上的 `gpu-memory-utilization < 0.9` 时，
+     先跳到 `0.9`，再做常规 `0.92/0.94/.../0.97` hill-climb。
+
+3. **已加本地回归测试，但还没做真机证明。**
+   已通过的 planner tests：
+   - `test_harness_brackets_down_from_bad_high_tp_start_before_runtime_tuning`
+   - `test_harness_jumps_low_gpu_mem_util_to_nominal_floor_after_topology_settles`
+   - 以及已有 topology-first / gmu-climb 相关回归测试。
+
+因此，当前状态是：planner 侧已经能给出正确方向；paper 级别还需要真机 bad-start
+recovery run 来确认真实 vLLM 测量下是否稳定收敛。
+
+## 准备中的真机实验
+
+实验目的不是再证明默认起点能 work，而是证明：
+
+```text
+same workload + same SLO + same no-LLM harness
+不同初始 config
+  -> 是否收敛到同一 near-optimal region
+  -> 是否保持可解释 trial path
+```
+
+Base spec 使用已验证的 Qwen30B community vLLM 0.20 harness setup：
+
+```text
+configs/examples/dash0_qwen30b_a3b_community_vllm020_harness.json
+```
+
+运行时需要设置：
+
+```json
+{
+  "llm": {
+    "use_harness": true,
+    "endpoint": null
+  }
+}
+```
+
+建议最小矩阵：
+
+| Case | Base flags 变化 | 要验证的机制 | 预期 trial path |
+| --- | --- | --- | --- |
+| trusted-start-control | 保持现有可信 base | 对照已有 stopfix run | `TP=2 -> TP=4 -> TP=2+gmu climb -> stop` |
+| bad-topology | `TP=8, DP=1` | 高 TP 起点是否会向下 bracket | `TP8 baseline -> TP4 -> TP2/或同等 better topology -> runtime` |
+| bad-runtime | `TP=2, DP=1, gmu=0.5, max-num-seqs=8` | 低 KV headroom 是否跳回正常区间 | `gmu 0.5 baseline -> gmu 0.9 -> 0.92/...` |
+| combined-bad | `TP=8, DP=1, gmu=0.5, max-num-seqs=8` | topology recovery 和 runtime recovery 能否串起来 | `TP8 -> TP4 -> TP2/nearby -> gmu 0.9 -> climb -> stop` |
+
+成功判据：
+
+- 不配置 LLM endpoint；所有 proposal 来自 harness；
+- 不重复相同 config signature；
+- high-TP 起点必须先出现相邻低 TP probe，而不是先做 `gmu=0.52`；
+- low-gmu 起点必须先跳到 `0.9`，而不是 `0.52`；
+- 在 12 个 measured trials 内达到 reference stopfix best 的 `>=95%`：
+
+```text
+reference best = 3.4333 req/s/GPU
+95% threshold  = 3.2616 req/s/GPU
+```
+
+- 最终 stop 必须是 validator 授权，例如 `harness_stop`，而不是因为没有 proposal source
+  失败退出。
+
+如果真机结果失败，需要保留失败路径并分析是哪类机制不足：
+
+- topology bracket 找到低 TP，但 runtime 仍无法恢复；
+- `max-num-seqs=8` 导致 admission 太差，需要 admission recovery floor；
+- baseline 自身全不可行，当前 harness 缺少 completed incumbent，不能进入正常 guided loop；
+- vLLM launch/OOM 造成 failure memory 覆盖了可恢复路径。
+
+## 一句话总结
+
+No-LLM harness 能自动找到配置，是因为它已经实现了一个面向 serving 机制的实验 planner：
+先把 trial 观测归因成 bottleneck，再把 bottleneck 映射成合法 intervention family，按
+SLO-constrained req/s/GPU 真实测量更新 incumbent，最后由 validator 判断是否停止。
+LLM 只是这个 planner 的一个可替换 proposal backend，而不是 AITuner 的必要核心。

docs/harness-ablation/prefill-scheduler-normalized-design-20260629.md

@@ -0,0 +1,336 @@
+# Prefill Scheduler Interaction Harness 设计与 Review
+
+日期：2026-06-29
+
+## 背景
+
+case3 的 ablation 结果显示，`gpt-5.5 no-harness` 找到了一个 runtime/scheduler 方向：
+
+```text
+enable-chunked-prefill=true
+max-num-batched-tokens 较低/中等
+max-num-seqs 适中
+block-size=16
+```
+
+而当时 harness 主要做两类动作：
+
+- 单点打开 `enable-chunked-prefill`；
+- 对 `max-num-batched-tokens` 做单调 raise。
+
+这个 gap 不能用“把 8192/32 这组值加入 candidate grid”来修补。那会把 case3 的答案硬编码成更大的候选表，仍然是 rule-based overfitting。
+
+## 设计原则
+
+新增的设计不是一个 fixed value set，而是一个 normalized control dimension：
+
+```text
+prefill_quantum_ratio = max-num-batched-tokens / prompt_tokens_p95
+admission_pressure    = max-num-seqs relative to trace.max_concurrency
+scheduler_mode        = enable-chunked-prefill
+```
+
+因此，candidate generator 不直接说“试 8192”，而是说：
+
+- 如果 long-tail prefill + TTFT/prefill bottleneck，且当前 `prefill_quantum_ratio` 太大，则沿 log-ratio 方向降低 prefill quantum；
+- 如果 prefill quantum 远小于 prompt scale，可能是过度切碎导致 overhead，则沿 log-ratio 方向提高 prefill quantum；
+- 如果 admission/queueing 是瓶颈，则只按 relative step 调整 admission pressure；
+- 所有 concrete flag value 都是最后一步从 normalized target 映射到 engine flag，并按 engine granularity round。
+
+当前实现使用几何中点作为 trust-region step：
+
+```text
+target_mbt = sqrt(current_mbt * prompt_tokens_p95)
+```
+
+这对应在 log space 走半步。它比固定乘以 0.5/1.5 更接近 scale-invariant：prompt scale 变大时，下一步 MBT 也会变大。
+
+## Agent Loop
+
+当前 harness 的 loop 可以形式化为：
+
+```text
+trial result
+  -> observation extractor
+  -> bottleneck classifier
+  -> candidate family selector
+  -> normalized candidate generator
+  -> scoring / coverage ranking
+  -> validator / no-repeat / stop guard
+  -> next trial
+```
+
+每一层承担不同责任：
+
+1. observation extractor 只把 trial result 转成可比较的事实，例如
+   request_rate_per_gpu、pass_rate、失败原因、TTFT/TPOT 分位数。
+2. bottleneck classifier 把事实归入 `ttft_prefill`、`decode_tpot`、
+   `admission_or_queueing` 等机制瓶颈，不直接输出配置值。
+3. candidate family selector 决定要验证哪个系统假设，例如 topology frontier、
+   prefill scheduler、admission pressure 或 GPU memory headroom。
+4. normalized candidate generator 才把机制变量映射成具体 engine flag。
+5. scoring / coverage ranking 负责排序：未覆盖但机制上相关的维度应优先于
+   已知方向上的微调。
+6. validator 使用 normalized full-config signature 防止重复测试，并用 stop guard
+   避免在仍有高价值 falsification candidate 时过早停止。
+
+因此，harness 的核心不是“把 LLM prompt 写好”，而是把黑盒搜索拆成带因果方向的
+white-box falsification loop。LLM 可以参与生成候选或解释候选，但候选必须通过
+harness 的 family、signature、scoring 和 validator 约束。
+
+## 实现映射
+
+代码入口：
+
+- `src/aituner/harness.py::_runtime_candidate_actions`
+  - 在 topology frontier settled 后调用新的 `_prefill_scheduler_candidate_actions`。
+  - 仍保留 topology-before-runtime guard，runtime family 不抢未覆盖的 topology frontier。
+
+新增逻辑：
+
+- `_prefill_scheduler_workload_applies`
+  - 只在非 decode-only、long-tail/moderate-tail prefill workload、非 high-prefix-reuse 场景激活。
+- `_next_prefill_quantum_step`
+  - 使用 `current_mbt / prompt_scale` 判断方向。
+  - 通过几何中点做相对 step。
+- `_next_admission_pressure_step`
+  - 使用 `max-num-seqs / trace.max_concurrency` 作为 normalized admission pressure。
+  - 当 admission/queueing 受限且 admission pressure 过低时 raise。
+  - 当 TTFT/prefill 受限且 admission pressure 明显高于 trace concurrency scale 时 lower。
+- `_prefill_scheduler_candidate_actions`
+  - 输出 `prefill-scheduler-interaction` family。
+  - `score_factors` 显式记录 current/target `prefill_quantum_ratio`，方便后续实验解释。
+  - `score_factors` 同时记录 current/target admission pressure ratio，避免只解释 MBT。
+  - 当 scheduler dimension 还没有被 materialized config 覆盖时，加入
+    `uncovered_scheduler_dimension_bonus`，让该 family 在 topology settled 后优先于
+    `gpu-memory-utilization` 这类 resource micro-tuning。
+- 当该 family 已生成有效候选时，旧的 standalone `raise_mbt`、
+  `enable_chunked_prefill`、`raise_mbt_and_max_num_seqs` 只作为 fallback，不作为同级
+  prefill runtime 候选抢排序。
+- `gpu-memory-utilization` 仍保留小步 hill-climb，但继续爬升必须由同拓扑
+  request_rate_per_gpu 改善支撑；仅仅 launch 成功或打平 incumbent 不再算成功。
+
+## 为什么不是 rule-based hack
+
+禁止的实现形态：
+
+- 不允许引用 case3、具体 trace 名、模型名、机器名；
+- 不允许出现 `if TP=2 and gmu=0.7 and mns=8 then MBT=8192`；
+- 不允许把 case3 发现扩成 `{4096,8192,12288,16384} x {16,32,64}` 这种固定 grid；
+- 不允许 bypass normalized full-config signature。
+
+当前实现满足：
+
+- trigger 来自 L-C-A profile、bottleneck classifier、topology frontier、tunable flags；
+- proposal 是相对当前 incumbent 的 direction，不是固定答案；
+- concrete value 随 prompt scale 和 current config 改变；
+- validator/no-repeat 仍使用 normalized effective full-config signature；
+- runtime gate 和正式 topology frontier 共用 higher-TP frontier patch 构造，避免
+  DP 非 base 时 scheduler 抢跑；
+- short prompt、decode-only、high prefix reuse 不激活该 family。
+
+但这不是完备性证明。当前能 claim 的是更严格的工程性质：
+
+- 不引用特定 case identity；
+- 不把已知 winner 写成候选表；
+- 每个 concrete proposal 都能追溯到一个 normalized mechanism variable；
+- 每次 trial 都能被解释成对一个系统假设的 falsification；
+- 失败时会留下可审计的 candidate sequence 和 score factors。
+
+## Review 结论
+
+### 之前实现的问题
+
+1. `enable-chunked-prefill` 是 standalone toggle，无法表达 scheduler interaction。
+2. TTFT/prefill bottleneck 下 MBT 主要单调 raise，无法发现“降低 prefill quantum 减少 HoL blocking”。
+3. 旧测试断言了固定 `16384` 等值，容易把 harness 叙事拉回 heuristic table。
+
+### 当前改动的效果
+
+1. 引入 `prefill-scheduler-interaction` 作为新的 mechanistic family。
+2. candidate 的 action id 表达方向：
+   - `lower_prefill_quantum_with_chunked_prefill`
+   - `raise_prefill_quantum_with_chunked_prefill`
+   - `seed_chunked_prefill_quantum`
+   - `adjust_admission_pressure_with_chunked_prefill`
+3. 测试改为验证 normalized direction 和 scale sensitivity，而不是固定 absolute value。
+
+### 当前实现仍需警惕的风险
+
+1. `_PREFILL_QUANTUM_HEAD_OF_LINE_RATIO=1.0` 和
+   `_PREFILL_QUANTUM_FRAGMENTATION_RATIO=0.5` 仍是机制阈值，不是定理。
+   它们必须通过 scaled prompt / negative workload 实验验证，而不能只靠 case3。
+2. `uncovered_scheduler_dimension_bonus` 是 coverage 排序策略。它的合理性来自
+   “先覆盖未 materialized 的机制维度，再做 GMU 微调”，但必须通过 candidate
+   sequence 证明它不会在 topology frontier 未覆盖时抢跑。
+3. `block-size=16` 目前没有被纳入这个 family。不能把它作为 case3 固定答案加入；
+   如果后续要处理，需要单独设计 allocator/layout family，从 engine capability 和
+   memory block waste observation 推导，而不是在 prefill scheduler family 里硬塞。
+4. 现有实现仍保留旧的 standalone `enable-chunked-prefill` 和 `raise_mbt` 路径作为
+   fallback。它们不能在 `prefill-scheduler-interaction` 已生成有效候选时作为同级
+   prefill runtime 候选抢排序。
+
+### 2026-06-29 独立 review 后的修正
+
+独立 review 指出了三个需要立即收紧的泛化风险：
+
+1. 旧 standalone MBT/chunked 候选仍可能让整体 harness 表现得像 heuristic table。
+2. admission pressure 只有 raise，没有处理 `max-num-seqs` 过高导致 TTFT/prefill 干扰。
+3. runtime gate 的 topology-settled 判断和正式 topology frontier 在 DP 非 base 时不完全一致。
+
+对应修正：
+
+- 当 `prefill-scheduler-interaction` 有有效候选时，旧的 standalone MBT/chunked/joint
+  prefill-runtime 候选降为 fallback。
+- admission pressure 改为 normalized ratio，并支持 raise/lower 两个方向：
+  `raise_admission_pressure_with_chunked_prefill` 和
+  `lower_admission_pressure_with_chunked_prefill`。
+- 抽出 `_higher_tp_frontier_patch`，让 runtime gate 与
+  `_topology_frontier_status` 使用同一套 higher-TP signature。
+- GMU hill-climb 改为 measurement-gated：同拓扑 GMU trial 没有提升
+  request_rate_per_gpu 时，阻断继续向更高 GMU 爬升，避免连续浪费 trials。
+
+### 2026-06-29 远端 review feedback
+
+在 dash1 用 `36c301c` 启动 case3 bad-runtime 重跑后，trial-0003 的
+candidate-set 已经出现 `prefill-scheduler-interaction`：
+
+```text
+action_id = seed_chunked_prefill_quantum
+patch     = enable-chunked-prefill=true, max-num-batched-tokens=8192
+ratio     = target prefill_quantum_ratio ~= 1.05
+```
+
+但初始 scoring 仍让 `raise_gpu_memory_utilization` 排在前面。这说明 family
+接入是正确的，但排序仍偏向 resource micro-tuning。随后实现加入
+`uncovered_scheduler_dimension_bonus`：当 topology frontier 已覆盖、prefill scheduler
+dimension 还没有被 materialized config 测过时，优先测试 scheduler hypothesis，
+避免重复旧 harness 先爬 GMU 的失败轨迹。
+
+## 单元验证
+
+新增/更新的测试覆盖：
+
+- long-tail TTFT 下，过大的 `prefill_quantum_ratio` 会下降；
+- prompt length scale 变大时，下一步 MBT target 也变大；
+- topology frontier 已覆盖后，未覆盖的 scheduler dimension 排在 GMU 微调之前；
+- short prompt workload 不激活 prefill scheduler family；
+- 原有 prefill stop guard 仍不允许在有 high-value candidate 时停止；
+- normalized full-config no-repeat 语义不变。
+
+本地全量测试：
+
+```text
+PYTHONPATH=src python3 -m unittest discover -s tests
+156 tests OK
+```
+
+本地重点回归：
+
+```text
+PYTHONPATH=src python3 -m unittest \
+  tests.test_core_flow.CoreFlowTests.test_prefill_scheduler_coverage_precedes_gmu_microtune \
+  tests.test_core_flow.CoreFlowTests.test_prefill_scheduler_admission_pressure_only_uses_normalized_seq_cap \
+  tests.test_core_flow.CoreFlowTests.test_prefill_scheduler_lowers_excess_admission_pressure \
+  tests.test_core_flow.CoreFlowTests.test_prefill_scheduler_negative_applicability_matrix \
+  tests.test_core_flow.CoreFlowTests.test_prefill_scheduler_does_not_preempt_open_topology_frontier \
+  tests.test_core_flow.CoreFlowTests.test_prefill_scheduler_lowers_quantum_by_normalized_ratio \
+  tests.test_core_flow.CoreFlowTests.test_prefill_scheduler_quantum_step_scales_with_prompt_length \
+  tests.test_core_flow.CoreFlowTests.test_prefill_scheduler_not_active_for_short_prompt_workload
+8 tests OK
+```
+
+## 还需要真机实验验证
+
+下一步实验不应该只看 case3 是否复现，而要攻击这个 family 的边界：
+
+1. case3 bad runtime start：
+   - 目标：验证 LLM+harness / no-LLM harness 是否能从 bad runtime start 找到 chunked-prefill scheduler 方向。
+2. scaled prompt case：
+   - 目标：验证 proposal 不固定在同一个 MBT，而会随 `prompt_tokens_p95` 改变。
+3. short/decode negative case：
+   - 目标：验证该 family 不会在不适用 workload 上误触发。
+4. topology frontier case：
+   - 目标：验证 topology 未覆盖时 runtime scheduler 不抢跑。
+
+核心指标：
+
+- best request_rate_per_gpu；
+- time-to-best / trial-to-target；
+- candidate family sequence；
+- `prefill_quantum_ratio_current -> target` 的方向是否与 bottleneck evidence 一致；
+- 是否出现 repeated normalized full-config signature。
+
+## 当前 dash1 真机状态
+
+当前正在验证提交 `bfd8579`：
+
+```text
+run     = .aituner/badstart-prefill-scheduler-bfd8579-20260628T173102Z
+case    = badstart-expanded-9accf25-20260626T184911Z-runtime_tp2_dp1_gmu070_mns8
+session = aituner-prefill-scheduler-case3-bfd8579
+```
+
+截至 2026-06-29 01:53 UTC+8 左右：
+
+- baseline trial-0001 已完成，best request_rate_per_gpu 约为 2.025；
+- trial-0002 TP4 topology frontier probe 已完成，best request_rate_per_gpu 约为 2.000，
+  没有超过 baseline；
+- candidate-set-0002 的 top action 是 topology frontier，符合 topology-before-runtime；
+- candidate-set-0003 的 top action 已变为 `seed_chunked_prefill_quantum`：
+
+```text
+score    = 0.69
+patch    = enable-chunked-prefill=true, max-num-batched-tokens=8192
+ratio    = prefill_quantum_ratio_target ~= 1.0536
+baseline = raise_gpu_memory_utilization score 0.64
+```
+
+这说明 `uncovered_scheduler_dimension_bonus` 达到了设计目的：topology frontier 覆盖后，
+未 materialized 的 scheduler dimension 会先于 GMU 微调被验证。
+
+trial-0003 已完成，best request_rate_per_gpu 约为 2.025，和 baseline 持平，没有形成
+性能提升。这个结果不能 claim scheduler seed 是 winner，但它提供了有价值的
+falsification evidence：coverage priority 改变了探索顺序，具体 `chunked + MBT ~= p95`
+hypothesis 被验证后没有改进。系统随后进入 candidate-set-0004，开始测试
+`gpu-memory-utilization=0.9`。trial-0004 同样完成在约 2.025，没有超过 baseline；
+trial-0005 的 `gpu-memory-utilization=0.92` 仍然打平 baseline，旧 run 随后继续排
+`gpu-memory-utilization=0.94`。这暴露出旧实现的 GMU hill-climb 问题：它把 launch
+成功当成 climb 成功，而没有要求 request_rate_per_gpu 改善。最新本地实现已经修正为
+measurement-gated GMU climb；下一轮应使用新提交重新跑，验证 GMU tie 后是否转向
+admission pressure、topology/DP 或其他 family。
+
+## Hardened Run Feedback
+
+使用提交 `6b25d56` 在 dash1 重新启动：
+
+```text
+run     = .aituner/badstart-prefill-hardened-6b25d56-20260628T180104Z
+case    = badstart-expanded-9accf25-20260626T184911Z-runtime_tp2_dp1_gmu070_mns8
+session = aituner-prefill-hardened-6b25d56
+```
+
+截至 2026-06-29 02:27 UTC+8 左右，同一 run 内的 trial sequence 是：
+
+| trial | patch | request_rate_per_gpu | observation |
+| --- | --- | ---: | --- |
+| 0001 | baseline bad-start | 2.983 | 同 run incumbent，明显高于旧 run baseline，说明跨 run 数字不能直接混用 |
+| 0002 | `tensor-parallel-size=4` | 1.629 | topology TP4 被 falsify |
+| 0003 | `enable-chunked-prefill=true, max-num-batched-tokens=8192` | 2.025 | standalone scheduler seed 被 falsify |
+| 0004 | `gpu-memory-utilization=0.9` | 3.258 | GMU=0.9 是当前 best，达到已知 no-harness 水平 |
+| 0005 | GMU=0.9 + scheduler seed | 2.025 | GMU 与 scheduler seed 的组合被 falsify |
+| 0006 | `gpu-memory-utilization=0.92` | 3.258 | 与 GMU=0.9 打平，没有继续提升 |
+| 0007 | `tensor-parallel-size=4, data-parallel-size=2` | 1.000 | DP/topology probe 被 falsify |
+
+candidate-set-0007 没有继续提出 `gpu-memory-utilization=0.94`，而是转向
+`tensor-parallel-size=4, data-parallel-size=2` topology probe。这验证了
+measurement-gated GMU climb：GMU=0.92 只是打平时，不再继续向更高 GMU 盲目爬升。
+candidate-set-0008 在 TP4/DP2 被 falsify 后继续测试 `tensor-parallel-size=8`。
+
+当前最重要的机制结论：
+
+- scheduler seed 的 priority 和 no-repeat 都按设计工作；
+- scheduler seed 在这个 case 不是独立 winner，必须被 measurement falsify；
+- GMU=0.9 是当前真正有效的机制维度；
+- GMU 的后续 climb 已经从 launch-gated 修正为 improvement-gated；
+- 后续应看 topology/DP、MNS 或 allocator/layout family 是否能进一步超过 3.258。

docs/harness-ablation/qwen235b-prefill-2x2-progress-20260623.md

@@ -0,0 +1,138 @@
+# Qwen235B prefill 2x2 progress - 2026-06-23
+
+Snapshot: 2026-06-23 18:24 CST / 10:24 UTC.
+
+本文整理当前 dash1/dash2/dash3 上的 Qwen235B prefill 2x2 实验进度。这个
+case 仍在跑 strong-model arm，因此本文是 progress report，不是最终 aggregate
+结论。
+
+## 当前远端状态
+
+| Host | 当前状态 | 说明 |
+| --- | --- | --- |
+| dash1 | running | `aituner-q235b-2x2-gpt55-20260623T010038Z` 仍在跑，当前是 `gpt-5.5 + naive` 的 trial-0004；8 张 H20 被 vLLM 占用。 |
+| dash2 | idle | 没有 tmux/GPU 任务；最近完成的是 `qwen235b-prefill-jointprobe-harness-dash2-20260622T132010Z` harness-only 验证。 |
+| dash3 | idle | 没有 tmux/GPU 任务；`gpt-5.4-mini` 2x2 arm 已完成并生成 report。 |
+
+注意：三台机器共享 `/home/admin/cpfs/wjh/aituner/aituner`，所以 `.aituner` 和
+`.aituner-reports` 在不同 dash 节点上看到的是同一批产物。
+
+## 已完成：gpt-5.4-mini 2x2 arm
+
+Report:
+
+```text
+.aituner-reports/qwen235b-prefill-2x2-gpt54mini-dash3-20260623T010038Z/report.md
+```
+
+Aggregate:
+
+| Arm | Kind | Trials | Final req/s/GPU | Final/ref | TTT | AUC | Failed | No feasible |
+| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| `harness` | harness | 8 | 0.3217 | 1.0000 | 3 | 0.9483 | 0 | 1 |
+| `naive` | naive | 8 | - | - | - | 0.0000 | 2 | 8 |
+
+Interpretation:
+
+- `gpt-5.4-mini + harness` 找到了 `0.3217 req/s/GPU`，达到该 report 的
+  reference best。
+- `gpt-5.4-mini + naive` 8 个 trials 都没有找到 feasible config，其中 2 个是
+  engine launch failure。
+- Report 中 `Harness-vs-naive pass/checks: 0/1` 是 aggregator 对
+  `best_naive_final_per_gpu = null` 的保守处理：因为 naive 没有 feasible best，
+  final ratio 无法计算，所以 pass 记为 false。就实际 tuning 结果而言，这个 arm
+  是 harness dominates naive。
+
+Harness trajectory:
+
+| Trial | Patch | req/s/GPU | Pass rate | 说明 |
+| ---: | --- | ---: | ---: | --- |
+| 1 | `TP=8, DP=1` | 0.2879 | 0.9522 | 初始 topology 满足 SLO，但未达到最终 best。 |
+| 2 | `TP=8, max-num-seqs=96` | 0.2879 | 0.9537 | 单独调 `max-num-seqs` 无明显提升。 |
+| 3 | `TP=8, max-num-batched-tokens=16384, max-num-seqs=96` | 0.3085 | 0.9568 | joint runtime probe 提升。 |
+| 4 | `TP=8, max-num-seqs=144, max-num-batched-tokens=32768` | 0.2879 | 0.9530 | 过大的 batching/seq 组合回退。 |
+| 5 | `TP=4, DP=2` | - | - | 无 feasible best，说明 DP-heavy/mixed topology 不解决该 prefill path。 |
+| 6 | `TP=8, max-num-seqs=96, max-num-batched-tokens=24576` | 0.2708 | 0.9523 | batching 进一步增大后回退。 |
+| 7 | `TP=4, DP=1, max-num-seqs=96, max-num-batched-tokens=16384` | 0.2338 | 0.9590 | 少用 GPU 的 TP4/DP1 per-GPU 不占优。 |
+| 8 | `TP=8, DP=1, max-num-seqs=128, max-num-batched-tokens=16384` | 0.3217 | 0.9508 | 当前 best。 |
+
+这个结果说明：在 Qwen235B prefill case 上，harness 的价值不只是 topology
+选择，还包括在 TTFT/prefill 方向下做受约束的 runtime joint probe。最终 best 是
+`TP=8, DP=1, max-num-seqs=128, max-num-batched-tokens=16384`。
+
+## 正在运行：gpt-5.5 2x2 arm
+
+Session:
+
+```text
+tmux: aituner-q235b-2x2-gpt55-20260623T010038Z
+driver log: .aituner/qwen235b-prefill-2x2-gpt55-dash1-20260623T010038Z.driver.log
+```
+
+Driver timeline:
+
+```text
+harness clean pair start 2026-06-23T01:00:40+00:00
+harness clean pair done  2026-06-23T08:21:13+00:00
+naive clean pair start   2026-06-23T08:21:13+00:00
+```
+
+Harness side has completed all 8 trials:
+
+| Trial | Patch | req/s/GPU | Pass rate |
+| ---: | --- | ---: | ---: |
+| 1 | `TP=8, DP=1` | 0.2879 | 0.9522 |
+| 2 | `TP=8, max-num-seqs=96` | 0.2879 | 0.9530 |
+| 3 | `TP=8, max-num-batched-tokens=16384, max-num-seqs=96` | 0.3085 | 0.9561 |
+| 4 | `TP=8, max-num-batched-tokens=32768, max-num-seqs=144` | 0.2783 | 0.9543 |
+| 5 | `TP=8, DP=1, max-num-batched-tokens=24576, max-num-seqs=96` | 0.2654 | 0.9513 |
+| 6 | `TP=4, DP=2, max-num-batched-tokens=16384, max-num-seqs=96` | - | - |
+| 7 | `TP=8, DP=1, max-num-batched-tokens=16384, max-num-seqs=80` | 0.3156 | 0.9505 |
+| 8 | `TP=8, max-num-batched-tokens=32768, max-num-seqs=120` | 0.2879 | 0.9508 |
+
+Current harness best: `trial-0007`, `0.3156 req/s/GPU`.
+
+Naive side is still running. Current state:
+
+- Completed/recorded through trial-0003, with current best `0.2879 req/s/GPU`.
+- trial-0004 is active with `TP=8, DP=1, max-num-batched-tokens=8192,
+  max-num-seqs=128`.
+- trial-0004 probe history so far:
+
+| threshold | request rate | req/s/GPU | pass rate | feasible | main failures |
+| ---: | ---: | ---: | ---: | --- | --- |
+| 0.0625 | 1.5750 | 0.1969 | 0.9651 | true | TTFT misses and TTFT threshold violations |
+| 0.09375 | 2.3650 | 0.2956 | 0.7308 | false | `slo_pass_rate_unrecoverable`, TTFT violations |
+| 0.078125 | 1.9567 | 0.2446 | 0.9591 | true | TTFT misses and TTFT threshold violations |
+| 0.0859375 | 2.1667 | 0.2708 | 0.9546 | true | TTFT misses and TTFT threshold violations |
+
+As of the snapshot, vLLM is still processing requests for trial-0004, so the naive
+side has not produced its final result or report yet.
+
+## Prior Qwen235B context
+
+These earlier runs explain why the current 2x2 matters:
+
+| Run | Result | What it showed |
+| --- | --- | --- |
+| `qwen235b-prefill-clean-gpt55-dash1-20260621T160712Z` | harness 0.2879, naive 0.3217 | Earlier harness stopped/refined too weakly; naive found better final config. |
+| `qwen235b-prefill-seqguard-gpt55-dash1-20260622T064445Z` | harness 0.2879, naive 0.2577 | Seq guard prevented the worst early-stop failure but still did not reach the old naive best. |
+| `qwen235b-prefill-jointprobe-harness-dash2-20260622T132010Z` | harness-only 0.3085 | Joint `max-num-batched-tokens + max-num-seqs` probe improved over seqguard. |
+| `qwen235b-prefill-2x2-gpt54mini-dash3-20260623T010038Z` | harness 0.3217, naive no feasible | Weak model plus harness now reaches the old best and dominates weak naive. |
+
+The current evidence points to the harness needing both:
+
+1. topology discipline: stay on `TP=8, DP=1` for this prefill-heavy 235B setup;
+2. runtime joint probing: tune `max-num-batched-tokens` and `max-num-seqs` together
+   instead of stopping after the first feasible TP8 result.
+
+## Open item
+
+The final Qwen235B 2x2 conclusion is blocked on the still-running
+`gpt-5.5 + naive` arm on dash1. Once it completes, generate an aggregate report
+combining:
+
+- `qwen235b-prefill-2x2-gpt55-dash1-20260623T010038Z`
+- `qwen235b-prefill-2x2-gpt54mini-dash3-20260623T010038Z`
+
+and then update this progress report into a final ablation report.

docs/harness-ablation/qwen27b-tight-2x2-model-ablation-20260623.md

@@ -0,0 +1,366 @@
+# Qwen27B tight-SLO 2x2 harness ablation - 2026-06-23
+
+本文整理以下 aggregate report，并解释 harness 为什么能够让 tuning 更快、更有效：
+
+```text
+.aituner-reports/qwen27b-tight-2x2-aggregate-20260623T005838Z/report.md
+```
+
+这个实验是一个 2x2 ablation：模型强弱和是否启用 `use_harness` 交叉。
+核心问题是：harness 是否提供了可复用的搜索结构，而不仅仅是更强 LLM
+或者更长 prompt 带来的偶然收益。
+
+## 实验设计
+
+Case: `qwen27b-tight-slo-2x2-aggregate`。
+
+实验基座：
+
+- Served model: `qwen3.5-27b-256k-0223-internal`。
+- Hardware: H20，最多 8 GPUs。
+- Trace: `chat_w20260311_1000`，输入长度过滤到 0-8192 tokens，
+  `replay_time_scale=1.0`，`max_concurrency=32`。
+- SLO: pass rate >= 0.95；TTFT step rule 为 <=4096 input tokens 时 2s，
+  <=32768 input tokens 时 4s，更长输入时 6s；TPOT <= 50 ms。
+- Search: 在 `sampling_u in [0, 0.0625]` 上二分探测，tolerance 0.001，
+  max 6 probes。
+- Tunable envs: `VLLM_ENABLE_TORCH_COMPILE`。
+- Tunable flags: `tensor-parallel-size`, `data-parallel-size`,
+  `expert-parallel-size`, `gpu-memory-utilization`, `block-size`,
+  `max-num-batched-tokens`, `max-num-seqs`, `enable-prefix-caching`,
+  `enable-chunked-prefill`。
+- Topology constraints: TP 和 DP 均在 `{1,2,4,8}` 中，允许的 TP*DP product 为
+  `{1,2,4,8}`，本 case 中 EP 固定为 1。
+
+2x2 arms:
+
+| Arm | Tuner model | Harness | Trial budget used |
+| --- | --- | --- | ---: |
+| `gpt55_harness` | `gpt-5.5` | on | 2 |
+| `gpt55_naive` | `gpt-5.5` | off | 10 |
+| `gpt54mini_harness` | `gpt-5.4-mini` | on | 2 |
+| `gpt54mini_naive` | `gpt-5.4-mini` | off | 10 |
+
+同一个 tuner model 内，主要差异是 `use_harness`。跨模型比较则用来判断：
+更弱模型加 harness 是否能匹配或超过更强模型的 naive tuning。
+
+## Aggregate result
+
+Reference best: `0.4429 req/s/GPU`。
+Convergence target: reference 的 95%，即 `0.4208 req/s/GPU`。
+
+| Arm | Kind | Trials | Final req/s/GPU | Final/ref | Trials to target | Normalized AUC | Failed | No feasible |
+| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| `gpt55_harness` | harness | 2 | 0.4429 | 1.0000 | 2 | 0.9484 | 0 | 0 |
+| `gpt55_naive` | naive | 10 | 0.0273 | 0.0616 | - | 0.0588 | 2 | 2 |
+| `gpt54mini_harness` | harness | 2 | 0.4429 | 1.0000 | 2 | 0.9450 | 0 | 0 |
+| `gpt54mini_naive` | naive | 10 | 0.0231 | 0.0522 | - | 0.0498 | 1 | 1 |
+
+Harness-vs-naive 检查全部通过：
+
+| Harness arm | Final vs best naive | AUC vs best naive | Pass |
+| --- | ---: | ---: | --- |
+| `gpt55_harness` | 16.2290x | 16.1296x | true |
+| `gpt54mini_harness` | 16.2290x | 16.0720x | true |
+
+最关键的 ablation 信号是：`gpt-5.4-mini + harness` 和
+`gpt-5.5 + harness` 达到同一个 final throughput，也都是 2 trials 达到 target；
+而两个 naive arms 用满 10 trials 后仍低于 harness arms 16x 以上。
+
+## Agent loop 流程图
+
+下面是当前 harness 化 agent loop 的抽象流程。LLM 仍然可以参与 proposal，
+但它拿到的不是裸文本历史，而是结构化 observation、bottleneck diagnosis、
+candidate actions 和 validator 约束；同时 validator 可以授权 stop，也可以阻止
+重复失败或不合法配置。
+
+```mermaid
+flowchart TD
+    A[Study spec: trace, SLO, search range, tunable knobs] --> B[Run one engine config]
+    B --> C[Binary-search probes over sampling_u]
+    C --> D[Build observation o_t]
+    D --> E[Bottleneck classifier]
+    E --> F[Candidate family generator]
+    F --> G[Score candidate actions]
+    G --> H[Prompt renderer / planner]
+    H --> I[LLM or deterministic harness proposal]
+    I --> J{Config validator}
+    J -- invalid, repeated, unsafe --> F
+    J -- valid config_patch --> B
+    G --> K{Stop validator}
+    K -- search_high_saturated_by_incumbent --> L[Stop and keep incumbent]
+    K -- useful candidates remain --> H
+```
+
+这个 loop 中，harness 的作用不是把 prompt 写得更漂亮，而是把 tuning 变成
+一个受测量约束的决策过程：
+
+```text
+measurement -> diagnosis -> candidate family -> scored action -> validated proposal/stop
+```
+
+## 形式化设计：observation
+
+每个 trial 结束后，AITuner 不只记录一段自然语言总结，而是形成结构化 observation：
+
+```text
+o_t = (
+  config_t,
+  probe_history_t,
+  pass_rate_t,
+  latency/SLO_failure_profile_t,
+  request_rate_t,
+  parallel_size_t,
+  launch_status_t,
+  prior_failures_t,
+  incumbent_t
+)
+```
+
+本实验里 observation 中最重要的字段是：
+
+- `config_t`: 当前 trial 的 `flag_patch` 和 `env_patch`，例如 `TP=2, DP=1`。
+- `probe_history_t`: 在不同 `sampling_u` 下二分探测得到的 feasible/infeasible
+  结果。
+- `pass_rate_t`: 是否满足 target pass rate 0.95。
+- `latency/SLO_failure_profile_t`: TTFT 和 TPOT 哪个先触发 SLO pressure。
+- `request_rate_t`: 当前配置在 SLO 下能承载的 request rate。
+- `parallel_size_t`: 该配置实际使用的并行规模，用于归一化 per-GPU objective。
+- `prior_failures_t`: 之前哪些配置 launch failed 或 no feasible，避免重复试错。
+- `incumbent_t`: 当前最优配置及其 `request_rate_per_gpu`。
+
+目标函数是：
+
+```text
+J(config_t) = request_rate_t / parallel_size_t
+subject to pass_rate_t >= 0.95
+```
+
+也就是说，harness 优化的是满足 SLO 后的 `req/s/GPU`，不是 raw throughput，
+也不是 LLM 主观认为“更强”的配置。
+
+## 形式化设计：bottleneck classifier
+
+`bottleneck classifier` 把 observation 映射成 ranked bottleneck hypotheses：
+
+```text
+b_t = ranked_bottleneck(o_t)
+```
+
+它判断的不是“哪个 knob 看起来常用”，而是“当前 SLO failure 和 latency profile
+说明哪个系统环节在限制 objective”。
+
+常见分类包括：
+
+| Bottleneck | 典型证据 | 倾向 knob family |
+| --- | --- | --- |
+| `ttft_prefill` | 长 prompt 下 TTFT 接近或超过 SLO，prefill service time 是瓶颈 | 提高 TP，调整 prefill batching |
+| `decode_tpot` | TPOT p95/p99 超 SLO，decode token latency 是瓶颈 | 调整 `max-num-seqs`，提高 TP，降低 decode contention |
+| `admission_queueing` | waiting/arrival lag 增长，服务时间未必单独变差 | 提高 DP，调整 admission/concurrency knobs |
+| `memory_kv` | KV cache pressure、preemption、OOM、launch failure | 调整 `gpu-memory-utilization`、`block-size`、sequence/token caps |
+| `topology_comm` | TP 增加降低 latency 但 per-GPU efficiency 下降 | 回退 TP，比较 DP/TP tradeoff |
+
+本实验里，两个 harness arms 都把 ranked bottleneck 识别为
+`ttft_prefill`。原因是 workload 有 heavy-tailed long prompts，并且 TTFT SLO 很紧；
+这意味着单个请求的 prefill service time 是主要限制。DP-only 只能增加 replica，
+不能缩短一个长 prompt 的 prefill 路径，因此不是第一优先级。
+
+## 形式化设计：candidate family
+
+`candidate family generator` 根据 bottleneck 和 topology constraints 生成可比较的
+action family：
+
+```text
+A_t = candidate_knob_families(
+  b_t,
+  topology_constraints,
+  prior_failures_t,
+  incumbent_t
+)
+```
+
+在这个 case 中：
+
+- `b_t = ttft_prefill`。
+- 允许的 TP frontier 是 `TP=1 -> TP=2 -> TP=4 -> TP=8`。
+- 允许的 DP frontier 是 `DP=1,2,4,8`，但 DP-only 不直接缓解单请求 prefill
+  latency。
+- EP 固定为 1，因此不探索 expert parallel。
+- 之前没有 failed topology，因此相邻 TP probe launch risk 低。
+
+所以 harness 选择了：
+
+```text
+trial-0001: TP=2, DP=1
+trial-0002: TP=4, DP=1
+```
+
+这不是写死“Qwen27B 应该 TP4”。如果 classifier 输出的是
+`admission_queueing`，candidate family 会更偏向 DP 或 `max-num-seqs`；如果输出是
+`memory_kv`，则会更偏向 memory/cache/sequence knobs。
+
+## 形式化设计：scoring
+
+每个 candidate action 都按同一个抽象打分：
+
+```text
+score(a) = expected_bottleneck_relief(a)
+         + information_gain(a)
+         + launch_safety(a)
+         - regression_risk(a)
+         - measurement_cost(a)
+```
+
+这些项在本实验里的含义是：
+
+- `expected_bottleneck_relief`: TP2/TP4 预计能降低 long-prefill compute latency，
+  直接作用于 `ttft_prefill`。
+- `information_gain`: TP frontier probe 可以区分“需要 compute-latency relief”
+  还是“只是 admission/replica 不够”。
+- `launch_safety`: TP2/TP4 均满足 topology constraints，没有重复 failed signature。
+- `regression_risk`: TP 增加会带来通信开销，可能损害 per-GPU efficiency，所以必须用
+  `request_rate_per_gpu` 验证。
+- `measurement_cost`: 每个 GPU trial 成本高；因此高信息量的 topology probe 优先于
+  多个局部 runtime tweak。
+
+实际结果验证了这个 scoring：
+
+| Arm | Trial | Patch | req/s/GPU | Pass rate | 解释 |
+| --- | ---: | --- | ---: | ---: | --- |
+| `gpt55_harness` | 1 | `TP=2, DP=1` | 0.2142 | 0.9572 | 相邻 TP probe 已满足 SLO，但仍未饱和 search high。 |
+| `gpt55_harness` | 2 | `TP=4, DP=1` | 0.4429 | 0.9718 | TP frontier 继续缓解 prefill bottleneck，达到 reference best。 |
+| `gpt54mini_harness` | 1 | `TP=2, DP=1` | 0.1992 | 0.9707 | 弱模型也选择同一机制路径。 |
+| `gpt54mini_harness` | 2 | `TP=4, DP=1` | 0.4429 | 0.9727 | 弱模型加 harness 匹配强模型加 harness。 |
+
+## 形式化设计：validator stop
+
+Stop 不是 LLM 自己说“我觉得差不多了”。Stop 必须通过 `stop validator`：
+
+```text
+stop(o_t, incumbent_t, search_state_t, candidate_set_t) -> true/false
+```
+
+本实验里 stop 的记录是：
+
+```text
+tuning_stop_reason: harness_stop
+validator_reason: search_high_saturated_by_incumbent
+diagnosis: The incumbent's highest measured probe is feasible and is within the
+configured binary-search resolution of search.high.
+```
+
+含义是：
+
+1. 当前 incumbent 的最高测量 probe 已经 feasible。
+2. 该 feasible probe 距离 `search.high` 已经在 binary-search tolerance 内。
+3. 在当前搜索区间和 SLO 约束下，继续花 GPU trial 很难提高 measured objective。
+4. 因此 validator 授权 stop，并保留当前 incumbent。
+
+这给 harness 带来了 stop discipline：它既不会因为 LLM 过早自信而随便停，也不会在
+已经 saturate search high 后继续 burn budget。
+
+## 实际 tune 了哪些 knobs
+
+Harness winning path 只改了 topology：
+
+```text
+base config + tensor-parallel-size=4, data-parallel-size=1
+```
+
+它没有在 winning path 中调 scheduler/cache/memory knobs，因为 `ttft_prefill`
+bottleneck 下，首要动作是缩短单请求 prefill service time。
+
+Naive arms 则走了另一个方向：
+
+| Arm | 所有 trials 使用的 topology | 变化过的 runtime knobs | Best req/s/GPU |
+| --- | --- | --- | ---: |
+| `gpt55_naive` | `TP=1, DP=8` | `max-num-batched-tokens`, `max-num-seqs`, `block-size`, `gpu-memory-utilization`, prefix caching, chunked prefill | 0.0273 |
+| `gpt54mini_naive` | `TP=1, DP=8` | `max-num-batched-tokens`, `max-num-seqs`, `block-size`, `gpu-memory-utilization` | 0.0231 |
+
+`gpt55_naive` 的第一个 proposal 明确选择 `TP=1, DP=8`，理由是模型能单卡放下，
+因此 horizontal data parallelism 应该最大化 request rate，而 TP 会带来通信开销。
+之后 naive proposals 一直保留 DP-heavy topology，只围绕 runtime knobs 搜索。
+两个 naive arms 合计 20 个 trial slots 都没有进入 TP2/TP4 topology frontier。
+
+## 为什么比 baseline 更好
+
+Baseline 失败的原因是优化了错误的因果路径。
+
+对 `ttft_prefill`-bound workload，关键服务时间是单个请求的 prefill latency。
+DP-heavy topology 可以增加 replica 数，但每个 replica 仍用 TP1 处理长 prompt；
+它不能显著缩短单请求 prefill path。在 tight TTFT SLO 下，这会导致 feasible
+`sampling_u` 很低；再除以 GPU 数得到 `req/s/GPU` 后，结果只有
+`0.02-0.027 req/s/GPU`。
+
+Harness 的优化路径是：
+
+```text
+observed SLO pressure
+-> classify as ttft_prefill
+-> choose legal TP frontier probe
+-> measure feasible req/s/GPU under the same SLO
+-> stop only when search.high is saturated by incumbent
+```
+
+这条路径是可测量、可反驳的。如果 TP4 降低了 latency 但
+`request_rate_per_gpu` 明显下降，harness 会 reject 这个 hypothesis。如果
+bottleneck 是 admission/queueing 而不是 TTFT/prefill，同一个 knob-effect model
+会偏向 DP 或 `max-num-seqs`，而不是 TP frontier。
+
+因此，这个结果不是“Qwen27B case 里我们 prompt 诱导模型说 TP4”。更准确的结论是：
+harness 用 SLO-derived bottleneck evidence 把搜索导向了正确的 knob family，
+再用 per-GPU objective 和 validator stop 验证这个方向。
+
+## 证据边界
+
+这份报告强支撑 Qwen27B tight-SLO case 上的 harness 机制，但不能单独当作通用性证明。
+当前可成立的结论是：
+
+- 在这个 case 中，harness 同时提升了 final quality、convergence speed、AUC 和
+  stop discipline。
+- `gpt-5.4-mini + harness` 匹配 `gpt-5.5 + harness`，并显著超过
+  `gpt-5.5 + naive`，说明收益主要来自 harness 的结构化状态和 validator，而不是
+  单纯来自更强模型。
+- 成功路径用的是通用机制：SLO-derived bottleneck classification、topology
+  constraints、knob-effect scoring、per-GPU objective、validator-authorized stop。
+- 还需要在其他 bottleneck/case 上继续验证，例如 prefill scheduler pressure、
+  decode TPOT pressure、memory/KV pressure、admission/queueing pressure。
+
+## 原始 aggregate report 摘录
+
+```text
+# qwen27b-tight-2x2-aggregate-20260623T005838Z
+
+## Aggregate
+
+- Cases: `1`
+- Harness-vs-naive pass/checks: `2`/`2`
+- Winner counts: `{"final_best": {"gpt55_harness": 1}, "fastest_to_target": {"gpt55_harness": 1}, "normalized_auc": {"gpt55_harness": 1}}`
+
+## By Kind
+
+| Kind | Arms | Mean final/ref | Mean AUC | Target reached |
+| --- | ---: | ---: | ---: | ---: |
+| `harness` | 2 | 1.0000 | 0.9467 | 2 |
+| `naive` | 2 | 0.0569 | 0.0543 | 0 |
+
+## Cases
+
+### qwen27b-tight-slo-2x2-aggregate
+
+- Reference best req/s/GPU: `0.4429`
+- Target fraction: `0.95`
+- Winners: `{"final_best": "gpt55_harness", "fastest_to_target": "gpt55_harness", "normalized_auc": "gpt55_harness"}`
+
+| Arm | Kind | Trials | Final/GPU | Final/ref | TTT | AUC | Failed | No feasible |
+| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| `gpt55_harness` | `harness` | 2 | 0.4429 | 1.0000 | 2 | 0.9484 | 0 | 0 |
+| `gpt55_naive` | `naive` | 10 | 0.0273 | 0.0616 | - | 0.0588 | 2 | 2 |
+| `gpt54mini_harness` | `harness` | 2 | 0.4429 | 1.0000 | 2 | 0.9450 | 0 | 0 |
+| `gpt54mini_naive` | `naive` | 10 | 0.0231 | 0.0522 | - | 0.0498 | 1 | 1 |
+
+| Harness | Final vs best naive | Target speedup | AUC vs best naive | Pass |
+| --- | ---: | ---: | ---: | --- |
+| `gpt55_harness` | 16.2290 | - | 16.1296 | `True` |
+| `gpt54mini_harness` | 16.2290 | - | 16.0720 | `True` |
+```

docs/harness-ablation/qwen27b-tp-sweep-20260616.md

@@ -0,0 +1,51 @@
+# Qwen3.5-27B TP sweep under length-aware TTFT SLO — 2026-06-16
+
+Branch `feat/two-stop`. Deterministic ground-truth A/B (proposal files, no LLM):
+TP1 vs TP2 vs TP4 on the dense Qwen3.5-27B (internal 256k, fp8, spec-decode) at
+0–8k chat, vLLM 0.11.1, H20, `replay_time_scale=1.0` (no smoke), Stop-A enabled,
+pinned to GPUs 2–7.
+
+**SLO**: TTFT ≤ `4000 + 0.125·L_in` ms (= 4s + L_in/8k), TPOT ≤ 50 ms, pass ≥ 95%.
+
+## Result
+
+| config | best_u | raw req/s | req/s/GPU | pass | saturated |
+| --- | --- | --- | --- | --- | --- |
+| TP1 | 0.00195 | 0.065 | **0.065** | 1.00 | no |
+| TP2 | 0.0195 | 0.585 | **0.2925** | 0.96 | no |
+| TP4 | 0.123 | 3.63 | **≥0.908** | 0.98 | **yes (best_u≈high=0.125)** |
+
+- **Per-GPU throughput rises sharply with TP for the dense 27B**: TP2 = 4.5× TP1,
+  TP4 ≥ 14× TP1. Opposite of the MoE Qwen3-30B-A3B (TP1 best per-GPU) — confirms the
+  dense-vs-MoE distinction.
+- **Mechanism**: TP1 is TPOT-bound — one H20 cannot decode a 27B under 50 ms/token
+  once the batch grows, so it saturates at ~0.065 req/s/GPU. Loosening TTFT (2s→4-5s)
+  did *not* change TP1 (still 0.065), confirming TPOT — not TTFT — is TP1's binding
+  constraint. Each TP doubling speeds decode+prefill enough to more than recover the
+  added GPUs.
+- **TP4 saturated** the offered-load ceiling (`best_u=0.123 ≈ 0.125`): still feasible
+  after ~the whole trace, so 0.908 is a lower bound. True peak (and TP8) need a
+  raised `search.high` to measure.
+
+## Process findings (fed back into the harness)
+
+- **Bug fixed**: a request exceeding `request_timeout_s` raised a raw `TimeoutError`
+  mid-stream that escaped `_run_one_request` and crashed the whole trial; now wrapped
+  as `HttpClientError` (failed request, not failed trial). Commit `2fcaf80`.
+- **Open gap**: killing a `study tune` run orphans the `VLLM::EngineCore` workers
+  (SIGTERM/SIGKILL of the loop doesn't tear down the engine), which twice left leaked
+  GPU memory on GPUs 0/1 (dead PIDs still pinning KV, only clearable via root
+  `nvidia-smi --gpu-reset`). Fix: SIGTERM handler in the CLI loop + make
+  `_terminate_process_tree` match `EngineCore` workers, not just `vllm serve`.
+- Experiment hygiene: scale=1.0 makes each probe take real arrival time; `search.high`
+  must bracket the config's boundary (too wide wastes probes on a low-capacity config;
+  too low saturates a high-capacity one), and `request_timeout_s` must be modest so
+  overloaded probes drain fast.
+
+## Next
+
+- Re-measure TP4 (and TP8) with `search.high` raised (e.g. 0.5) to find the true peak
+  per-GPU and the TP knee.
+- Run the Stop-B agentic loop on this 27B stack: unlike the 30B (baseline already
+  optimal), here the loop should climb TP1→TP2→TP4 and stop — a real improving
+  trajectory (the original Phase-5 "A" goal).

docs/harness-ablation/qwen30b-slo-robustness-20260624.md

@@ -0,0 +1,164 @@
+# Qwen30B SLO robustness - 2026-06-24
+
+本文整理 Qwen30B-A3B community vLLM 0.20 case 在三档 SLO 下的 harness/naive
+对比，并解释不同 SLO 为什么没有导致完全不同的最终 topology，却改变了可承载负载边界
+和 bottleneck 判断。
+
+原始报告位于远端共享 checkout：
+
+```text
+.aituner-reports/qwen30b-slo-robust-gpt55-dash1-20260623T163521Z-strict/report.md
+.aituner-reports/qwen30b-slo-robust-gpt55-dash1-20260623T163521Z-medium/report.md
+.aituner-reports/qwen30b-slo-robust-gpt55-dash1-20260623T163521Z-loose/report.md
+```
+
+## 实验设计
+
+Case: `qwen30b-a3b-slo-{strict,medium,loose}-gpt55`。
+
+共同设置：
+
+- Served model: Qwen30B-A3B community vLLM 0.20。
+- Hardware: H20，允许 1/2/4/8 GPU topology。
+- Trace: chat 0-8k，输出长度 128。
+- Search: `sampling_u in [0, 1.0]`，tolerance 0.001，max 6 probes。
+- Objective: 在 pass rate >= 0.95 下最大化 `request_rate / used_gpu_count`。
+- Tuner model: `gpt-5.5`。
+
+三档 SLO：
+
+| SLO | TTFT step rule | TPOT |
+| --- | --- | ---: |
+| strict | <=4k: 1s, <=32k: 2s, else: 3s | 40 ms |
+| medium | <=4k: 2s, <=32k: 4s, else: 6s | 50 ms |
+| loose | <=4k: 4s, <=32k: 8s, else: 12s | 70 ms |
+
+## 结果摘要
+
+| SLO | Harness final req/s/GPU | Naive final req/s/GPU | Final speedup | AUC speedup | Harness TTT |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| strict | 2.2083 | 0.8000 | 2.7604x | 2.7886x | 1 |
+| medium | 3.2583 | 0.8000 | 4.0729x | 4.0729x | 1 |
+| loose | 3.2583 | 1.0458 | 3.1155x | 4.4622x | 1 |
+
+三个 SLO 下 harness 都在第一个 trial 到达该 SLO 下的 reference best。naive 在 8 个
+trials 内没有达到 95% reference target。
+
+## 最终 tune 出来的配置
+
+三档 SLO 的最终 best topology 都是：
+
+```text
+tensor-parallel-size = 2
+data-parallel-size = 1
+enable-expert-parallel = false
+```
+
+但这不表示 SLO 没有影响。SLO 改变的是同一个 topology 的可行负载上限：
+
+| SLO | Best config | Best sampling_u | Total req/s | req/s/GPU | Pass rate |
+| --- | --- | ---: | ---: | ---: | ---: |
+| strict | `TP=2, DP=1` | 0.484375 | 4.4167 | 2.2083 | 1.0000 |
+| medium | `TP=2, DP=1` | 0.750000 | 6.5167 | 3.2583 | 1.0000 |
+| loose | `TP=2, DP=1` | 0.750000 | 6.5167 | 3.2583 | 1.0000 |
+
+strict 到 medium/loose 的主要变化是 feasible frontier 右移：同一个 `TP=2, DP=1`
+配置在 strict 下只能稳定承载 `sampling_u=0.484375`，在 medium/loose 下可以承载
+`sampling_u=0.75`。
+
+## 为什么 `TP=2, DP=1` 稳定胜出
+
+AITuner 的 scoring 不是 raw throughput，而是 SLO-constrained per-GPU throughput：
+
+```text
+J(c, SLO) = max_u request_rate(c, u) / used_gpu_count(c)
+subject to pass_rate(c, u, SLO) >= 0.95
+```
+
+这解释了为什么 `TP=4` 没有赢。`TP=4` 的单请求 latency 更低、总吞吐可以更高，
+但它使用两倍 GPU，per-GPU objective 反而下降：
+
+| SLO | Config | Total req/s | Used GPUs | req/s/GPU | 解释 |
+| --- | --- | ---: | ---: | ---: | --- |
+| strict | `TP=2, DP=1` | 4.4167 | 2 | 2.2083 | strict best |
+| strict | `TP=4, DP=1` | 4.4167 | 4 | 1.1042 | latency 更低，但 GPU efficiency 更差 |
+| medium/loose | `TP=2, DP=1` | 6.5167 | 2 | 3.2583 | medium/loose best |
+| medium/loose | `TP=4, DP=1` | 8.3667 | 4 | 2.0917 | raw throughput 更高，但 per-GPU 不划算 |
+
+因此 harness 学到的不是“越多 GPU 越好”，而是更具体的机制：
+
+```text
+TP=1: 单请求 prefill/decode latency 偏高，SLO-constrained load frontier 低。
+TP=2: 足够缓解 latency，同时 GPU 数量仍低，per-GPU objective 最优。
+TP=4: 继续降低 latency，但通信和 GPU 数量成本超过收益。
+```
+
+## SLO 改变 bottleneck 的方式
+
+strict 下，`TP=2, DP=1` 在 `sampling_u=0.484375` 可行，但下一档
+`sampling_u=0.5` 直接进入 queueing collapse：
+
+| Point | Pass rate | 主要失败原因 |
+| --- | ---: | --- |
+| strict, `u=0.484375` | 1.0000 | 无 |
+| strict, `u=0.5` | 0.0290 | `tpot_ms>40`, `ttft_ms>1000/2000`, `slo_pass_rate_unrecoverable` |
+
+medium/loose 下，TTFT 阈值放宽后，同一 topology 能承载更高 arrival intensity。
+但是在 `u=0.765625` 仍会进入不可恢复的排队区：
+
+| SLO | Feasible point | Next infeasible point | 主要失败原因 |
+| --- | --- | --- | --- |
+| medium | `u=0.75`, pass 1.0000 | `u=0.765625`, pass 0.6900 | `tpot_ms>50`, `slo_pass_rate_unrecoverable` |
+| loose | `u=0.75`, pass 1.0000 | `u=0.765625`, pass 0.2900 | `tpot_ms>70`, `slo_pass_rate_unrecoverable` |
+
+这说明 SLO 放宽不是无限提高吞吐。服务系统还有 queueing stability frontier；
+超过 frontier 后，即使单个请求的 steady-state latency 看起来可控，排队也会让 pass rate
+迅速崩掉。
+
+## 其他候选配置的信号
+
+`TP=1, DP=1` 对 SLO 更敏感：
+
+| SLO | `TP=1, DP=1` req/s/GPU | 解释 |
+| --- | ---: | --- |
+| strict | 2.2000 | 接近 strict best，但略低于 `TP=2` |
+| medium | 2.2000 | 仍低于 `TP=2` |
+| loose | 2.8500 | 宽松 SLO 下受益明显，但仍低于 `TP=2` |
+
+`gpu-memory-utilization=0.92` 在 medium/loose 中与 `TP=2` 打平：
+
+| SLO | Config | req/s/GPU |
+| --- | --- | ---: |
+| medium | `TP=2, gpu-memory-utilization=0.92` | 3.2583 |
+| loose | `TP=2, gpu-memory-utilization=0.92` | 3.2583 |
+
+这说明该 workload 的主瓶颈不是 KV memory headroom，而是 topology 和 queueing
+frontier。
+
+EP family 在该环境下不稳定：
+
+```text
+TP=4, EP=2/4, enable-expert-parallel=true -> engine_launch exit_code=2
+```
+
+这些失败 trial 没有进入 best candidate，但它们说明当前 failure memory 还可以继续加强：
+同一类 EP launch failure 出现后，后续 proposal 应更积极地屏蔽该 family。
+
+## 对 paper claim 的含义
+
+这组实验支持的 claim 是：
+
+1. Harness 对 SLO 变化有稳定收益：strict/medium/loose 三档均显著优于 naive。
+2. Harness 不是固定写死某个 knob。它通过 SLO-constrained probing 找到 feasible
+   frontier；在本 case 中最终 topology 相同，但可承载负载边界随 SLO 改变。
+3. Harness 的 value 来自 topology-first candidate family、per-GPU scoring 和
+   validator 对 failed family 的处理，而不是自然语言 prompt 的偶然表达。
+
+这组实验尚不能单独 claim：
+
+- 所有模型和 workload 上都 robust。
+- `TP=2, DP=1` 是全局最优。
+- EP family 已经被最优处理。
+
+对应的后续证据应放在 roadmap 中跟踪：局部 grid/near-optimum、跨模型 2x2、跨 workload
+SLO robustness，以及 failure-memory ablation。

docs/harness-ablation/stop-a-validation-20260615.md

@@ -0,0 +1,121 @@
+# Stop-A validation (Phase 3) — 2026-06-15
+
+Branch `feat/two-stop`. Stop-A = truncate each probe's replay once the offered
+L-C-A of the replayed prefix converges to the full set (pure L-C-A criterion +
+C-gate). This note records the CPU calibration and the GPU fidelity check.
+
+## 1. Calibration (CPU, no serving)
+
+`scripts/stop_a_calibration.py` on the dash0 0321 10:00–10:10 windows:
+
+| dim | chat (19239 req, hit≈7%) | coder (2451 req, structured reuse) |
+| --- | --- | --- |
+| A | ≥0.95 by frac 0.10 | fast |
+| L | ≥0.96 from frac 0.05 | 0.05=0.75 (heavy tail) → ≥0.94 by 0.20 |
+| **C (slowest)** | noisy, dips (0.50→0.885, 0.55→0.835), stable ≥0.92 only ~0.85 | smooth, stable ≥0.92 by ~0.70 |
+
+Stop fraction (τ_L=τ_A=0.90, W=3):
+
+| τ_c | chat | coder |
+| --- | --- | --- |
+| 0.85 | 0.45 (273s) | 0.45 (255s) |
+| 0.90 | 0.70 (423s) | 0.55 (318s) |
+| 0.92 | 0.85 (513s) | 0.70 (411s) |
+
+Findings:
+- **C is the slowest dimension in both workloads** — reproduces paper §5.2 / Fig 9.
+- **What makes C hard to call converged is signal *noise*, not reuse magnitude.**
+  Low-reuse chat has a sparse/spiky ideal-hit-length series, so its C similarity
+  oscillates and is *harder* to stabilize than the structured, higher-reuse coder.
+  Consequence: a strict τ_c (0.92) gives chat only ~15% saving. A more robust C
+  feature for the low-reuse regime is future work.
+
+## 2. GPU fidelity check (Qwen3-30B-A3B, vLLM 0.11.1, H20)
+
+One full-window run (`adaptive_stop` disabled, `replay_time_scale=1.0`, window
+`chat_w20260311_1000`, 0–8k, out=128), then `scripts/stop_a_validate.py`
+recomputes each probe's convergence prefix and compares the truncated verdict to
+the full verdict — so a single GPU run validates truncation fidelity (no second run).
+
+Trial result: best feasible `sampling_u=0.078125`, request_rate **2.30 req/s**,
+pass_rate 0.973.
+
+Per-probe verdict (τ=0.9):
+
+| τ_c | verdict matches | mean replay saved |
+| --- | --- | --- |
+| 0.85 | 3/4 | 54% |
+| 0.90 | 3/4 | 52% |
+| 0.92 | 3/4 | 38% |
+
+The mismatch is the same probe at every τ_c — the feasibility knee `0.08594`:
+
+```
+thresh    full_pass  prefix_pass  full_feas  prefix_feas
+0.08594   0.946      0.956–0.961  False      True   <- mismatch
+0.07812   0.973      0.987–0.990  True       True
+0.06250   0.986      1.000        True       True
+0.09375   0.268      0.49–0.54    False      False
+```
+
+## 3. Interpretation
+
+- **Stop-A works and saves ~50% of replay** (vs the full 600 s window) while
+  preserving 3/4 probe verdicts. (The paper's ~70% is vs a 30-min fixed baseline;
+  our baseline is the 600 s window, so the percentages are not directly comparable.)
+- **The one failure is a boundary false-positive at the feasibility knee.** At
+  `0.08594` the full window is 0.946 (just below the 0.95 SLO) but the prefix is
+  0.956–0.961 (just above): the *second half* of the window degraded — engine-state
+  drift (KV fill / fragmentation / later-arriving harder requests) that the
+  *offered* L-C-A cannot see. The C-gate did not help because offered-C had
+  converged; the divergence is in the measured pass-rate, not in C.
+- If Stop-A were enabled, the binary search would accept `0.08594`, overestimating
+  the peak sustainable rate by one binary step (~10%).
+
+**This is the boundary jitter we accepted when choosing the pure-L-C-A criterion.**
+The data now argues for revisiting the previously-declined **SLO-boundary guard**:
+keep replaying while the measured pass-rate is within ±δ of the target, even after
+L-C-A converges. It targets exactly this knee case at low extra cost (it only
+extends replay on probes sitting on the feasibility boundary). Recommend adding it
+as a small Stop-A enhancement before enabling Stop-A in production studies.
+
+## 4. SLO-boundary guard (implemented + validated)
+
+Added `trace.adaptive_stop.boundary_delta` (default 0.02): when a truncated probe's
+measured pass-rate lands within ±δ of the SLO target, re-measure on the full window
+and use that verdict. Re-ran the same config with `adaptive_stop` enabled
+(τ=0.9, τ_c=0.90, δ=0.02):
+
+| threshold | feasible | pass | selected | replayed | boundary_extended |
+| --- | --- | --- | --- | --- | --- |
+| 0.06250 | True | 1.000 | 1086 | 487 (45%) | — |
+| 0.09375 | False | 0.444 | 1656 | 822 (50%) | — |
+| 0.07812 | True | 0.994 | 1378 | 682 (49%) | — |
+| 0.08594 | **False** | 0.947 | 1523 | **1523 (100%)** | **True** |
+
+Result: best feasible `sampling_u=0.078125` (rate 2.30 req/s) — **identical to the
+full-replay baseline**. The guard fired on exactly the one knee probe and
+re-measured it to the correct infeasible verdict; the other three probes truncated
+to ~45–50%. Net replayed 3514/5643 requests ≈ **38% replay saved on this trial
+while recovering the correct peak rate** (no one-step overestimate).
+
+**Conclusion: Stop-A with the boundary guard is correct (verdict matches full
+replay) and still saves replay time. Safe to enable.** Configs:
+`dash0_qwen30b_a3b_stopA_fulldata.json` (OFF baseline) and
+`dash0_qwen30b_a3b_stopA_on.json` (ON).
+
+## Repro
+
+```
+# calibration
+PYTHONPATH=src python3 scripts/stop_a_calibration.py \
+  --jsonl <DIR>/qwen_chat_blksz_64_032109-032111.jsonl --block-size 64 \
+  --window-start 3600 --window-end 4200 --gpu-count 8 --label chat
+# GPU run + fidelity
+PYTHONPATH=src python3 -m aituner.cli study tune \
+  --spec configs/examples/dash0_qwen30b_a3b_stopA_fulldata.json \
+  --store-root .aituner/stopA-fulldata --max-trials 1
+PYTHONPATH=src python3 scripts/stop_a_validate.py \
+  --spec configs/examples/dash0_qwen30b_a3b_stopA_fulldata.json \
+  --store-root .aituner/stopA-fulldata --tau 0.9 --tau-c 0.90
+```

docs/harness-ablation/stop-b-e2e-20260615.md

@@ -0,0 +1,86 @@
+# Stop-B end-to-end validation (Phase 5) — 2026-06-15
+
+Branch `feat/two-stop`. Real agentic loop on dash0: Qwen3-30B-A3B / vLLM 0.11.1 /
+8×H20, `gpt-5.4` (via codex/prism) proposing configs, Stop-A enabled to accelerate
+each evaluation, `use_harness=True` so the Stop-B deterministic validator + LLM-stop
+veto are active. Config `dash0_qwen30b_a3b_stopB_e2e.json`, `search.high=1.0`,
+`max_probes=6`, `--max-trials 8`.
+
+## Two stop paths exercised
+
+**Run A (`search.high=0.125`)** — the default config already saturates the offered-load
+search range, so Stop-B fired immediately via the **search-high-saturation** path:
+`stop_authorized_by: validator`, reason *"the incumbent's highest measured probe is
+feasible and within the binary-search resolution of search.high."* Correct
+measurement-ceiling stop (no point proposing configs when the load range, not the
+config, is the bound).
+
+**Run B (`search.high=1.0`)** — forces a real multi-iteration search:
+
+| trial | TP | DP | EP | feasible | raw req/s | **req/s/GPU** | source |
+| --- | --- | --- | --- | --- | --- | --- | --- |
+| 0001 | 1 | 1 | 1 | ✓ | 2.90 | **2.900** | baseline |
+| 0002 | 2 | 1 | 1 | ✓ | 4.42 | 2.208 | harness TP-seed |
+| 0003 | 2 | 1 | 2 | ✗ launch-fail | — | — | harness (EP) |
+| 0004 | 1 | 2 | 1 | ✓ | 4.42 | 2.208 | LLM (after veto) |
+| 0005 | 2 | 2 | 1 | ✓ | 8.37 | 2.092 | harness |
+| 0006 | 2 | 2 | 2 | ✗ launch-fail | — | — | harness (EP) |
+| 0007 | 4 | 1 | 1 | ✓ | 8.37 | 2.092 | LLM |
+| (0008) | — | — | — | **STOP** | — | — | LLM stop, honored after veto budget |
+
+Incumbent: **trial-0001 (TP1), 2.90 req/s/GPU — never beaten.**
+
+> **⚠️ The per-GPU trajectory above is NOT a valid benchmark — it validates only
+> the Stop-B *mechanics*.** Two confounds:
+> 1. **Trace-ceiling saturation.** TP2·DP2 and TP4 reached `best_sampling_u≈0.98`
+>    (still feasible after consuming ~the whole window), so their *true* peak
+>    per-GPU is higher than the 2.09 shown — we ran out of offered load to push
+>    them to their boundary. Only TP1 (u=0.31), TP2 (u=0.48) and DP2 (u=0.48)
+>    found real boundaries. The `sampling_u` axis maxes at the full trace, so any
+>    config that sustains more than the window's offered rate cannot be measured.
+> 2. **Smoke regime.** This run inherited `replay_time_scale=0.1` +
+>    `max_requests_per_probe=512` (README: convergence test, *not* a benchmark) —
+>    compressed arrivals distort A and the 512 cap imposes a ~8.4 req/s ceiling.
+>
+> The below-ceiling TP1 (2.90) > TP2 (2.21) ordering *may* be real for this model
+> (Qwen3-30B-A3B is an MoE with ~3B active params → little compute per token → TP
+> adds all-reduce overhead with little benefit), which differs from the dense
+> Qwen3.5-27B where TP2 wins. But this run cannot establish it. A valid benchmark
+> needs `scale=1.0`, no cap, and enough offered-load headroom that strong configs
+> are not trace-saturated — see the 27B TP A/B follow-up.
+
+## Phase-5 acceptance
+
+- **No regression.** The primary metric `request_rate_per_gpu` stayed 2.90 the whole
+  run. Scaling TP/DP raised *raw* throughput (4.42, 8.37) but lowered per-GPU
+  efficiency (2.21, 2.09); the loop correctly kept the TP1 baseline as incumbent and
+  never adopted a worse-per-GPU config. (Matches the paper: long-prompt, low-reuse
+  chat prefers small TP for per-GPU efficiency.)
+- **Stop-B authority validated live.** At trial 4 the LLM tried to stop
+  (`should_stop=true`); the deterministic validator **vetoed** it
+  (`validator_did_not_authorize_stop`, `continue_harness_guided_search`), forcing one
+  more confirmation (DP2, which also failed to beat baseline). After the budget, the
+  LLM's later, well-justified stop was honored (`stop_authorized_by:
+  llm_after_veto_budget`). The bounded veto behaved exactly as designed.
+- **Auditable reason chain.** Every stop/veto carries a diagnosis grounded in the
+  measured evidence (e.g. *"increasing TP 1→2 lowers per-GPU efficiency even though
+  token latency improves … EP is explicitly blocked by launch-failure evidence"*).
+- **Launch-failure robustness.** Two EP configs (trial-0003, 0006) failed to launch
+  under vLLM 0.11.1; the harness recorded them as hard-negative evidence and the LLM
+  explicitly stopped proposing EP.
+
+## Notes / limitations
+
+- For this workload the baseline (TP1) is already per-GPU optimal, so iterations-to-
+  *best* = 1; the remaining trials are the loop *confirming* no config beats baseline
+  before stopping. A workload with an under-tuned default would show an improving
+  trajectory; this run validates the stop/no-regression machinery, not a tuning win.
+- The final stop came via `llm_after_veto_budget` (validator vetoed once, then
+  deferred), not a pure deterministic validator stop — because the deterministic
+  conditions (3-within-2%, saturation, validation-exhausted) did not cleanly fire
+  when every trial was a distinct config with a distinct per-GPU rate. The validator
+  acted as the *guard* (preventing premature stop), which is its designed role.
+- 7 trials > the paper's 3–6 average, inflated by the wider search range, 2 EP
+  launch-failures, and the veto. Acceptable for a validation run.
+- LLM token: the non-interactive shell lacks `OPENAI_API_KEY`; export it from the
+  codex `auth.json` (`~/.codex/auth.json`) before the run.

docs/harness-ablation/stop-b-e2e-27b-20260616.md

@@ -0,0 +1,64 @@
+# Stop-B end-to-end on dense Qwen3.5-27B (the improving trajectory) — 2026-06-16
+
+Branch `feat/two-stop`. Real `gpt-5.4` agentic loop (codex/prism), Stop-A enabled,
+length-aware TTFT SLO (4s + L_in/8k, TPOT ≤ 50 ms), vLLM 0.11.1, H20, GPUs 2–7,
+`replay_time_scale=1.0`, `search.high=0.25`, `inherit_incumbent_floor=true`.
+Config `dash0_qwen27b_stopB_loop.json`. Companion to the 30B run
+(`stop-b-e2e-20260616.md`); together they cover all Stop-B behaviors.
+
+## Trajectory (incumbent = TP4 @ 1.00 req/s/GPU)
+
+| iter | proposed by | config | per_gpu | adopted? |
+| --- | --- | --- | --- | --- |
+| 1 | baseline | TP1 | 0.123 | incumbent |
+| 2 | gpt-5.4 | TP2 | 0.2925 (2.4×) | ✅ new incumbent |
+| 3 | gpt-5.4 | TP4 | **1.0012 (8.1×)** | ✅ new incumbent |
+| 4 | gpt-5.4 | TP4 + chunked-prefill + mbt=16384 | 0.942 | ❌ **worse → rejected** |
+| 5 | gpt-5.4 | TP2 + DP2 | (loop stopped before completing) | — |
+
+(Run stopped manually after iter-4 — see "Why stopped" below. Incumbent preserved
+at TP4.)
+
+## What this demonstrates (the piece the 30B run could not)
+
+- **A genuine improving climb.** `gpt-5.4` + the harness raised per-GPU throughput
+  TP1 → TP2 → TP4 (0.123 → 0.29 → 1.00, 8.1×), each step a correctly-diagnosed real
+  gain: TP1 is TPOT-bound, so the agent scaled tensor-parallelism, then — once
+  topology was won — pivoted to **runtime tuning on the winning family** (chunked
+  prefill + larger batched tokens).
+- **No regression.** The runtime tweak (iter-4) measured *below* plain TP4
+  (0.942 < 1.00), and the harness correctly **kept TP4 as the incumbent** rather than
+  adopting the worse config — the core Stop-B guarantee, shown live.
+- Combined with the 30B run (search-high-saturation `validator`-authorized stop +
+  premature-LLM-stop veto), every Stop-B behavior is now validated end-to-end:
+  improving climb, correct bottleneck-driven proposals, no regression, deterministic
+  stop authority, and the LLM-stop veto.
+
+## Process wins / findings
+
+- **SIGTERM teardown fix validated in practice.** This loop was stopped with a plain
+  SIGTERM and the engine + EngineCore workers torn down cleanly — GPUs 2–7 freed, no
+  orphan, no leaked memory (contrast: the pre-fix runs twice leaked GPU0/1). Commit
+  `b17b213`.
+- **Timeout-as-failed-request fix** (`2fcaf80`) held — no trial crashed on
+  request timeouts this run.
+
+## Why stopped (efficiency finding — feeds next round)
+
+The loop was stopped after iter-4 rather than run to an explicit Stop-B firing,
+because each TP4-family trial took ~3 h: at `scale=1.0`, infeasible high-θ probes
+each run to the **`early_stop_max_elapsed_s=900` cap** (`probe_elapsed_s>900`), and
+the primary+fallback binary search doubles the probe count. Stop-A truncates a
+*converged* replay but does not shortcut an *overloaded* probe that simply runs out
+the clock. **For a practical agentic loop at scale=1.0, lower `early_stop_max_elapsed_s`
+(≈300 s)** so overloaded probes die fast; consider also having an infeasible-and-
+overloaded probe early-stop on a fast lag/throughput signal rather than the elapsed
+cap. The convergence itself was already evident (iter-4's runtime tweak and the
+queued TP2+DP2 were not beating TP4).
+
+## Next
+
+- Lower the elapsed cap and (optionally) re-run to capture the explicit Stop-B stop
+  on this 27B stack.
+- Land the deferred items: more robust C feature for the low-reuse regime; Stop-C
+  cross-day retune trigger; §7 baselines (SCOOT/naive/community).

docs/intervention-response-v0-protocol-20260714.md

@@ -0,0 +1,71 @@
+# Telemetry intervention-response v0 protocol
+
+Status: **FROZEN BEFORE V0 ANALYSIS**.
+
+Date: 2026-07-14 (Asia/Singapore).
+
+## Claim boundary
+
+The closed residual route asked whether one absolute engine-state snapshot can
+predict unmeasured configurations.  V0 asks a different, narrower question:
+
+> Does an adjacent, controlled MNS intervention produce an early engine-state
+> response that is distinguishable from same-config repeat noise?
+
+Passing this gate only authorizes a matched real-GPU pilot.  It does not prove
+that telemetry improves tuning, that any metric is a causal mediator, or that
+the response transfers to a new workload, topology, or knob family.
+
+## Data and estimand
+
+- Source: Phase 6 solo-authoritative Qwen3-30B-A3B/vLLM 0.24 Layer-1 streams.
+- Action pairs: primary runs at identical study hash, TP, sampling anchor, and
+  request-order hash, with adjacent `MNS={8,16,32,64}` values.
+- Noise pairs: primary versus confirmation at the same complete config,
+  anchor, and request-order hash.  Only primary-to-confirmation pairs are used;
+  confirmations are not combined into pseudo-independent all-pairs.
+- Fixed early windows: 5 seconds and 10 seconds from the measured interval
+  start.  All runs exceed 10 seconds, so early-stop censoring cannot change the
+  telemetry window.
+- Full-run pass rate and feasibility are descriptive only because an early
+  stop can make full elapsed durations differ.
+
+The statistical unit is a run pair.  Scheduler steps are summarized within a
+run and are never counted as independent trials.
+
+## Frozen response gate
+
+The directly measured gate features are scheduler-step rate, decode-batch
+mean, prefill-token fraction, waiting/running queue mean, KV-usage mean, and
+CUDA-graph padding fraction.
+
+A feature qualifies at one horizon only if:
+
+1. at least 75% of nonzero action deltas have the same sign;
+2. median absolute action delta is at least 2x the median absolute repeat
+   delta; and
+3. at least 50% of action deltas exceed the repeat-noise absolute p95.
+
+V0 opens a GPU pilot only if:
+
+- there are exactly 17 frozen adjacent-MNS action pairs;
+- there are at least 20 primary/confirmation repeat pairs;
+- all identity, finite-value, counter, and ratio invariants pass; and
+- at least two gate features qualify at both 5 and 10 seconds.
+
+Any data red flag stops the analysis before interpreting the response.
+
+## If V0 passes
+
+Register a dash0 pilot around a known scaling knee.  The pilot must use the
+same request sequence and arrival times, one serving job at a time, one changed
+knob, randomized `A/B` versus `B/A` order, common non-censored measurement
+windows, and trial-level repetitions.  It must compare a response-aware next
+action against an outcome-only policy under complete startup, warm-up, and
+H20-hour accounting.
+
+## If V0 fails
+
+Do not add telemetry fields or train a larger model.  The current Layer-1 state
+does not identify even an MNS intervention above repeat noise on this task, so
+the telemetry-guided tuning route remains diagnostic only.

docs/intervention-response-v0-results-20260714.md

@@ -0,0 +1,157 @@
+# Telemetry intervention-response v0/v1 results
+
+Date: 2026-07-14 (Asia/Singapore).
+
+## Decision
+
+**STOP before a new H20 pilot.**  The current Layer-1 aggregate telemetry does
+not identify a sufficiently general early response to an MNS intervention,
+and it does not improve action-efficacy prediction over exact external prefix
+outcomes on the available development tasks.
+
+This is a negative result about the present state representation and
+experiment design.  It does not establish that engine telemetry is useless for
+tuning, and it is not held-out evidence.
+
+## Hypothesis and frozen test
+
+The tested hypothesis was:
+
+> With the workload and all non-MNS settings held fixed, increasing MNS causes
+> a 5--10 second engine-state response that is larger than same-config repeat
+> noise and that predicts whether the action makes the full run feasible.
+
+A response feature had to satisfy all three frozen conditions at both 5 and
+10 seconds: at least 0.75 sign consistency, median absolute action effect at
+least 2x the repeat median, and at least 0.50 of action deltas above the repeat
+absolute p95.  At least two features had to pass.  A telemetry feature was
+decision-relevant only if its leave-one-repeat-out balanced accuracy was at
+least 0.75 and at least 0.15 above the best exact external prefix outcome.
+
+## What was implemented
+
+- A common-window analyzer over the existing per-scheduler-step Layer-1 stream.
+- Exact action pairing with request-order hash, offered load, TP, load role,
+  and repetition held fixed.
+- Same-config repeat-noise estimation without treating scheduler steps as
+  independent samples.
+- Exact 5/10-second request-prefix outcomes using monotonic completion times.
+- A one-feature leave-one-repeat-out efficacy audit; no multivariate model was
+  fitted to the 12 examples.
+- Input hashes, stream hashes, frozen thresholds, pair-level deltas, and sanity
+  invariants in machine-readable audit artifacts.
+- Trial-by-trial validation against the P1 manifest, plus content hashes for
+  every result, request file, and Layer-1 stream.
+
+## Experiment A: Phase-6 retrospective audit
+
+Phase 6 supplied 17 adjacent-MNS actions and 29 same-config
+primary/confirmation pairs.  No feature passed at either horizon, producing
+`STOP_NO_IDENTIFIABLE_RESPONSE`.
+
+The confirmation sample is not a clean replication distribution: confirmations
+were selectively run after disputed primary outcomes.  Several same-config
+pairs consequently followed radically different trajectories.  This result
+therefore remains a valid failure of the frozen v0 gate, but it cannot by itself
+separate normal run variance from confirmation-selection bias.
+
+## Experiment B: prospective-repeat confirmation
+
+P1 supplied three pre-arranged, disjoint request bands for every cell/load.
+Exact matched actions exist for TP1 `MNS 8 -> 64` and TP4 `MNS 16 -> 64`, at
+low/high load and repetitions 1/2/3.  This yields 12 action pairs and 24
+same-config consecutive-repeat pairs.
+
+The 24 adjacent repeat differences share their middle repetition within each
+three-run group.  They define a conservative empirical noise reference; they
+are not used as 24 independent samples in an inferential test.
+
+The result is `STOP_NO_PROSPECTIVE_RESPONSE`: zero features passed the response
+gate at either horizon.
+
+The strongest response was mean waiting-queue occupancy:
+
+| Horizon | Sign consistency | Action/repeat median | Action above repeat p95 | Gate |
+|---|---:|---:|---:|---|
+| 5 s | 1.000 | 1.292x | 0.167 | fail |
+| 10 s | 1.000 | 2.611x | 0.250 | fail |
+
+The direction is real enough to merit diagnosis, but the effect is not broad
+enough to guide a general action.  It is large for TP4/high-load trials and
+small or absent in other regimes.
+
+Full-run transitions contain six beneficial actions (`false -> true`) and six
+non-beneficial actions (three `false -> false`, three `true -> true`).  The
+beneficial label is also perfectly confounded with TP4 in this small dataset,
+so it cannot support a topology-general claim.
+
+| Horizon | Best telemetry delta | Balanced accuracy | Best external prefix delta | Balanced accuracy | Telemetry advantage |
+|---|---|---:|---|---:|---:|
+| 5 s | waiting queue | 0.750 | max TPOT / SLO | 0.833 | -0.083 |
+| 10 s | waiting queue | 0.750 | outstanding / admitted | 0.750 | 0.000 |
+
+No telemetry feature reaches the preregistered `+0.15` incremental threshold.
+
+## What this rules out
+
+It rules out using the current vector of 5/10-second global means as a solid
+mechanism for choosing the next config.  In particular, adding these aggregates
+to an LLM prompt or fitting a larger predictor would currently hide, rather
+than solve, the identifiability problem.
+
+It does not rule out an instrumentation-aware tuner built around a deliberately
+excited local system.  The existing runs were designed for endpoint/fidelity
+evaluation, not system identification: the MNS action is large, efficacy is
+confounded with TP, repeat bands contain different requests, and global means
+erase when queue buildup or service-rate changes occur.
+
+## Required redesign before spending H20-hours
+
+The next admissible experiment is a randomized, local A/B system-identification
+pilot around one fixed TP and one load knee:
+
+1. Replay the exact same request sequence and arrival times for both endpoints.
+2. Use small adjacent actions and randomized `A/B` versus `B/A` order.
+3. Record event-aligned response curves, including queue growth/drain rate,
+   prefill/decode service rate, and per-step service time, rather than only one
+   global mean.
+4. Separate a mechanism gate (repeatable response) from the end-to-end gate:
+   fewer trials or H20-hours to select a feasible near-optimal config than an
+   outcome-only tuner.
+5. Hold out a second load/workload for the final policy comparison.
+
+Until that design is frozen, a wider sweep would only generate more correlated
+observations and is not justified by the evidence above.
+
+## Reproduction
+
+```bash
+python3 runs/intervention-response-v0/test_analysis.py
+python3 runs/intervention-response-v0/test_p1_analysis.py
+
+python3 runs/intervention-response-v0/analyze_phase6.py \
+  --metrics runs/opprof-phase6/phase6/metrics.json \
+  --raw-root runs/opprof-phase6/phase6/solo-authoritative/cells \
+  --output runs/intervention-response-v0/phase6-audit.json
+
+python3 runs/intervention-response-v0/analyze_p1.py \
+  --run-root /home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b \
+  --manifest /home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/pilot-manifest.json \
+  --output runs/intervention-response-v0/p1-audit.json
+```
+
+## Data sanity
+
+- Phase 6: action pairs `n=17`, repeat pairs `n=29`, trials `n=66`; MNS
+  action size min/max `8/32`, `3` distinct; action-state vectors `n=17`, `17`
+  distinct; streams `n=12`, bytes min/max `12,745,297/52,957,710`, `12`
+  distinct.
+- P1: action pairs `n=12`, repeat pairs `n=24`, trials `n=36`; MNS action
+  size min/max `48/56`, `2` distinct; efficacy labels `n=12`, min/max `0/1`,
+  `2` distinct; streams `n=6`, bytes min/max `17,449,143/29,431,988`, `6`
+  distinct.
+- Checked invariants: exact action request hashes and offered loads match;
+  all `36/36` P1 trials match the manifest; expected pair counts hold; all
+  deltas are finite; non-negative counters and bounded ratios hold; per-config
+  state vectors are not all identical; both efficacy classes are present.  No
+  red flags were observed.

docs/intervention-response-v1-p1-protocol-20260714.md

@@ -0,0 +1,58 @@
+# Intervention-response v1 prospective-repeat confirmation
+
+Status: **FROZEN AFTER PHASE-6 V0 FAILURE AND BEFORE P1 RESPONSE ANALYSIS**.
+
+Date: 2026-07-14 (Asia/Singapore).
+
+## Why this is a new confirmation, not a relaxed V0
+
+Phase-6 V0 failed its frozen global response gate.  Its 29 same-config
+confirmations were triggered after disputed outcomes, and the resulting noise
+sample contains extreme trajectory divergence by construction.  V0 remains
+failed and its thresholds are unchanged.
+
+The already-completed P1 campaign supplies a distinct test: three
+prospectively scheduled, disjoint repeat bands for every cell/load.  TP1 and
+TP4 use identical offered loads and exact request-order hashes across their MNS
+endpoints.  V1 asks whether an MNS response is identifiable against this
+prospective workload-repeat noise, and whether that response predicts action
+efficacy beyond exact external prefix outcomes.
+
+P1 is now development data.  No result here is held-out or paper-facing.
+
+## Frozen pairs
+
+- Action pairs: TP1 `MNS 8 -> 64` and TP4 `MNS 16 -> 64`, at low/high load and
+  repeat 1/2/3.  Endpoints must have identical TP, offered rate, repeat role,
+  and request-order hash. Expected `n=12`.
+- Repeat-noise pairs: consecutive pre-arranged repeat bands within each of six
+  cells and low/high load: `rep1 -> rep2`, `rep2 -> rep3`. Expected `n=24`.
+  Repeat bands intentionally contain different requests and therefore include
+  workload-sampling noise rather than pretending to be identical trials.
+  Adjacent differences share the middle run; the gate uses their empirical
+  magnitude only and does not treat the 24 differences as independent samples
+  for a p-value or confidence interval.
+- Prefix horizons: 5 and 10 seconds. Exact monotonic request completion times
+  and the same Layer-1 intervals are used.
+
+## Frozen gates
+
+The response-identifiability thresholds are exactly the Phase-6 V0 thresholds:
+75% sign consistency, 2x median effect/repeat noise, and at least 50% of action
+deltas above repeat absolute p95. At least two response features must qualify
+at both horizons.
+
+Action efficacy is one only for an infeasible-to-feasible full-run transition.
+The 12 action pairs must contain at least four examples of each class.
+
+For decision relevance, each individual external-outcome response feature and
+each individual telemetry-response feature is evaluated by leave-one-repeat-
+band-out threshold fitting. This intentionally avoids a multivariate model on
+12 examples. At least one telemetry feature must, at both horizons:
+
+1. reach balanced accuracy at least 0.75; and
+2. exceed the best external-outcome response feature by at least 0.15.
+
+Only if data validity, response identifiability, and incremental decision
+relevance all pass does V1 open a newly registered matched GPU pilot. No
+threshold or feature is changed after observing V1.

docs/intervention-response-v2-phase-aware-protocol-20260714.md

@@ -0,0 +1,94 @@
+# Phase-aware telemetry intervention-response v2 protocol
+
+Status: **INVALID OPERATIONAL ATTEMPT; SUPERSEDED BY V3**.
+
+Date: 2026-07-14 (Asia/Singapore).
+
+The first MNS=16 session timed out while draining the 3.125 requests/s/GPU
+workload after its 300-second arrival window.  It produced no high-load result,
+and no MNS=64 endpoint was run.  No comparative conclusion is drawn from this
+attempt; see `intervention-response-v3-two-load-protocol-20260714.md`.
+
+## Correction to v0/v1
+
+The 5/10-second analyses tested an ultra-early verifier.  They did not test
+whether telemetry observed after the engine has developed queue, batch, and KV
+state can guide tuning.  The P1 replay lasts 60 seconds after time scaling, and
+the 5/10-second prefixes contain only a small fraction of its requests.
+
+V2 therefore replaces absolute cutoffs with replay phase.  The old audits and
+their negative decisions remain immutable, but their claim is narrowed to the
+first 5/10 seconds.
+
+## Historical corrective audit
+
+The historical audit is development-only and cannot become confirmatory after
+the horizon concern was observed.
+
+- Infer each trial's intended replay duration as selected requests divided by
+  offered requests per second.  All trials must agree.
+- Find every complete 10% replay decile supported by every trial.  Analyze all
+  such deciles; selecting only the best horizon is forbidden.
+- At each decile report both:
+  - cumulative state from replay start to the checkpoint; and
+  - the non-overlapping 10%-wide state block ending at the checkpoint.
+- Report admitted/completed request coverage, response-versus-repeat statistics,
+  telemetry versus external-outcome efficacy, and per-feature trajectory drift.
+- Reuse the frozen v1 action and repeat pairs and the frozen response and
+  incremental-efficacy thresholds.  These thresholds are descriptive in V2;
+  passing one post-hoc horizon does not open a contribution claim.
+
+If early stopping prevents complete observation of the replay phases, the
+historical decision is `REQUIRES_UNCENSORED_PHASE_AWARE_PILOT`, independent of
+which early decile looks best.
+
+## Uncensored matched pilot
+
+The pilot is a mechanism gate, not paper evidence.
+
+- Hardware/engine/model: solo placement on dash0, 4 NVIDIA H20 GPUs, patched
+  vLLM `0.24.1.dev3+opprof`, Qwen3-30B-A3B, fixed `TP=4`.
+- Action: `MNS 16 -> 64`; topology, model, engine build, workload, arrival
+  sequence, offered load, and all other settings remain fixed.
+- Workload: `chat_w20260312_1000` at replay-time scale `0.5`, hence 300 seconds.
+- Offered loads per GPU: `1.5`, `2.125`, and `3.125` requests/s.  These supply a
+  low control and the two already-observed P1 pressure regimes.
+- Repetitions: three disjoint session bands, exact request sequence matched
+  across action endpoints.  Endpoint order alternates `A/B`, `B/A`, `A/B`;
+  load order is counter-rotated across repetitions.
+- A fresh server receives the accepted long-request warm-up and a bounded
+  burn-in before each measured session.
+- SLO-unrecoverable early stop is disabled.  Every run must observe the full
+  300-second arrival window; a separate 360-second safety deadline may mark a
+  run invalid but cannot manufacture a full-run label.
+- Cumulative checkpoints: 10%, 25%, 50%, 75%, and 100%, or 30/75/150/225/300
+  seconds.  Quarter blocks are analyzed separately from cumulative means.
+- A measured Layer-1 interval is complete only when its start-boundary,
+  end-boundary, and maximum internal record gaps are each at most one second;
+  timestamps must be monotonic.
+- Placement is serialized.  Co-location remains forbidden because Phase 6
+  observed material co-location-induced outcome shifts.
+- Hard cap: 8 H20-hours including startup, warm-up, burn-in, invalid attempts,
+  and cleanup.
+
+## Gates
+
+Data validity requires complete 300-second Layer-1 coverage, zero dropped
+records, exact request/arrival/length hashes across action endpoints, monotonic
+timestamps, full request accounting, idle GPUs before and after each session,
+and no co-resident GPU process.
+
+Mechanism evidence requires at least two telemetry features whose matched action
+response exceeds same-config repeat noise at the same pair of consecutive
+checkpoints under the unchanged v1 response thresholds.  Those features must
+also have a consistent direction in at least two of the three load regimes.
+
+Decision evidence additionally requires both action-efficacy classes and at
+least one of the phase-stable mechanism features to reach
+leave-one-repetition-out balanced accuracy at least 0.75 and exceed the best
+external prefix outcome by at least 0.15 at two adjacent predeclared
+checkpoints from 25% onward.  Without label balance the pilot can adjudicate
+mechanism evidence only.
+
+No H20 run is launched if the local analyzer/tests, manifest preflight, GPU
+probe, command dry-run, projected cost, or cleanup plan fails.

docs/intervention-response-v3-results-20260714.md

@@ -0,0 +1,160 @@
+# Phase-aware telemetry intervention-response v3 results
+
+Date: 2026-07-14 (Asia/Singapore).
+
+Decision: **`STOP_NO_INCREMENTAL_TUNING_SIGNAL`**.
+
+## Claim tested
+
+After the replay has developed queue, batch, and KV state, does increasing MNS
+from 16 to 64 create telemetry responses that exceed workload-repeat noise, and
+does any such response identify whether the action repairs the full-run SLO
+better than external prefix outcomes alone?
+
+The first clause passed.  The second clause failed.  Long-window telemetry is
+mechanistically informative, but this pilot does not support its necessity for
+tuning this action.
+
+## Setup
+
+- dash0 GPU 0-3: four NVIDIA H20 GPUs; Qwen3-30B-A3B; patched vLLM
+  `0.24.1.dev3+opprof`; TP=4.
+- Action: MNS `16 -> 64` with exact request, arrival, and input-length hashes.
+- Trace: `chat_w20260312_1000`, replay-time scale 0.5, 300 seconds.
+- Loads: 1.5 and 2.125 requests/s/GPU; three disjoint bands; endpoint order
+  A/B, B/A, A/B; load order low/mid, mid/low, low/mid.
+- Five cumulative checkpoints: 30, 75, 150, 225, and 300 seconds; four
+  non-overlapping quarter blocks.
+- Six fresh-server sessions, 12 measured runs, six action pairs, and eight
+  same-config repeat pairs.
+
+The prior three-load attempt is not part of the result.  Its MNS=16 workload at
+3.125 requests/s/GPU could not drain by the 450-second client timeout and
+produced no high-load result.  V3 reran every retained point from scratch.
+
+## End-to-end outcome
+
+All three low-load pairs remained feasible (`true->true`, label 0).  All three
+pressure-load pairs changed from infeasible to feasible (`false->true`, label
+1).  MNS=16 pressure pass rates were 0.5604, 0.3145, and 0.2635; all three
+MNS=64 pressure runs reached 1.0.  This yielded a balanced 3/3 action label set.
+
+## Mechanism result
+
+No telemetry feature passed the action-versus-repeat gate at 10% or 25%.  At
+50%, graph padding first passed.  At both 75% and 100%, graph padding and queue
+waiting passed, satisfying the requirement for two features at the same pair
+of adjacent checkpoints and with consistent directions in both load regimes.
+
+| Feature | Direction for MNS 16->64 | 75% effect/repeat median | 75% above repeat p95 | 100% effect/repeat median | 100% above repeat p95 |
+|---|---:|---:|---:|---:|---:|
+| `queue_waiting_mean` | lower | 1346.22 | 3/6 | 898.70 | 3/6 |
+| `graph_padding_fraction` | higher | 5.00 | 4/6 | 5.74 | 5/6 |
+
+The very large queue effect/median-repeat ratios should not be read alone: its
+repeat p95 was much larger than its repeat median, so the independent p95
+coverage criterion remained binding.  Full-window queue-waiting deltas were
+-0.19 to -0.32 at low load and -20.99 to -32.16 at pressure load.  Graph
+padding increased in every pair, by 0.00133-0.00217 at low load and
+0.00705-0.00873 at pressure load.
+
+The mechanism is therefore a real tradeoff: larger MNS reduces queueing,
+especially under pressure, while increasing CUDA-graph padding.
+
+## Tuning-signal result
+
+Leave-one-repetition-out balanced accuracy was evaluated against the best
+external prefix-outcome feature at every predeclared checkpoint.
+
+| Replay phase | Best external BA | Best telemetry BA | Incremental telemetry gate |
+|---:|---:|---:|---|
+| 10% | 0.833 | 0.833 | fail |
+| 25% | 1.000 | 1.000 | fail |
+| 50% | 0.833 | 1.000 | pass at this checkpoint only |
+| 75% | 1.000 | 1.000 | fail |
+| 100% | 1.000 | 1.000 | fail |
+
+At 50%, several telemetry features exceeded the external baseline by at least
+0.15, including the phase-stable mechanism feature
+`graph_padding_fraction`.  The advantage did not hold at either adjacent
+checkpoint.  Consequently no feature passed the frozen two-adjacent-phase
+requirement.
+
+The important ordering is that external TTFT already classified the action
+perfectly at 25%, whereas the robust two-feature mechanism response did not
+emerge until 75%-100%.  In this setup telemetry explains *why* MNS helps, but it
+does not provide earlier or more reliable action selection than direct prefix
+outcomes.
+
+## Research conclusion
+
+The 5/10-second negative result was indeed too narrow.  It only ruled out an
+ultra-early telemetry verifier; it did not rule out engine-state information.
+The 300-second pilot finds a clear and reproducible queueing-versus-padding
+response.
+
+However, this does not rescue the direct telemetry-guided tuning claim.  For
+this action and workload, the external signal is already as good or better
+before the telemetry mechanism becomes stable.  The project should therefore
+not claim that engine instrumentation is necessary for tuning on this evidence,
+and should not open an E2E policy test from this pilot.
+
+The narrower simulator-residual route remains logically open: telemetry may
+explain why a simulator misranks real configurations even when direct online
+outcomes can guide a tuner.  That is a different hypothesis and was not tested
+here.
+
+## Change and verification
+
+Change: absolute 5/10-second prefixes were replaced by phase-aware 30/75/150/
+225/300-second analysis; SLO early stop was disabled; full Layer-1 coverage,
+hash, request-accounting, controller, and stream/footer gates were added.  The
+mechanism gate was corrected to require two features at the same adjacent
+phase pair.
+
+Expected effect: distinguish “telemetry has not developed yet” from “telemetry
+does not identify or improve the action decision.”
+
+Verification: five local analysis/controller test suites passed; remote
+manifest preflight and command dry-run passed; six serialized sessions passed
+all stream invariants; the analyzer was rerun and produced byte-identical
+output.
+
+Result: mechanism evidence passed; incremental tuning evidence failed.  Audit
+SHA256: `45f6f248712f9cbd3ed72036837ff6dc5b5c14c0f2eb6ba5cd5daceb1aa4ddb7`.
+
+Remaining risk: this is a development pilot with one model, one TP, one action,
+two retained loads, three request bands, and six action labels.  It is adequate
+to reject opening the next direct-policy stage, not to establish a universal
+negative claim about telemetry.
+
+## Research-validity audit
+
+| Check | Verdict | Evidence / boundary |
+|---|---|---|
+| Real system and E2E outcome | PASS | Real H20/vLLM replay; full SLO outcome accompanies mechanism telemetry. |
+| Matched action baseline | PASS | Exact request/arrival/length hashes for MNS 16 and 64; external prefix outcome is the decision baseline. |
+| Repeats and order effects | PASS for pilot | Three disjoint bands; A/B, B/A, A/B endpoint order; counter-rotated load order. |
+| Selective load removal | PASS with narrowed claim | The 3.125 load produced no result before any action comparison; the failure and cost are retained, and all kept points were freshly rerun. |
+| Significance/generalization | NEEDS EVIDENCE for a paper claim | Only three bands, one model, one TP, one action, and six labels.  This is explicitly a stage gate. |
+| Calibration versus evaluation | NEEDS EVIDENCE for a positive policy claim | Frozen gates and leave-one-band-out folds reduce leakage, but a new workload/model hold-out is still required. |
+| Platform/reproducibility | PASS | Commit, commands, manifest, controller state, platform fingerprint, raw remote paths, and audit hashes are recorded. |
+
+## Data sanity
+
+- Measured runs: n=12; elapsed 300.346-317.012 seconds; 12 distinct; pass rate
+  0.2635-1.0 with 4 distinct values; selected requests 1800-2550 with 2
+  distinct values.
+- Sessions: n=6; 0.8413-0.8631 H20-hours; 6 distinct; Layer-1 records
+  58,465-64,776; 6 distinct.  V3 cost was 5.0924 H20-hours; total including
+  the invalid attempt was 6.4505, below the 8.0 cap.
+- Labels: n=6; min/max 0/1; 2 distinct.  Action pairs were 6 and repeat pairs
+  were 8 at every checkpoint.
+- Coverage-gap observations: n=60; start gaps 0.0427-0.1247 seconds; end gaps
+  0.00014-0.1705; maximum internal gaps 0.1695-0.6528, all below one second.
+- Checked invariants: exact pair hashes and counts, all runs uncensored, full
+  request accounting, monotonic admitted/completed coverage, monotonic Layer-1
+  timestamps, nonnegative counters, bounded ratios, non-identical per-config
+  states, contiguous step indices, zero drops, footer/sidecar agreement, no
+  controller failures, all sessions complete, GPU idle after completion.  No
+  red flags were found.

docs/intervention-response-v3-two-load-protocol-20260714.md

@@ -0,0 +1,73 @@
+# Phase-aware telemetry intervention-response v3 protocol
+
+Status: **FROZEN AFTER A NON-COMPARATIVE OPERATIONAL FAILURE AND BEFORE V3 RUNS**.
+
+Date: 2026-07-14 (Asia/Singapore).
+
+## Why v2 was invalid
+
+The first v2 session completed the 300-second arrival windows at 1.5 and 2.125
+requests/s/GPU.  At 3.125 requests/s/GPU, MNS=16 could not drain the admitted
+requests before the 450-second client timeout.  The session produced no result
+and no MNS=64 action endpoint was run.  V2 is therefore an invalid operational
+attempt, not evidence for or against the telemetry hypothesis.
+
+This failure was observed before any MNS action comparison.  V3 excludes only
+the unmeasurable overload point and reruns every retained point on fresh
+servers; it does not reuse the completed v2 low/mid results.
+
+## Question and hypothesis
+
+Question: after enough replay time for queue, batch, and KV state to develop,
+does an MNS intervention create telemetry responses that exceed workload-repeat
+noise, and does any such response predict whether the intervention repairs the
+full-run SLO outcome better than external prefix outcomes alone?
+
+Hypothesis: increasing MNS from 16 to 64 has little value at the 1.5
+requests/s/GPU control load but can repair the 2.125 requests/s/GPU pressure
+load.  Queue, running-set, batch, or KV telemetry should expose the difference
+at stable replay phases.  Label balance is an assumption to test, not a fact.
+
+## Frozen setup
+
+- Solo placement on dash0 GPU 0-3: 4 NVIDIA H20 GPUs, Qwen3-30B-A3B, patched
+  vLLM `0.24.1.dev3+opprof`, fixed TP=4.
+- Action: MNS `16 -> 64`; all other engine and workload parameters fixed.
+- Workload: `chat_w20260312_1000`, replay-time scale 0.5, hence 300 seconds.
+- Loads per GPU: 1.5 control and 2.125 pressure requests/s.  The failed 3.125
+  overload point is excluded from V3 and retained only as a failure artifact.
+- Three disjoint request bands.  Each MNS action pair has exact request,
+  arrival, and input-length hashes.  Endpoint order is A/B, B/A, A/B; load
+  order is low/mid, mid/low, low/mid.
+- Every session starts a fresh server, then runs the accepted 16-request long
+  warm-up and bounded burn-in before measured runs.
+- SLO-unrecoverable early stop is disabled.  Measured results must cover the
+  full 300-second arrival window and must not be early-stopped.
+- Cumulative checkpoints are 10%, 25%, 50%, 75%, and 100%; non-overlapping
+  quarter blocks are also reported.
+- A Layer-1 interval is complete only if timestamps are monotonic and its
+  start, end, and maximum internal record gaps are each at most one second.
+- Incremental V3 cap is the unused portion of the original 8 H20-hour cap.
+  The exact prior cost and V3 cap are machine-recorded in the manifest.
+
+## Frozen gates
+
+Data validity requires six uncensored sessions, six action pairs, eight
+same-config repeat pairs, exact action-pair hashes, full request accounting,
+zero Layer-1 drops, continuous coverage, all stream/footer invariants, no
+co-resident compute process, idle GPUs before and after sessions, nonnegative
+counters, bounded ratios, non-identical per-config state, and monotonic request
+coverage across checkpoints.  Any red flag stops analysis.
+
+Mechanism evidence requires at least two telemetry features to exceed the
+unchanged v1 repeat-noise thresholds at the same pair of adjacent checkpoints.
+Those features must have a direction consistent in both retained load regimes.
+
+Decision evidence additionally requires at least two positive and two negative
+full-run action-efficacy labels, valid leave-one-repetition-out folds, and at
+least one phase-stable mechanism feature whose balanced accuracy is at least
+0.75 and at least 0.15 above the best external prefix-outcome feature at two
+adjacent predeclared checkpoints from 25% onward.
+
+V3 remains a development mechanism pilot.  Even `OPEN_E2E_POLICY_TEST` opens a
+held-out tuning-policy experiment; it is not itself a paper performance claim.

docs/opprof-simfid-overview-20260713.md

@@ -0,0 +1,131 @@
+# SimFid + OpProf Campaign Overview
+
+Date: 2026-07-13. Status: both campaigns **CLOSED**. This is the entry-point
+summary; every claim below links to a frozen protocol and a results document.
+Decision history lives in `docs/opprof_campaign_state.md` (this repo) and
+`replayserve/docs/simfid_campaign_state.md`.
+
+## Why these campaigns exist
+
+The paper's motivation requires three evidence-backed claims:
+
+1. **Simulators cannot replace real replay** for config tuning → SimFid.
+2. **The leverage point is engine-knob configuration, not operator
+   implementation**, at the measured regimes → OpProf P3–P5.
+3. **Tuning is genuinely hard**: the surface is workload-conditioned
+   (sign-flips defeat defaults), real evaluation is expensive (measured GPU
+   cost), and the surface churns across engine versions → OpProf P4/P6.
+
+All three now rest on pre-registered protocols with frozen decision rules,
+Holm-corrected contrasts, and no imputation over censored data.
+
+## Campaign 1 — SimFid (replayserve repo)
+
+**Question.** Can a Frontier-class simulator replace real replay for AITuner
+config tuning? **Verdict: NOT ADEQUATE** under the pre-declared decision rule.
+
+- S2-E (3-config TP family, same model/HW): ranking perfect after throughput
+  calibration, but latency is uncalibrated (TTFT p95 sim/real 0.30–0.38, TPOT
+  0.63–0.79) → false-feasible counterexample across a 50 ms TPOT SLO.
+- S2-R-b (12-cell TP×MNS surface, exact C1 workload replayed in Frontier,
+  184 runs, 0 failures): the decision-bearing frozen-calibrated
+  throughput-proxy reading picks TP1/MNS64 → **30.46% real top-1 regret**;
+  trap detection 3/6; LOAO 0/92. All three adequacy components FAIL.
+- Post-hoc diagnostic (NOT decision-bearing): an SLO-gated reading achieves
+  0–0.76% regret, tau-b 0.967, trap 6/6 — anchor-level SLO errors partially
+  cancel at per-cell peaks. This motivates hybrid sim-prune + real-final
+  designs but must always carry the post-hoc label.
+
+Sources: `replayserve/docs/simfid_s2e_report.md`,
+`replayserve/docs/simfid_s2rb_results.md`.
+
+## Campaign 2 — OpProf (this repo, `docs/opprof/`)
+
+**Question.** Do operators face materially different patterns online than in
+offline rectangular benchmarks ("workload-conditioned operator profiling"),
+and if so, where is the recoverable gain? Patched vLLM 0.24.0
+(`patches/vllm-0.24.0-opprof/`), Qwen3-30B-A3B BF16, dash0 H20.
+
+| Phase | Deliverable | Verdict / headline |
+|---|---|---|
+| P0–P2 | Dual-layer instrumentation (always-on Layer-1 telemetry + sampled Layer-2 Kineto) | Overhead **−0.04%** (CI [−0.17, +0.05]) after the compile-factor fix; Layer-2 perturbs 51.3% → sampled-only by design |
+| P3 | 10-pattern × config matrix (`phase3-{protocol,results}.md`) | **H1b PASS** (5/6 evaluable contrasts, Holm p≈0): irregular patterns carry R64 raggedness +23.0 to +44.8 pp over rectangular controls with 8.3–44.7% useful-token efficiency loss. **H1a INCONCLUSIVE** (Layer-2 window representativeness) |
+| P4 | Ranked optimization plan (`phase4-optimization-plan.md`) | #1 prefix-affine routing: **+82.14%** saturation req/s (P08 vs matched P07, 62.3% fewer prefill tokens). #4 MNS is pattern-conditioned: MNS64 gives +3.4/+3.7% on P06/P10 but **−24.27% on P01** — the sign flips by workload |
+| P5 | Mechanism decomposition (`phase5-{protocol,results}.md`) | All four intuitive mechanisms ≈0 at rho=0.60: raggedness 3.8% n.s., capture-size fix +1.6% n.s. (padding 13.74%→2.40% with no E2E gain), prefix ~0. **Real finding:** P3's P10-vs-P04 gap was largely an arrival-uniformization artifact — replaying recorded (bursty) arrival recovers +12.9%; burstiness helps batch formation at low rate |
+| P6 | Cross-version churn, paired 12-cell surface, vLLM 0.20→0.24 (`phase6-{protocol,results}.md`) | Old #2 config TP2/MNS64 **−29.41%** (solo-confirmed, bounded both sides); TP1 plateau **+13.5%** at MNS8; old argmax TP2/MNS32 held (−0.76%). Formal ARGMAX/RANKING/TRAP **INCONCLUSIVE** — 4 cells right-censored/non-monotonic, no imputation |
+
+**P6 mechanism note.** TP2/MNS64's old anchor now fails with decode-batch
+means 11.5–18.1 and 4.6–9.5% of steps executing outside cudagraph coverage;
+all 37 solo primaries had zero preemptions.
+
+**Combined churn claim (paper-usable).** Across one ~2-month engine upgrade
+the surface below the argmax reorganized (rank-2 config −29%, plateau +13%)
+even though the argmax survived → warm-start/transfer from stale tuning
+surfaces is unreliable; every engine upgrade is a retuning trigger.
+
+## Cross-cutting methodological findings (reusable beyond this paper)
+
+1. **Co-location validity is metric-dependent.** 21 exact same-request pairs
+   (co-located vs solo, P6): throughput and operator shares move <3%, but SLO
+   pass rates flip by up to **+92.86 pp** (0.071→1.000) at frontier anchors —
+   feasible/infeasible verdicts invert. Deltas are cell- and anchor-dependent
+   (0 to +92.9 pp), so no fixed correction exists. Rule adopted: SLO-frontier
+   measurements must be solo; mean-type metrics may be co-located behind the
+   pre-registered A-P3-1 validity gate (which rejected 8-way, passed 4-way).
+2. **Uniformizing arrival distorts efficiency** (−12.9% at low rate) in the
+   opposite direction from intuition — offline benchmarks that regularize
+   arrival misestimate real efficiency.
+3. **Env vars hashed into vLLM compile factors** silently cause cold
+   torch.compile caches and ~4% slower artifacts (root cause of our phantom
+   overhead; fixed with a one-line ignore-list entry). Upstream-report
+   candidate.
+4. **Long-context real traces break short-load-calibrated harness
+   assumptions** (drain deadlines, warm-up stabilization); P10/TP2 never
+   stabilizes (36.8% drift).
+
+## Corrections and honest limits — do NOT quote these stale readings
+
+- P3's "P10 is 14.3% worse than P04" is **superseded by P5**: largely a
+  materialization artifact of uniformized arrival. Quote pattern-vs-control
+  raggedness/padding effects (H1b) and the P5-corrected arrival result
+  instead. The remaining P10-vs-P03 gap (~36%) is workload physics, not
+  recoverable waste at this regime.
+- H1a (operator bottleneck-ranking inversions) is **not refuted** — it is
+  inconclusive at the measured regimes; saturation-regime decomposition
+  remains open.
+- P6 formal verdicts are INCONCLUSIVE by right-censoring, not by data
+  invalidity; the −29.41%/+13.5%/−0.76% numbers are bounded and quotable.
+- Only solo-tier SLO numbers are quotable; co-located W1–W3 artifacts are
+  preserved but superseded.
+- The <3% co-location bound is an empirical gate verified at moderate load on
+  specific patterns, not a theorem; re-run the gate before reusing 4-way
+  placement in new regimes.
+- SimFid's SLO-gated 0–0.76% regret reading is post-hoc, not decision-bearing.
+
+## Artifact map
+
+| Path | Content |
+|---|---|
+| `docs/opprof_campaign_state.md` | Full OpProf decision ledger (echoes, amendments, acceptances) |
+| `docs/opprof/phase{0,2}-*.md`, `docs/opprof/patch-design.md` | Recon, patch design, smoke + overhead evidence |
+| `docs/opprof/phase{3,5,6}-protocol.md` | Frozen pre-registered protocols incl. amendments |
+| `docs/opprof/phase{3,5,6}-results.md`, `phase4-optimization-plan.md` | Results; phase6 metrics pinned by SHA-256 `290ba7fc…` |
+| `patches/vllm-0.24.0-opprof/` | 7-patch series + apply.sh + tests (base `ee0da84a`) |
+| `runs/opprof-phase{3,5,6}/` | Decision-bearing evidence (metrics/manifests/validation); raw Layer-1 JSONL streams are git-ignored (507 MB, kept on disk and dash0) |
+| `docs/simulator-fidelity-frontier-20260711.md` | Standalone review: does the data show simulator mis-ranking rigorously? |
+| `replayserve/docs/simfid_*` | SimFid protocols, results, ledger |
+
+## GPU accounting
+
+OpProf total ≈ **22.2 H20-hours** (P0–P5 ≈ 16.5, P6 5.64 of a 6.0 cap).
+SimFid accounting lives in the replayserve ledger. No prompt or generated
+text appears in any committed artifact; the prompt-bearing trace copy stays
+in git-ignored `trace_windows/`.
+
+## Open items (not committed to)
+
+- Bound the 3 right-censored TP4 cells + TP2/MNS16 (~1.6 H20-h, exceeds the
+  P6 cap) to convert ARGMAX/TRAP into formal verdicts.
+- Saturation-regime mechanism decomposition (P5 analogue at high rho).
+- Upstream reports: compile-factor env poisoning; co-location SLO validity.
+- Synthesis of both campaigns into the paper's motivation section.

docs/opprof/oracle-gap-protocol.md

@@ -0,0 +1,240 @@
+# Static-policy oracle-gap protocol
+
+Status: **FROZEN WITH A-OG-1 THROUGH A-OG-4 AMENDMENTS**.
+
+Date frozen: 2026-07-13 (Asia/Singapore). Existing Phase-3 measurements were
+inspected only to choose the workload pair and rate brackets. They are
+exploratory calibration data, not primary observations in this protocol.
+
+### A-OG-4 — close a majority-shifted boundary (before confirmation scores)
+
+After all four primary frontiers and 70 scores were complete, the controller
+was interrupted before the first confirmation produced a score. The partial
+`P01-C10-r26-rep1` attempt contained no result, sanity file, or score and is
+archived separately. This amendment is therefore blind to confirmation
+outcomes.
+
+The original confirmation schedule still runs first. Afterward, each cell's
+majority-vote frontier is recomputed. If either side of the *actual* final
+boundary has fewer than three trials because the provisional boundary moved,
+the controller runs only enough repetitions at that existing rate to reach
+three, high-to-low, on a fresh server for that config. It then recomputes the
+boundary and repeats for at most three closure rounds. No new rate anchor may
+be added. Failure to obtain a monotone, bracketed boundary with three trials
+per side within three rounds or the 6 H20-hour cap makes the experiment
+inconclusive.
+
+This fills a stopping-rule omission: it does not change any existing score,
+majority rule, phase, config, rate grid, SLO, or oracle calculation.
+
+### A-OG-3 — freeze a per-logical-trial transport retry rule (after 62 scores)
+
+The first attempt at `P06-C10-r1.9-rep0` reproduced the same isolated local
+transport signature seen in A-OG-2: one clean request raised
+`ClientOSError` 3.56 ms after admission, with HTTP status 0, no first token,
+and no output. The other 343 requests produced exactly 512 tokens, the server
+stayed healthy, drain took 4.00 s, and every other client invariant passed.
+The controller again rejected the attempt before creating a `score.json`.
+
+For the remainder of this experiment, an attempt may be quarantined and the
+identical logical trial retried once on a fresh server only if all of the
+following hold: the sole failed client invariant is `clean_failures_zero`;
+exactly one clean request failed; its error is `ClientOSError`, HTTP status is
+0, it produced neither a first token nor output, and it completed within 10
+ms of admission; every successful request has exact output; and the server
+has no crash or error. The attempt is never scored. A second invalid attempt
+for the same logical key, more than one failed request, or any other error
+signature is a stop condition and makes the experiment inconclusive.
+
+This rule is independent of config, rate, and observed performance, and
+supersedes the one-key wording in A-OG-2 without changing how that retry was
+executed. Existing scores remain immutable. No client, grid, policy,
+threshold, order, SLO, or oracle calculation changes.
+
+### A-OG-2 — retry one transport-invalid attempt (after 49 scored trials)
+
+The first attempt at `P06-C01-r2.2-rep0` produced one local
+`ClientOSError` 3.27 ms after admission, with HTTP status 0, no first token,
+and no output. The other 399 requests completed successfully, every successful
+request produced exactly 512 tokens, the server stayed healthy, and all other
+client invariants passed. The controller rejected the attempt before creating
+a `score.json`; no TTFT/TPOT result from this attempt was used to choose the
+recovery rule.
+
+This is a measurement-transport failure, not an observed server SLO outcome.
+The entire attempt is content-hashed and moved under `invalid-attempts/`, then
+the identical logical trial (phase, config, rate, repetition, derived manifest,
+seeds, timeline, and SLO) is run once on a fresh C01 server. All 49 existing
+scores remain immutable. A second client transport failure in the retry is a
+stop condition and makes the experiment inconclusive; it must not be retried
+again. No grid, policy, threshold, order, or oracle calculation changes.
+
+### A-OG-1 — extend the P06 upper bracket (after trial 33)
+
+The controller stopped as registered after C00/P06 remained SLO-feasible at
+every original and upward-extension anchor through 2.3 requests/s. At that
+point 33 trials and 1.519475 H20-hours were complete, all GPU memory had been
+released, and no C00/P06 infeasible upper bound existed. No oracle inference
+was performed.
+
+This amendment changes only the P06 upward-extension list from
+`2.1,2.2,2.3` to `2.1,2.2,2.3,2.4,2.5,2.6,2.8,3.0`. The controller stops at
+the first bracket exactly as before. Existing trials are immutable and are
+reused; config order, P01 rates, SLO, timelines, repetitions, placement,
+metrics, and decision threshold do not change. The resumable controller may
+accept the new code/protocol fingerprint only when all immutable runtime,
+client, model, manifest, config, and base-grid fields match the pre-amendment
+state, and it records both fingerprints under `A-OG-1`.
+
+## Question and decision gate
+
+The candidate motivation is:
+
+> A single global static batching policy leaves at least 10% end-to-end
+> SLO-goodput on the table when serving temporally heterogeneous phases; a
+> phase-aware runtime policy can recover that gap without changing hardware,
+> model, precision, or tensor-parallel topology.
+
+This experiment tests a necessary condition in the existing TP1 policy space
+`{C00,C10,C01,C11}`. The optimistic oracle knows the phase and switches with
+zero delay, zero state-transfer cost, and no prediction error. If even this
+oracle cannot beat the best one-config-for-all-phases policy by 10%, an online
+controller over these MNS/MBT choices cannot do so either.
+
+The primary gate uses a conservative capacity bracket:
+
+- `L[p,c]`: highest offered rate accepted as SLO-feasible for phase `p` and
+  config `c`;
+- `U[p,c]`: lowest higher offered rate accepted as SLO-infeasible;
+- oracle upper bound at phase-time weights `w`:
+  `sum_p w[p] * max_c U[p,c]`;
+- best-static lower bound:
+  `max_c sum_p w[p] * L[p,c]`.
+
+We scan every P01/P06 time mixture, including pure endpoints. The current
+motivation is **REFUTED** if the maximum conservative ratio
+`oracle_upper / static_lower - 1` is below 10%. It is **NOT ESTABLISHED** if the
+bound crosses 10% but the observed point estimate does not. A positive result
+requires a point-estimate gap of at least 10% and then a separately
+pre-registered interleaved-trace validation; this frontier experiment alone
+cannot establish a positive E2E contribution.
+
+The conclusion is scoped to the measured MNS/MBT policy family and the chosen
+strongest-conflict phase pair. It does not rule out new scheduling mechanisms,
+KV-state policies, topology changes, or other workload phases.
+
+## Fixed system boundary
+
+| Item | Frozen value |
+|---|---|
+| Host | `dash0`, one run at a time on physical GPU0 |
+| GPU | NVIDIA H20; no other GPU process anywhere on the host |
+| Model | `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`, BF16 |
+| Runtime | `/tmp/wjh-opprof-phase2-dash0-20260711/.venv`, vLLM `0.24.1.dev3+g668cfb7e2` |
+| vLLM source | `/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0` |
+| Topology | TP1, one server, no data/pipeline parallelism |
+| Fixed mechanisms | chunked prefill on; prefix caching on |
+| Client | Phase-5 timestamp/fixed-rate wrapper over the Phase-3 exact-token client |
+| Seeds | workload `20260712`; trial token-domain seed derived only from phase/rate/repetition, never config |
+
+SLO co-location results in Phase 6 showed pass-rate flips despite small
+throughput deltas. Therefore unused H20s remain idle: parallel placement is not
+authoritative for this experiment.
+
+## Workloads and policies
+
+The pair is chosen before new measurements because Phase 3 showed the strongest
+opposing static preference:
+
+- **P01:** input `U[128,512]`, output exactly 64 tokens, deterministic steady
+  arrivals. C10 lost 24.27% saturation throughput relative to C00.
+- **P06:** 50/50 input mixture `U[128,512]`/`U[4096,8192]`, output exactly 512
+  tokens, deterministic bursts of eight. C10 gained 3.37% over C00.
+
+Both reuse the immutable 32,768-row Phase-3 manifests. For every trial a
+derived manifest preserves request order, lengths, outputs, and arrival class,
+but applies a trial-specific token-seed offset. The same derived manifest is
+used for all four configs. This prevents prefix-cache carry-over when a hot
+server executes several anchors without changing the logical workload.
+
+| Config | Effective MNS | Effective MBT | Extra flags |
+|---|---:|---:|---|
+| C00 | 1024 | 8192 | none |
+| C10 | 64 | 8192 | `--max-num-seqs 64` |
+| C01 | 1024 | 2048 | `--max-num-batched-tokens 2048` |
+| C11 | 64 | 2048 | both flags |
+
+Startup logs must confirm these values. A default drift is a stop condition.
+
+## Load grid, order, and repetitions
+
+Primary grids:
+
+- P01: `{26,28,30,32,34,36}` requests/s; execution order
+  `32,26,36,28,34,30`.
+- P06: `{1.4,1.5,1.6,1.7,1.8,1.9,2.0}` requests/s; execution order
+  `1.7,1.4,2.0,1.5,1.9,1.6,1.8`.
+
+Every primary anchor runs once. For each phase/config, the highest primary
+feasible anchor and its next higher primary anchor are then run two more times,
+giving three trials at both sides of the boundary. If all primary anchors are
+feasible, extend upward in the fixed order P01 `38,40,42` or P06
+`2.1,2.2,2.3,2.4,2.5,2.6,2.8,3.0`.
+If all are infeasible, extend downward in the fixed order P01 `24,22,20` or P06
+`1.3,1.2,1.1`. Stop extending at the first bracket.
+
+One primary server is launched per config in order `C11,C00,C01,C10`.
+Confirmation servers are fresh and launch in reverse order
+`C10,C01,C00,C11`; their boundary anchors run high-to-low. This balances
+machine-time drift and makes confirmation independent of the primary server's
+cache/compiler state.
+
+Timelines:
+
+- P01: 60 s warm-up + 60 s clean measurement; drain cap 120 s.
+- P06: 60 s warm-up + 120 s clean measurement; drain cap 240 s.
+- no Kineto profiling; exact greedy output with `ignore_eos`; maximum client
+  concurrency 256.
+
+A trial is SLO-feasible when at least 95% of requests admitted during the clean
+interval eventually finish successfully and individually satisfy both:
+
+- TTFT <= 2 s for input <= 4,096 tokens; <= 4 s for input <= 32,768; <= 6 s
+  otherwise;
+- TPOT <= 50 ms, computed as `(completion - first_token)/(output_tokens - 1)`.
+
+SLO-goodput is the number of those passing clean-admission requests divided by
+clean seconds. Client schedule lag must stay <=1 s and achieved clean offered
+rate must be within 5% of target. Failure of either condition makes the anchor
+infeasible; its admitted-only latency is not used to rescue it.
+
+At a repeated boundary, feasibility is the majority of three trial verdicts.
+All accepted anchor verdicts must be monotone in offered rate. A persistent
+non-monotone result after the registered repeats is a red flag and stops the
+oracle-gap inference.
+
+## Validity and stopping rules
+
+Before every server launch record host, GPU, driver, clocks, runtime package
+versions, git/source hashes, manifest hashes, exact commands, and process
+contamination. Stop on another GPU process, request/output mismatch, manifest
+drift, server crash, non-finite latency, ratio outside `[0,1]`, negative
+counter, or discontinuous/non-monotone accepted frontier.
+
+The controller is detached and resumable. It kills only process groups it
+created, checks zero GPU memory after every server, never overwrites a complete
+trial, and writes state atomically. The hard budget is 6 H20-hours; expected
+cost is 3.0--4.0 H20-hours and approximately the same wall time because runs
+are serialized.
+
+## Required report
+
+The report includes every trial's target/achieved rate, clean cohort size,
+pass count/rate, SLO-goodput, TTFT/TPOT percentiles, schedule lag, failure
+reasons, accepted frontier brackets, per-phase oracle choices, best static
+choice, equal-time gap, worst-mixture conservative gap, and GPU-hours.
+
+The final statistics section ends with a data-sanity block containing `n`,
+min/max, distinct-value counts, and checks for non-negative counters, ratios in
+`[0,1]`, non-identical per-config results, exact output work, monotone
+frontiers, and continuous rate brackets.

docs/opprof/oracle-gap-results.md

@@ -0,0 +1,199 @@
+# Static-policy oracle-gap results
+
+Status: **FINAL — REFUTED WITHIN THE FROZEN TP1 MNS/MBT POLICY SPACE**.
+
+Date: 2026-07-13. The registered experiment completed 104 valid trials on
+`dash0` and returned `REFUTED`. Even a phase-perfect oracle with zero detection,
+switching, and state-transfer cost is bounded below the registered 10%
+SLO-goodput contribution gate.
+
+The machine result is
+`runs/opprof-oracle-gap/metrics.json` (SHA-256
+`250ba4c1657a8830795ee06392eea4e21c62d958fea11701ba60581ef0266543`).
+All 104 trial-level measurements are in
+`runs/opprof-oracle-gap/trials.csv` (SHA-256
+`60cd901f18bbf107eb130f0095e867c3b6d47a78a42e83bcbe57fecf23cc5f9c`).
+The final resumable controller state is
+`runs/opprof-oracle-gap/controller-state.json` (SHA-256
+`27ec5e9a3cd32a871d583ba8eb6d7d3fe2a338a8fd42369233ada57cd4da6436`).
+
+## Decision
+
+The tested motivation was:
+
+> One static batching policy leaves at least 10% end-to-end SLO-goodput on the
+> table across temporally heterogeneous phases, and a phase-aware runtime can
+> recover it by switching MNS/MBT policies.
+
+The registered 10,001-point mixture scan finds a worst conservative gap of
+**8.333219%**, at P01 time weight `0.0244`. Equal phase time gives
+**7.260726%**. An independent implementation that evaluates every exact
+static-policy crossover gives a slightly more conservative exact maximum of
+**8.333333% = 1/12**, at P01 weight `1/41`. This leaves 1.666667 percentage
+points below the 10% gate.
+
+The negative result is stronger than a failed online prototype. The oracle
+already knows the current phase and pays no switching cost. A realizable
+controller over the same four policies cannot exceed this oracle bound.
+
+This is a scoped refutation, not a universal claim about adaptive serving. It
+rules out the current contribution based on phase-aware selection among the
+four MNS/MBT configurations for the frozen P01/P06 pair, Qwen3-30B-A3B, TP1,
+H20, vLLM 0.24, and the registered SLO. It does **not** rule out a new scheduler,
+KV/cache policy, routing-aware mechanism, topology change, or a different
+workload family.
+
+## Fixed setup
+
+| Item | Value |
+|---|---|
+| Placement | `dash0`, serialized on physical GPU0; GPUs 1-7 idle |
+| GPU | NVIDIA H20, driver 580.95.05 |
+| Model | Qwen3-30B-A3B, BF16 |
+| Runtime | vLLM `0.24.1.dev3+g668cfb7e2`, source `4b253fd8619764b6971a7f2e3a3aa7545f6ace05` |
+| Topology | TP1; one server and one client |
+| Fixed mechanisms | chunked prefill on; prefix caching on |
+| P01 | input `U[128,512]`, exactly 64 output tokens, deterministic steady arrivals |
+| P06 | 50/50 input `U[128,512]` / `U[4096,8192]`, exactly 512 output tokens, deterministic bursts of eight |
+| P01 timeline | 60 s warm-up + 60 s clean measurement |
+| P06 timeline | 60 s warm-up + 120 s clean measurement |
+| SLO feasibility | at least 95% of clean-admitted requests pass both TTFT and TPOT |
+| TTFT SLO | <=2 s for input <=4096; <=4 s for input <=32768; otherwise <=6 s |
+| TPOT SLO | <=50 ms/token |
+
+The four policies were:
+
+| Config | MNS | MBT |
+|---|---:|---:|
+| C00 | 1024 | 8192 |
+| C10 | 64 | 8192 |
+| C01 | 1024 | 2048 |
+| C11 | 64 | 2048 |
+
+## Final SLO frontiers
+
+`L` is the highest majority-feasible offered rate. `U` is the next higher
+majority-infeasible rate. Both sides below have at least three trials, and all
+accepted rate sequences are monotone.
+
+| Config | P01 `L` | P01 `U` | P06 `L` | P06 `U` |
+|---|---:|---:|---:|---:|
+| C00 | 28.0 | 30.0 | 2.3 | 2.4 |
+| C10 | 24.0 | 26.0 | 2.4 | 2.5 |
+| C01 | 28.0 | 30.0 | 2.2 | 2.3 |
+| C11 | 24.0 | 26.0 | 2.2 | 2.3 |
+
+The optimistic oracle upper bound chooses C00/C01 at `U=30` for P01 and C10
+at `U=2.5` for P06. The static lower bound chooses C10 below P01 weight `1/41`
+and C00 above it; they tie at the exact worst point.
+
+| Mixture | Oracle upper | Best-static lower | Conservative gap |
+|---|---:|---:|---:|
+| Pure P06 | 2.500000 | 2.400000 | 4.166667% |
+| Exact worst, P01 weight `1/41` | 3.170732 | 2.926829 | **8.333333%** |
+| Equal phase time | 16.250000 | 15.150000 | **7.260726%** |
+| Pure P01 | 30.000000 | 28.000000 | 7.142857% |
+
+The specialization exists but is too small. Reducing MNS from 1024 to 64
+raises the conservative P06 lower bound from 2.3 to 2.4 req/s, while lowering
+P01 from 28 to 24 req/s. Even after using infeasible `U` values for the oracle
+and feasible `L` values for the static baseline, the best possible phase-aware
+selection cannot reach 10%.
+
+## Robustness finding
+
+The strongest system finding is not config specialization but a repeat-level
+mode flip at P01/26 rps. Both C10 and C11 show the identical verdict sequence
+`rep0=0% pass`, `rep1=100% pass`, `rep2=0% pass`. For C10, TTFT p50 is
+3864.62, 252.71, and 3994.88 ms; for C11 it is 6533.62, 593.44, and
+5812.71 ms. TPOT remains below the 50 ms SLO in these trials.
+
+These trials are measurement-valid: exact outputs, offered-rate tolerance,
+schedule lag, timestamps, and client invariants all pass. The majority rule
+therefore classifies 26 rps as infeasible for both configs. The result does not
+change the oracle-gap decision because neither config supplies the P01 oracle
+maximum or the relevant best-static P01 lower bound.
+
+Token-domain seed and server execution history change together across
+repetitions, so this experiment cannot attribute the flip to MoE routing,
+cache/compiler state, or another source. It does motivate a narrower factorial
+study that crosses token seed with fresh/reused server state and randomizes
+order, while recording routed-expert and per-step scheduler telemetry. That is
+a mechanism question; it should not be presented as evidence for a phase-aware
+MNS/MBT controller.
+
+## Execution and audit history
+
+The 104 scored trials comprise 52 base-grid primaries, 18 registered upward or
+downward extensions, 32 boundary confirmations, and two boundary-closure
+trials. The final closure moved C00/P06 from the provisional `[2.4,2.5)` to
+`[2.3,2.4)` and repeated 2.3 rps to 3/3 feasible.
+
+Four frozen amendments are recorded in the protocol:
+
+- A-OG-1 extended only the P06 upward anchors after C00 remained feasible
+  through the original 2.3-rps limit.
+- A-OG-2 quarantined one P06/C01/2.2 local `ClientOSError` attempt and retried
+  the identical logical trial on a fresh server.
+- A-OG-3 generalized the same pre-score transport rule after one
+  P06/C10/1.9 attempt showed the identical signature.
+- A-OG-4, frozen before any confirmation score, added closure for a
+  majority-shifted boundary without adding new anchors.
+
+The two transport-invalid attempts and one pre-A-OG-4 interrupted attempt
+created no score and do not enter any metric. Their retained tree hashes are,
+respectively,
+`434863ba90513cbc54534ffbc1a13c980b3ef7d567190a0aa3f97b55650acbb2`,
+`a57b1ac5f090680bb70c16b5d709eb2b8ac47dce57c7a03b8077cd9b6d80d831`,
+and `bc3feb53514601e641ef1db204c74cffe3d282b24ad797e5b161468f5d15de5c`.
+
+Ten deliberately overloaded primary anchors exceed both the 1 s schedule-lag
+gate and the 5% offered-rate tolerance. They are correctly classified as
+infeasible and are not used as final boundary points. All 48 final boundary
+trials pass both client-side gates; their maximum schedule lag is 573.59 ms.
+
+The final experiment fingerprint uses repository commit `16177b0`, analyzer
+SHA-256
+`d86ecb1f077472906cbb729bd2c9d4b3a82ac6dfdc90838a17ae300d0767110d`,
+controller SHA-256
+`f7c5c2f74f2002f1e4b097e608d165a3a2e9374fbf07935a4d6d6a7c5d45a83a`,
+and protocol SHA-256
+`173f969a4428643cf6c4b950413aa82bef25cafd163e20d7944370a5d87af435`.
+The archived launch log SHA-256 is
+`54e2f4804a5efee072b670043c4092cf41f1a27664faa05ea71bfc1412c3e9db`.
+
+## GPU accounting and cleanup
+
+The campaign used **4.8877033845 H20-hours**, below the 6.0-hour cap, across
+5 h 41 min 51 s wall time including amendment, audit, restart, and server
+startup intervals. At completion, all eight H20s report 0 MiB, 0% utilization,
+and zero compute processes.
+
+## Sanity block
+
+There are no data-sanity red flags. The P01/26 all-or-none mode flip is a
+scientific robustness finding, not an invalid trial signature; it repeats in
+two configs and all measurement invariants pass.
+
+| Numeric family | n | Min | Max | Distinct | Checked invariant/result |
+|---|---:|---:|---:|---:|---|
+| Score-row indicator | 104 | 1 | 1 | 1 expected | 104 score files; no overwrite |
+| Target rate (req/s) | 104 | 1.4 | 36.0 | 19 | Non-negative; fixed grid/extensions only |
+| Clean cohort per trial | 104 | 168 | 2160 | 30 | Non-empty and non-negative |
+| Pass rate | 104 | 0.0 | 1.0 | 57 | All ratios in `[0,1]` |
+| SLO-goodput (req/s) | 104 | 0.0 | 27.816667 | 59 | Non-negative; per-cell results not all identical |
+| Boundary pass rate | 48 | 0.0 | 1.0 | 33 | Both final sides have three trials |
+| Boundary max schedule lag (ms) | 48 | 2.145861 | 573.593768 | 48 | All below the 1000 ms gate |
+| Exact-output indicator | 85,402 | 1 | 1 | 1 expected | Every clean request produced the requested token count |
+| Frontier cells | 8 | 8 | 8 | 1 expected | 8/8 bracketed and monotone |
+| Registered weight scan | 10,001 | 0.0 | 1.0 | 10,001 | Continuous 0.0001 increments, endpoints included |
+| Campaign H20-hours | 1 | 4.887703 | 4.887703 | 1 | Non-negative and below 6.0 |
+| Final GPU memory/utilization | 8 | 0 MiB / 0% | 0 MiB / 0% | 1 expected | Zero compute processes |
+
+Checked invariants: fixed model, runtime, manifests, SLOs, config values, seeds,
+and serialized placement; exact output work; nondecreasing timestamps;
+non-negative counters and latencies; pass ratios in range; offered-rate and
+schedule gates; majority-of-three final boundaries; monotone accepted
+frontiers; complete 10,001-point scan; independent exact-crossover
+recomputation; transport-attempt quarantine; GPU hard-cap compliance; and
+complete GPU cleanup.

docs/opprof/patch-design.md

@@ -0,0 +1,434 @@
+# OpProf dual-layer instrumentation patch design
+
+Status: **design for review; no vLLM patch has been implemented**.
+
+Target source: vLLM `v0.24.0`, commit `ee0da84ab9e04ac7610e28580af62c365e898389`.
+Target campaign: Qwen3-30B-A3B serving on NVIDIA H20 (SM90), using the V1 engine.
+All source paths and line numbers below are relative to the pinned clone.
+
+## Approved dispositions (2026-07-11)
+
+- Use JSONL with `msgspec`, the proposed context/chunk histogram edges, and an
+  8192-record bounded queue.
+- Keep exact expert loads Layer-2-only.
+- Use the community BF16 Qwen3-30B-A3B checkpoint. TP1 is primary, with TP2 and
+  TP4 counterpoints; record the selected MoE backend logs in every run.
+- Use two profiler warm-up iterations followed by eight active iterations.
+- Apply the 3% Layer-1 overhead gate to the **upper bound of the 95% confidence
+  interval** for every primary serving metric.
+- Reject `--disable-log-stats` when OpProf is enabled; retain the five-file
+  fail-fast design.
+
+## Goal and success criteria
+
+The patch should make every serving iteration conditionable on its request and
+execution pattern while keeping the always-on path below a 3% serving overhead
+budget. Heavy kernel tracing and exact MoE routes are sampled separately.
+
+The proposed split is:
+
+1. **Layer 1:** one compact composition record per scheduler step, emitted by
+   the scheduler process rather than every tensor-parallel worker.
+2. **Layer 2:** short, sampled windows using vLLM's existing torch-profiler
+   configuration and `/start_profile` and `/stop_profile` endpoints.
+
+The design deliberately does not add hooks to GPU kernels, attention layers,
+the Qwen model, or the OpenAI serving API. Those surfaces are not needed to
+answer the Phase 0 profiling questions.
+
+## Assumptions and tradeoffs
+
+- The campaign uses the V1 engine and the default vLLM scheduler. Both the
+  synchronous and `AsyncScheduler` paths inherit the base scheduler hooks.
+- The H20 backend statement assumes the usual unquantized BF16/FP16
+  Qwen3-30B-A3B checkpoint, `moe_backend=auto`, and no LoRA. Quantization,
+  batched expert format, or an explicit backend can change kernel selection.
+- Layer 1 requires normal stats collection, which is enabled by default. If
+  OpProf is enabled together with `--disable-log-stats`, initialization should
+  fail with an actionable error. Supporting that unusual combination would
+  require an extra stats path and a larger patch.
+- The context-length histogram records the sequence length at the end of the
+  scheduled model input (`num_computed_tokens` before scheduling plus tokens
+  scheduled in this step). This matches the worker-side sequence-length
+  construction at `vllm/v1/worker/gpu_model_runner.py:2010-2019` and avoids
+  retaining raw request lengths.
+- “Decode tokens” means tokens scheduled for requests classified by vLLM as
+  generation-phase requests, including scheduled speculative tokens. It is not
+  the number of subsequently accepted output tokens. The existing classifier
+  and its chunked-prefill semantics are at `vllm/v1/utils.py:780-813`.
+
+## Existing facilities to reuse
+
+These are zero-patch wins:
+
+| Need | Existing v0.24.0 facility | Design consequence |
+|---|---|---|
+| Scheduled request and token map | `SchedulerOutput.num_scheduled_tokens`, total tokens, preempted IDs, and new/cached request data at `vllm/v1/core/sched/output.py:180-219` | Derive composition in the scheduler; do not add fields to `SchedulerOutput`. |
+| Prefill/decode classification | `compute_iteration_details()` at `vllm/v1/utils.py:780-813` | Reuse the exact definition already used by the iteration log. |
+| Queue/KV/prefix stats | `SchedulerStats` at `vllm/v1/metrics/stats.py:170-198` and construction at `vllm/v1/core/sched/scheduler.py:2228-2264` | Reuse field semantics, but snapshot at schedule time to avoid async misattribution. |
+| Per-step CUDA-graph descriptor | `CUDAGraphStat` at `vllm/compilation/cuda_graph.py:32-37`, populated at `vllm/v1/worker/gpu_model_runner.py:3919-3934` | Reuse the object; only broaden the condition under which it is returned. |
+| Sampled CPU/CUDA tracing | profiler config and schedule at `vllm/config/profiler.py:33-105`, endpoints at `vllm/entrypoints/serve/profile/api_router.py:21-45` | Layer 2 needs configuration and orchestration, not a new profiler implementation or endpoint. |
+| Exact routed expert IDs | routed-expert capture callback and buffers at `vllm/model_executor/layers/fused_moe/routed_experts_capturer.py:58-84` and `vllm/v1/worker/gpu_model_runner.py:7382-7437` | Use only in a separate sampled Layer-2 run; it is too expensive for Layer 1. |
+
+## Layer 1: always-on composition records
+
+### Data flow and hook placement
+
+```text
+Scheduler.schedule()
+  -> snapshot request composition, queues, KV and prefix deltas
+  -> SchedulerOutput -> TP workers -> ModelRunnerOutput.cudagraph_stats
+  -> Scheduler.update_from_output()
+  -> finalize the same step record -> bounded queue -> background JSONL writer
+```
+
+The scheduler is the ownership boundary for Layer 1. It sees the full logical
+batch once, so recording in each TP worker would duplicate data and require an
+aggregation protocol.
+
+Exact proposed hooks against the pinned clone:
+
+1. **Initialize one recorder in the base scheduler constructor.** The scheduler
+   already owns stats state and passes the normal `log_stats` setting into the
+   KV-cache manager (`vllm/v1/core/sched/scheduler.py:78-87` and
+   `vllm/v1/core/sched/scheduler.py:250-260`). If `VLLM_OPPROF_DIR` is set,
+   construct the recorder and reject `log_stats=False`.
+2. **Begin a record inside `Scheduler.schedule()`.** Read prefix counters at
+   method entry, then finish the snapshot immediately after `SchedulerOutput`
+   and connector metadata are constructed and before
+   `_update_after_schedule()` mutates request state
+   (`vllm/v1/core/sched/scheduler.py:1012-1100`). Prefix lookup counters are
+   updated synchronously during this schedule call
+   (`vllm/v1/core/kv_cache_manager.py:202-242` and
+   `vllm/v1/core/sched/scheduler.py:675-712,897-915`), so before/after deltas
+   remain attributable even when several batches are in flight.
+3. **Finalize in `Scheduler.update_from_output()`.** Pair the returned
+   `ModelRunnerOutput` with the exact `SchedulerOutput` at
+   `vllm/v1/core/sched/scheduler.py:1464-1477`, and enqueue after normal output
+   processing near `vllm/v1/core/sched/scheduler.py:1791-1803`. Store pending
+   drafts by `id(scheduler_output)` and remove them on finalize; the identifier
+   never leaves the process. This is robust to the batch queue, which retains
+   and later returns the matching output object
+   (`vllm/v1/engine/core.py:519-632`).
+4. **Return the existing CUDA-graph stat when OpProf is enabled.** Change the
+   condition at `vllm/v1/worker/gpu_model_runner.py:3919-3926` from only
+   `observability_config.cudagraph_metrics` to that flag **or**
+   `VLLM_OPPROF_DIR`. The file already imports `vllm.envs` at line 23. No CUDA
+   synchronization or tensor transfer is introduced.
+5. **Close through the existing scheduler shutdown.** Drain and join the writer
+   before the scheduler reports shutdown complete at
+   `vllm/v1/core/sched/scheduler.py:2285-2295`; retain an `atexit` fallback for
+   abnormal embedding/test lifecycles.
+
+The scheduler assigns a monotonically increasing local `step_index` at begin.
+Every real scheduler call is represented, including zero-token steps. A DP-only
+dummy execution has no `SchedulerOutput` and is outside this composition
+stream; if DP is later in campaign scope, add a distinct `dummy_step` marker
+rather than pretending it has a request composition.
+
+### Record schema
+
+Use schema-versioned JSON Lines. An illustrative record is:
+
+```json
+{"schema":1,"engine_id":"dp0-pid1234","step_index":42,"submit_wall_ns":1783761000000000000,"submit_mono_ns":8920000000,"complete_mono_ns":8922371000,"model_executed":true,"scheduled_requests":37,"decode_batch_size":32,"prefill_requests":5,"prefill_tokens":896,"decode_tokens":32,"chunked_prefill":{"first":2,"middle":1,"final":1,"unsplit":1,"tokens":896,"chunk_size_hist":[0,0,1,1,1,1,1,0,0]},"context_length_hist":[0,0,3,6,12,10,5,1,0,0,0,0],"preemptions":0,"queues":{"running":37,"waiting":9,"deferred":0},"kv":{"total_blocks":120000,"free_blocks":42000,"used_blocks":78000,"usage":0.65},"prefix":{"local":{"requests":2,"queries":4096,"hits":3072,"preempted_requests":0,"preempted_queries":0,"preempted_hits":0},"external":null},"cudagraph":{"hit":true,"runtime_mode":"PIECEWISE","unpadded_tokens":928,"bucket_tokens":1024,"padding_tokens":96},"moe_expert_load":null,"dropped_records_before":0}
+```
+
+Required field semantics:
+
+- `submit_wall_ns` permits joining to external workload logs;
+  `submit_mono_ns` and `complete_mono_ns` provide stable within-process
+  ordering and elapsed time without wall-clock jumps.
+- `scheduled_requests` is the number of entries in
+  `num_scheduled_tokens`. `decode_batch_size`, `prefill_requests`,
+  `prefill_tokens`, and `decode_tokens` reuse vLLM's classifier.
+- `chunked_prefill` contains aggregate split information, never request IDs.
+  A context request is `first` when the scheduled range ends before its prompt
+  ends, `middle` when it was already a prefill chunk and remains incomplete,
+  `final` when a prior chunk completes, and `unsplit` when its prefill completes
+  in one step. `chunk_size_hist` uses token-count buckets
+  `(16, 32, 64, 128, 256, 512, 1024, 2048, +inf)`.
+- `context_length_hist` includes every scheduled request, with fixed upper
+  edges `(128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536,
+  131072, +inf)`. The final array therefore has 12 bins. The example above is
+  illustrative and will be checked against the final chosen edges in tests.
+- `preemptions` is the size of the step's `preempted_req_ids`, created at
+  `vllm/v1/core/sched/scheduler.py:1059-1076`. It is not an interval-derived
+  Prometheus value.
+- Queue lengths and KV blocks are captured immediately after scheduling. KV
+  usage follows the existing null-block-adjusted calculation at
+  `vllm/v1/core/block_pool.py:692-711`. `deferred` means skipped/deferred
+  waiting requests, matching `num_skipped_waiting_reqs` in
+  `vllm/v1/metrics/stats.py:174-183`.
+- Local and external prefix fields are per-schedule deltas copied from the
+  existing mutable counters. The existing stats drain resets those objects at
+  `vllm/v1/core/kv_cache_manager.py:190-200` and
+  `vllm/v1/core/sched/scheduler.py:2235-2242`; OpProf must read, not drain or
+  replace, them. These retain vLLM's lookup semantics: a local lookup can be
+  counted before allocation later rejects that waiting request, so prefix
+  `requests` is not required to be less than or equal to scheduled prefill
+  requests.
+- `cudagraph.hit` is the operational serving definition `runtime_mode != NONE`.
+  `FULL` selects the full graph path and `PIECEWISE` selects graph-wrapped
+  compiled regions; the latter must not be interpreted as full-step graph
+  coverage. Startup normally captures all dispatcher descriptors at
+  `vllm/v1/worker/gpu_model_runner.py:6595-6643`, but the stat itself has no
+  capture-versus-replay bit, so a rare lazy first capture cannot be
+  distinguished from replay. `bucket_tokens` is `num_padded_tokens` in the
+  existing stat.
+- `moe_expert_load` is explicitly null in Layer 1. This avoids accidentally
+  presenting unavailable data as zero.
+
+### Encoding, buffering, and flush
+
+JSONL is recommended over a compact binary ring buffer for Phase 1 because the
+schema will evolve during campaign bring-up, records need to be directly
+inspectable beside Kineto traces, and `msgspec` is already a vLLM dependency
+(`requirements/common.txt:35`).
+Encoding one small dictionary with a reused `msgspec.json.Encoder` avoids the
+standard `json` module's larger CPU cost. Once the schema stabilizes, the same
+record can be moved to a binary format only if Phase 2 measurements show JSONL
+is the bottleneck.
+
+The foreground path encodes once and performs non-blocking `put_nowait()` into
+a bounded queue of 8192 records. A single daemon writer thread drains into one
+file per EngineCore/DP rank and process. It flushes userspace buffers every
+1 MiB or one second, whichever comes first, and on clean process exit; it never
+calls `fsync()` per record. When the queue is full, the serving thread drops the
+new record, increments a counter, and the next successful record reports the
+gap in `dropped_records_before`. Shutdown emits a footer with encoded, written,
+and dropped counts. The file name includes schema, DP rank, PID, and start time,
+but not TP rank because there is one scheduler record stream.
+
+The sole on/off switch is:
+
+```text
+VLLM_OPPROF_DIR=/absolute/output/directory
+```
+
+Unset or empty means a true no-op: no recorder, queue, thread, record
+construction, or graph-stat broadening. Directory validation and a clear
+startup log happen before serving. Runtime toggling is intentionally omitted;
+it adds synchronization and ambiguous partial files without helping the
+campaign.
+
+### Expected cost and Phase 2 overhead gate
+
+These are estimates, not measurements:
+
+- composition and two fixed histograms: about 20-80 microseconds per step,
+  linear in scheduled requests;
+- `msgspec` encoding plus a non-blocking queue insertion: about 5-20
+  microseconds per step;
+- CUDA-graph stat construction: below 2 microseconds and no GPU sync;
+- disk I/O: off the serving critical path, subject to bounded-queue drops.
+
+The expected foreground total is roughly 25-100 microseconds per step. At a
+2 ms decode step, the high end would exceed 3%, so the budget is a measurement
+gate, not a claim.
+
+Phase 2 should use the same Qwen3 checkpoint, request trace, random seed,
+hardware, vLLM commit, TP/EP topology, CUDA-graph configuration, and cache
+state for off/on comparisons. Alternate off/on ordering, warm up before each
+measurement, and run at least five paired repeats. Report throughput and
+p50/p95 TTFT and TPOT, plus CPU utilization, log bytes per step, queue high
+watermark, and dropped-record count. The acceptance rule is less than 3%
+regression for every declared primary serving metric, with bootstrap confidence
+intervals reported. Whether the point estimate or upper 95% bound is the hard
+gate is an open decision.
+
+Correctness checks for every run are: contiguous step indices except explicitly
+reported drops, scheduled token sums matching the prefill/decode split,
+histogram counts matching scheduled request/chunk counts, KV ratio in `[0,1]`,
+non-negative counters, and identical generated outputs for the deterministic
+test trace.
+
+## Layer 2: sampled kernel windows
+
+### Trigger and window
+
+Use the existing profiler API with a run-specific output directory and
+`ignore_frontend=true`. The worker start/stop RPC already fans out to all
+workers (`vllm/v1/executor/abstract.py:256-257` and
+`vllm/v1/executor/multiproc_executor.py:340-402`), and each GPU worker creates a
+CPU+CUDA profiler with a rank-qualified trace name
+(`vllm/v1/worker/gpu_worker.py:929-980`). No Layer-2 vLLM code patch is needed.
+
+Recommended initial configuration:
+
+```text
+profiler=torch
+torch_profiler_dir=<run>/kineto
+ignore_frontend=true
+delay_iterations=0
+max_iterations=8
+wait_iterations=0
+warmup_iterations=2
+active_iterations=8
+torch_profiler_record_shapes=false
+torch_profiler_with_memory=false
+torch_profiler_with_stack=false
+torch_profiler_with_flops=false
+torch_profiler_use_gzip=true
+torch_profiler_dump_cuda_time_total=true
+```
+
+The primary trigger should be step-count based: an external campaign controller
+POSTs `/start_profile` immediately before a desired composition regime. The
+built-in schedule records two warm-up plus eight active model iterations and
+`max_iterations=8` then stops the underlying profiler on the following worker
+step. The “exceeds max” control semantics are explicit at
+`vllm/profiler/wrapper.py:83-114` and in
+`tests/v1/worker/test_gpu_profiler.py:77-98`; the controller should still POST
+`/stop_profile` afterward to clear the active control state.
+Worker iteration boundaries already call `profiler.step()` and annotate context
+and generation token counts at `vllm/v1/worker/gpu_worker.py:803-827`. On-demand
+manual POST remains useful for debugging. A time-based trigger is a fallback
+for live traffic but is less reproducible because it yields a variable number
+of steps. Phase 1 should confirm that the trace contains exactly eight active
+iteration annotations; profiler scheduling and trace callbacks live at
+`vllm/profiler/wrapper.py:159-226,290-307`.
+
+The profile directory should be unique per campaign server run. The HTTP start
+endpoint does not accept a `profile_prefix`, and a worker that is restarted for
+another window retains the trace name chosen on first initialization
+(`vllm/v1/worker/gpu_worker.py:939-975`). Multiple windows in one server run
+therefore share a directory and need a controller manifest containing start/
+stop wall times and produced filenames. Use a new directory only when the
+server is restarted; adding a new endpoint parameter is not justified for
+Phase 1. Every TP worker emits its own trace, and the rank suffix contains DP,
+PP, TP, DCP, EP, and global rank information
+(`vllm/distributed/utils.py:664-695`). Keep these traces separate and aggregate
+only offline.
+
+### CUDA graphs and Kineto interpretation
+
+Keep CUDA graphs enabled in the primary sampled windows so kernel time reflects
+the serving configuration. The pinned code profiles CUDA activity
+(`vllm/v1/worker/gpu_worker.py:953-960`) while a captured path invokes
+`CUDAGraph.replay()` instead of rerunning the Python callable
+(`vllm/compilation/cuda_graph.py:233-360`). Therefore Kineto/CUPTI can observe
+CUDA activity launched during replay, but the Python/PyTorch operator scopes
+that created the graph are not re-executed and cannot be assumed to retain
+per-layer attribution. vLLM explicitly documents the related limitation that
+layerwise NVTX tracing does not work with CUDA graphs
+(`vllm/config/observability.py:60-63`).
+
+The clone does not contain a stronger guarantee about whether a particular
+PyTorch/CUPTI build will present every graph node as an individually named
+kernel versus a coarser graph-launch view. Treat that as an empirical Phase 2
+check. Run one matched eager (`cudagraph_mode=NONE`) taxonomy window to identify
+kernel families, but do not use its timings as the production baseline.
+
+### Calibration and MoE routing
+
+Join each profiler iteration annotation to Layer-1 records by rank-independent
+step order, the profiler window marker, timestamps, and the prefill/decode token
+counts. Aggregate kernels into attention, router/top-k, expert GEMMs, dense
+GEMMs, normalization/activation, collectives, and sampling. For TP, report both
+sum-of-rank GPU work and the maximum per-rank critical-path time. Fit or tabulate
+kernel-time attribution conditioned on composition, context histogram, graph
+runtime mode, and capture bucket, then validate on held-out windows rather than
+the same samples used for calibration.
+
+For the Phase 2 “operator time approximately equals iteration time” gate, do
+not naively add overlapping kernels. Compute the union of CUDA kernel intervals
+per rank, use the maximum rank as the distributed GPU critical path, and retain
+an explicit CPU/queue/unattributed residual against Layer 1's submit-to-complete
+span. Operator-category interval unions plus the residual must reconstruct the
+span; summed GPU work is reported separately and may legitimately exceed wall
+time.
+
+Exact per-layer expert loads are Layer-2-only. A separate server start with
+`--enable-return-routed-experts` exposes per-token, per-layer top-k IDs through
+the existing router callback (`vllm/model_executor/layers/fused_moe/router/base_router.py:233-278`).
+Aggregate those arrays offline into per-layer expert counts, entropy, max/mean
+load, coefficient of variation, and top-k imbalance. Do not enable it in the
+normal Layer-1 or baseline Kineto run: its worker transit buffer costs a few MB,
+and the scheduler-side slot buffer can reach multiple GB with a CPU fancy-index
+copy per step (`vllm/model_executor/layers/fused_moe/routed_experts_capturer.py:223-305`).
+
+## Patch surface estimate
+
+Proposed implementation and tests:
+
+| File | Approximate changed/new lines | Purpose |
+|---|---:|---|
+| `vllm/envs.py` | 4 | Declare and parse `VLLM_OPPROF_DIR`. |
+| `vllm/v1/opprof.py` (new) | 220 | Schema, schedule snapshot/deltas, histograms, pending-step pairing, encoder, bounded writer, drop/footer handling. |
+| `vllm/v1/core/sched/scheduler.py` | 32 | Initialize recorder; begin before state mutation; finalize with the matching model output. |
+| `vllm/v1/worker/gpu_model_runner.py` | 4 | Return existing `CUDAGraphStat` when either built-in metrics or OpProf needs it. |
+| `tests/v1/core/test_opprof.py` (new) | 190 | Schema/invariants, chunk classification, async pairing, disabled no-op, bounded-queue drops, shutdown flush. |
+| **Total** | **5 files / about 450 lines** | About 260 production lines and 190 test lines. |
+
+No patch is proposed for `ProfilerConfig`, profile endpoints, profiler wrapper,
+`SchedulerOutput`, `SchedulerStats`, Prometheus/logging, CUDA kernels, fused MoE
+kernels, the Qwen model, or frontend APIs. If support for
+`--disable-log-stats` is required, add about 12 lines in
+`vllm/v1/core/kv_cache_manager.py`, making the estimate six files and about 462
+lines; the smaller fail-fast design is recommended for this campaign.
+
+## Risks and mitigations
+
+- **Foreground CPU overhead:** request scanning and JSON encoding are the likely
+  hot spots. Use fixed-size arrays, a reused encoder, no raw lists, no blocking
+  I/O, and enforce the Phase 2 gate.
+- **Async-scheduling races:** queue/KV/prefix state observed during
+  `update_from_output()` may include later in-flight schedules. Snapshot it
+  inside the synchronous `schedule()` call; only the immutable returned
+  CUDA-graph stat is added at completion. Pair by the exact `SchedulerOutput`
+  object and assert one begin/one finalize.
+- **TP and EP aggregation:** Layer 1 is scheduler-owned and emitted once. Layer
+  2 remains per rank. Interpret TP critical path as the slowest rank and, for an
+  expert-parallel routed-expert sample, aggregate expert ownership offline.
+- **Log volume:** at 500 steps/s, a 1.0 KiB record is about 44 GB/day per
+  EngineCore. Rotate by campaign run, compress completed files, and use the
+  bounded queue/drop counters. Production-long captures need a later sampling
+  or rotation policy; Phase 1 files should be bounded by run duration.
+- **Graph semantics:** `PIECEWISE` is partial graph coverage, not a binary full
+  hit. Preserve `runtime_mode`, unpadded tokens, bucket, and padding rather than
+  reducing the record to one boolean.
+- **Profiler perturbation:** Kineto has medium-to-high overhead and trace flushes
+  can stall. Use short windows, unique directories, disabled stacks/shapes and
+  memory by default, and never use Layer-2 latency as an unprofiled performance
+  result.
+- **Backend drift:** the unquantized SM90 oracle prioritizes Triton, but
+  shape-specific fallbacks remain possible. Record startup backend logs and the
+  full checkpoint/parallel/quantization configuration with every run.
+
+## Open decisions for review
+
+1. Approve JSONL with `msgspec`, the two proposed histogram edge sets, and a
+   bounded 8192-record writer queue.
+2. Approve exact expert-load collection as Layer-2-only, rather than adding a
+   new always-on GPU histogram kernel.
+3. Confirm the Qwen3 checkpoint precision/quantization and intended TP/EP/DP
+   topology; the Triton backend conclusion depends on these inputs.
+4. Choose the first profiler window: recommended two warm-up plus eight active
+   iterations, or a longer active window.
+5. Decide whether the 3% gate applies to metric point estimates or to the upper
+   bound of their 95% confidence intervals.
+6. Confirm that campaign runs may reject `--disable-log-stats`; supporting it
+   adds one file and about 12 lines.
+
+## Data sanity block
+
+- **Patch estimate:** n=5 files; per-file line-estimate min=4, max=220,
+  distinct values=4; total about 450 lines, of which about 260 are production
+  and 190 are tests.
+- **Histogram definitions:** n=2 arrays; bin-count min=9, max=12,
+  distinct=2. In the illustrative record, context-bin sum=37 equals scheduled
+  requests, chunk-bin sum=5 equals prefill requests, KV used=120000-42000, and
+  graph padding=1024-928.
+- **Estimated foreground components:** n=3 timed components; min is below 2
+  microseconds for graph-stat construction, max is 80 microseconds for
+  composition; distinct ranges=3. Their conservative combined foreground
+  estimate is 25-100 microseconds per step.
+- **Invariants checked:** counters and histogram bins are non-negative; KV
+  usage and prefix hit ratios are bounded in `[0,1]`; graph bucket is at least
+  unpadded tokens; `NONE` is the only graph miss mode; no raw request IDs or
+  context-length lists are serialized; one scheduler stream avoids TP
+  duplicates.
+- **Measurement status:** n=0 benchmark runs; min/max and cross-configuration
+  distinctness are not applicable. The cost figures are estimates and cannot
+  support an overhead conclusion until Phase 2.

docs/opprof/phase0-recon-vllm-0.24.0.md

@@ -0,0 +1,446 @@
+# Phase 0 recon: vLLM 0.24.0 observability
+
+Status: source recon and patch design complete; **no instrumentation patch was
+implemented**.
+
+Campaign target: operator-level, pattern-conditioned profiling of
+Qwen3-30B-A3B serving on NVIDIA H20. This report is based on the pinned source
+clone only. No GPU was used, no package or model was installed, and no dash
+host was contacted.
+
+Unless explicitly prefixed with `v0.20.0:`, every source path and line number
+below is relative to `/home/gahow/phd/vllm-v0.24.0` at the pinned commit.
+
+## Outcome
+
+vLLM 0.24.0 already contains most of the raw ingredients, but they are split
+across scheduler internals, aggregate metrics, worker traces, and an optional
+routed-expert return path. The smallest useful patch is therefore not a new
+profiler. It is one scheduler-owned per-step record that joins existing batch
+composition to the existing `CUDAGraphStat`; short torch-profiler windows and
+exact expert routes remain sampled Layer 2 facilities.
+
+The proposed patch is detailed in
+[`patch-design.md`](patch-design.md). Estimated surface: five files and about
+450 lines including tests, with about 260 production lines.
+
+## 1. Source pin and compatibility record
+
+The requested tag existed and the exact requested clone command completed
+successfully:
+
+```text
+git clone --depth 1 --branch v0.24.0 https://github.com/vllm-project/vllm /home/gahow/phd/vllm-v0.24.0
+```
+
+| Field | Pinned value |
+|---|---|
+| Repository | `https://github.com/vllm-project/vllm` |
+| Local clone | `/home/gahow/phd/vllm-v0.24.0` |
+| Tag | `v0.24.0` (`git describe --tags --exact-match`) |
+| Commit | `ee0da84ab9e04ac7610e28580af62c365e898389` |
+| Clone completion | `2026-07-11T08:26:16Z` (`2026-07-11 16:26:16 +08:00`) |
+| Checkout | Detached HEAD, clean |
+| Top-level entries | 45, including hidden entries and `.git` |
+| Listing SHA-256 | `75222e5d3bacdd043e421c417496aaf0fa5a9429b7244698b6d3ca2218b85b58` |
+
+The listing digest is reproducible from entry names only, not file contents:
+
+```text
+LC_ALL=C find . -mindepth 1 -maxdepth 1 -printf '%f\n' |
+  LC_ALL=C sort | sha256sum
+```
+
+This definition includes `.git`, sorts bytewise under the C locale, and hashes
+the 45 LF-terminated root names. It is recorded to make the otherwise ambiguous
+phrase “sha256 of the top-level directory listing” precise.
+
+### Torch and CUDA requirements
+
+- Build and NVIDIA runtime requirements pin `torch==2.11.0` in
+  `pyproject.toml:1-13`, `requirements/build/cuda.txt:1-10`, and
+  `requirements/cuda.txt:1-10`; CMake repeats 2.11.0 as the supported CUDA and
+  ROCm torch version at `CMakeLists.txt:62-72`.
+- The NVIDIA requirements also pin `torchvision==0.26.0`,
+  `flashinfer-python==0.6.12`, `flashinfer-cubin==0.6.12`, and CUDA-13 extras
+  for CUTLASS DSL and Humming kernels (`requirements/cuda.txt:6-29`). Setup
+  strips/substitutes those extras for CUDA 12 builds
+  (`setup.py:1066-1083`).
+- The source default is CUDA 13.0 (`vllm/envs.py:85-87,555-563`) and the Docker
+  default is CUDA 13.0.2 (`docker/Dockerfile:14-42`). Wheel detection maps CUDA
+  major 12 to `cu129` and major 13 to `cu130` (`setup.py:528-568`).
+- The pinned installation document still describes the default precompiled
+  binaries as CUDA 12.9 and lists CUDA 12.8 and 13.0 variants
+  (`docs/getting_started/installation/gpu.cuda.inc.md:4-19,24-54`). This is a
+  packaging/default distinction, not evidence that 13.0 is unsupported.
+- NVIDIA compute capability 7.5+ is documented, and SM90 is in every relevant
+  CMake CUDA architecture set (`docs/getting_started/installation/gpu.cuda.inc.md:9-19`;
+  `CMakeLists.txt:105-118`). Optional FA3 and DeepGEMM builds need CUDA 12.3+,
+  FlashMLA needs 12.9+, and the Hopper CUTLASS MoE build needs 12.3+
+  (`setup.py:1111-1139`; `CMakeLists.txt:844-868`).
+
+Implication for dash0 later: H20/SM90 is a supported architecture, but the
+actual driver and installed CUDA were intentionally not queried in Phase 0.
+Phase 2 must choose a torch 2.11.0-compatible `cu129` or `cu130` artifact after
+checking the host driver. A wheel built against a different torch/CUDA build is
+not interchangeable; the source documentation explicitly warns about binary
+incompatibility (`docs/getting_started/installation/gpu.cuda.inc.md:14-19`).
+
+## 2. Built-in observability inventory
+
+### 2.1 Torch profiler integration
+
+**Configuration and control.** v0.24.0 does not contain the legacy
+`VLLM_TORCH_PROFILER_DIR` symbol: a repository-wide exact-name search at the
+pinned commit returned zero matches. Profiling is now configured through
+`ProfilerConfig`: `profiler` is `torch` or `cuda`, and
+`torch_profiler_dir` holds the absolute/URI output location
+(`vllm/config/profiler.py:33-46,125-145`). Optional controls include stacks,
+FLOPs, gzip, CUDA-time tables, shapes, memory, frontend exclusion, delayed
+start, maximum iterations, and wait/warmup/active iteration scheduling
+(`vllm/config/profiler.py:48-105`).
+
+The documented server form is a `--profiler-config` JSON object, for example
+`{"profiler":"torch","torch_profiler_dir":"/abs/run/kineto"}`, followed by
+POSTs to the two endpoints (`docs/contributing/profiling.md:46-85`).
+
+When profiler config is present, the serving router exposes POST
+`/start_profile` and `/stop_profile`; without profiler config the router is not
+attached (`vllm/entrypoints/serve/profile/api_router.py:21-45`). The endpoints
+are therefore not an unauthenticated always-on feature independent of server
+configuration; they exist within the configured serving application and should
+be protected like the rest of the control plane.
+
+**Captured activity.** Each GPU worker creates a torch profiler with both CPU
+and CUDA activities, using the configured shapes/memory/stack/FLOPs switches
+(`vllm/v1/worker/gpu_worker.py:929-980` and
+`vllm/profiler/wrapper.py:159-226`). TensorBoard-compatible traces are written
+with an optional gzip handler. The wrapper also writes CPU/CUDA aggregate
+tables; only rank 0 prints the table to stdout
+(`vllm/profiler/wrapper.py:251-287`). The source documentation calls the
+profiler medium-overhead and warns that trace size and final flushing can be
+large (`docs/contributing/profiling.md:1-38`).
+
+The V1 worker calls `profiler.step()` once per execution and wraps model
+execution in a record-function name containing context/generation request and
+token counts (`vllm/v1/worker/gpu_worker.py:803-827,895-898`). If frontend
+profiling is not ignored, `AsyncLLM` also creates a separate CPU-only trace
+(`vllm/v1/engine/async_llm.py:178-200`).
+
+**TP behavior.** `EngineCore.profile()` delegates to the executor
+(`vllm/v1/engine/core.py:662-663`), whose profile RPC is a collective worker
+call (`vllm/v1/executor/abstract.py:256-257`). Multiprocessing broadcasts calls
+that do not declare a unique output rank, so the profile call reaches every TP
+worker (`vllm/v1/executor/multiproc_executor.py:340-402`). Each worker emits its
+own rank-qualified trace; the suffix includes DP/PP/TP/DCP/EP/global rank
+components (`vllm/distributed/utils.py:664-695`). For TP, kernel analysis must
+therefore preserve per-rank files, use the slowest rank for critical-path time,
+and optionally sum ranks to quantify total GPU work.
+
+### 2.2 Iteration-level scheduler data and exported metrics
+
+The V1 scheduler interface states that each `schedule()` output corresponds to
+one model forward and maps request IDs to scheduled tokens
+(`vllm/v1/core/sched/interface.py:51-80`). `SchedulerOutput` already carries
+new/cached request data, per-request and total scheduled tokens, common-prefix
+blocks, and the request IDs preempted in that step
+(`vllm/v1/core/sched/output.py:180-219`). The base scheduler constructs this
+object at `vllm/v1/core/sched/scheduler.py:1012-1076`, then advances request
+computed-token and chunk state at `vllm/v1/core/sched/scheduler.py:1130-1155`.
+
+What is available today:
+
+| Requested datum | Internal per-step source | Externally exposed today |
+|---|---|---|
+| Batch size | `len(SchedulerOutput.num_scheduled_tokens)`; the map is exact | No raw per-step batch-size Prometheus series. Optional iteration log reports context and generation request counts. |
+| Scheduled prefill/decode tokens | `compute_iteration_details()` classifies new/cached context requests and sums scheduled tokens (`vllm/v1/utils.py:766-813`) | Optional log reports exact scheduled context/generation counts and elapsed time around the execution wait (`vllm/v1/engine/core.py:433-472`). Prometheus prompt/generation counters are cumulative output-side stats, not a raw scheduled-step stream. |
+| Chunked-prefill splits | `Request.is_prefill_chunk`, `num_computed_tokens`, and prompt/output state exist (`vllm/v1/request.py:130-175`); extended chunks are classified as context | No chunk ordinal, first/middle/final split, or chunk-size series. These must be derived before `_update_after_schedule()` mutates state. |
+| Preemptions | Exact step set in `SchedulerOutput.preempted_req_ids` | Stdout aggregates over its logging interval; `vllm:num_preemptions` is cumulative (`vllm/v1/metrics/loggers.py:219-281,624-631`). |
+| Running/waiting/deferred queues | `SchedulerStats.num_running_reqs`, `num_waiting_reqs`, and `num_skipped_waiting_reqs` (`vllm/v1/metrics/stats.py:170-183`) | Latest-value gauges `vllm:num_requests_running`, `vllm:num_requests_waiting`, and reason-labeled capacity/deferred gauges (`vllm/v1/metrics/loggers.py:453-496`). Running queue size is not the same as scheduled batch size. |
+| KV-cache usage | Scheduler reads `kv_cache_manager.usage`; the block pool computes a null-block-adjusted ratio in `[0,1]` (`vllm/v1/core/kv_cache_manager.py:181-188`; `vllm/v1/core/block_pool.py:692-711`) | Latest-value `vllm:kv_cache_usage_perc`; stdout prints the latest percent (`vllm/v1/metrics/loggers.py:250-260,524-532`). Raw total/free blocks are not exported per step. |
+| Prefix-cache counters | `PrefixCacheStats` contains request/query/hit and separately preempted request/query/hit counters (`vllm/v1/metrics/stats.py:114-143`) for local and optional connector caches | Prometheus exposes cumulative token queries/hits for local and external caches (`vllm/v1/metrics/loggers.py:547-565,1063-1101`); stdout reports interval-derived hit rates. It does not expose the preempted sub-counters separately. |
+| Total step tokens | Exact scheduled total is in `SchedulerOutput` | `vllm:iteration_tokens_total` is a histogram observed from output-side computed prompt plus generation tokens (`vllm/v1/metrics/loggers.py:693-724,1148-1171`), not a joinable raw step record. |
+
+`IterationStats` also has a wall timestamp, output-side prompt/generation data,
+preemptions, and finished-request latency samples
+(`vllm/v1/metrics/stats.py:325-347`). `Scheduler.make_stats()` produces latest
+queue/KV state, drains prefix stats, and passes through the step's CUDA-graph
+stat when an output is processed
+(`vllm/v1/core/sched/scheduler.py:2228-2264`). These are valuable existing
+plumbing, but async scheduling can schedule newer batches before an older
+output is processed. Consequently, those latest/drained values are not a safe
+per-step composition join without a schedule-time snapshot.
+
+Bottom line: no new scheduler instrumentation is needed to discover batch and
+token composition, preemptions, queues, KV use, or prefix events. A patch is
+needed only to serialize their **same-step association** and the missing
+histograms/chunk split.
+
+### 2.3 CUDA-graph observability
+
+Yes, v0.24.0 can determine the runtime graph mode and capture bucket for each
+model step internally:
+
+- `CUDAGraphStat` records unpadded tokens, padded tokens, padding, and runtime
+  mode (`vllm/compilation/cuda_graph.py:32-37`).
+- The GPU model runner asks the dispatcher for a concrete runtime mode and
+  `BatchDescriptor`, then conditionally attaches the stat to its output
+  (`vllm/v1/worker/gpu_model_runner.py:3822-3934`). It reaches the scheduler in
+  `ModelRunnerOutput` and `update_from_output()`
+  (`vllm/v1/outputs.py:231-281`;
+  `vllm/v1/core/sched/scheduler.py:1464-1477`).
+- The dispatcher is the source of truth: it pads to a captured descriptor,
+  tries `FULL`, then `PIECEWISE`, and returns `NONE` when no graph key matches
+  (`vllm/v1/cudagraph_dispatcher.py:235-324`; the design explanation is at
+  `docs/design/cuda_graphs.md:81-120`). At serving time, `FULL` and `PIECEWISE`
+  dispatch to a captured wrapper; `NONE` calls the runnable directly
+  (`vllm/compilation/cuda_graph.py:233-360`).
+
+Thus `runtime_mode != NONE` is a graph-path dispatch, and `num_padded_tokens` is
+the selected token bucket. `PIECEWISE` means graph-wrapped compiled pieces, not
+a full-step graph hit. Startup normally captures all dispatcher descriptors
+(`vllm/v1/worker/gpu_model_runner.py:6595-6643`), but `CUDAGraphStat` has no
+capture-versus-replay bit; a lazy first capture and a replay have the same stat.
+With built-in `cudagraph_metrics`, the stat is aggregated
+into a frequency table and printed at the logging interval
+(`vllm/config/observability.py:56-58`;
+`vllm/compilation/cuda_graph.py:40-124`); it is not exposed as a raw per-step
+Prometheus event. OpProf only needs to preserve the already-computed object on
+every enabled step.
+
+Capture modes and sizes live in `CompilationConfig`:
+`cudagraph_mode`, `cudagraph_capture_sizes`, and
+`max_cudagraph_capture_size` are defined at
+`vllm/config/compilation.py:587-688`. The V1 default is
+`FULL_AND_PIECEWISE`. Final sizes are generated or validated in
+`VllmConfig._set_cudagraph_sizes()`; the default is 1/2/4, then steps of 8 below
+256 and 16 above it, bounded by token/batch configuration, with user overrides
+written back to the final list (`vllm/config/vllm.py:1635-1800`).
+
+### 2.4 MoE routing and Qwen3-30B-A3B backend
+
+Qwen3 MoE creates a replicated gate and `FusedMoE` with the model's number of
+experts and top-k, and normally runs its router internally
+(`vllm/model_executor/models/qwen3_moe.py:137-249`). The causal-LM wrapper
+retains the list of MoE layers and their expert topology
+(`vllm/model_executor/models/qwen3_moe.py:662-729`). Router top-k IDs are
+available immediately after routing and before EPLB remapping
+(`vllm/model_executor/layers/fused_moe/router/base_router.py:233-278`).
+
+There is an exact, built-in access path: `--enable-return-routed-experts`
+(`vllm/config/model.py:209-215`; `vllm/engine/arg_utils.py:826-830`). It installs
+per-layer callbacks, writes top-k IDs to preallocated GPU buffers, copies the
+step to pinned CPU memory, and returns it through `ModelRunnerOutput`
+(`vllm/model_executor/layers/fused_moe/routed_experts_capturer.py:58-84`;
+`vllm/v1/worker/gpu_model_runner.py:3652-3666,4637-4683,7382-7437`). This is
+reachable without modifying the fused kernels or saving router logits.
+Raw router-logit tensors are not plumbed to engine outputs; exposing those
+would be deeper and much higher-volume surgery than using the existing top-k
+ID callback.
+
+It is not cheap enough for Layer 1. The per-worker transit buffer is a few MB;
+the scheduler's whole-slot buffer can reach multiple GB and performs a CPU
+fancy-index write each step
+(`vllm/model_executor/layers/fused_moe/routed_experts_capturer.py:223-305`;
+the scheduler store is at `vllm/v1/core/sched/scheduler.py:1501-1520`). It is
+also incompatible with pipeline parallelism greater than one and with KV
+connectors (`vllm/config/vllm.py:871-893`). Exact per-layer expert load should
+therefore be collected in a separate sampled Layer-2 run and reduced offline.
+EPLB has load state, but enabling it adds communication and its logs are
+aggregate balancing summaries; it is not a free observability tap
+(`vllm/distributed/eplb/eplb_state.py:62-171,478-575`).
+
+For the normal unquantized BF16/FP16 Qwen3-30B-A3B configuration on H20/SM90,
+the automatic unquantized oracle explicitly moves both FlashInfer backends
+behind Triton on Hopper because they are expected to be slower
+(`vllm/model_executor/layers/fused_moe/oracle/unquantized.py:43-86`). It selects
+the first supported backend and documents possible shape-specific fallbacks
+(`vllm/model_executor/layers/fused_moe/oracle/unquantized.py:152-250`). The
+primary backend is therefore **Triton** under those assumptions. A quantized
+checkpoint, LoRA, expert activation format, or explicit `--moe-backend` can
+change that conclusion and must be recorded with the campaign run.
+
+### 2.5 V1 execution loop and natural hooks
+
+The control path is:
+
+```text
+EngineCore busy loop
+  -> Scheduler.schedule()
+  -> executor.execute_model(SchedulerOutput)
+  -> GPUWorker -> GPUModelRunner.execute_model()
+  -> Scheduler.update_from_output(SchedulerOutput, ModelRunnerOutput)
+```
+
+Evidence and hook implications:
+
+- `EngineCore.run_busy_loop()` and `_process_engine_step()` drive steps at
+  `vllm/v1/engine/core.py:1259-1317`.
+- The normal step retains the exact scheduler output across the future and
+  passes it back to `update_from_output()`
+  (`vllm/v1/engine/core.py:479-508`). The batch-queue path does the same for
+  multiple in-flight batches (`vllm/v1/engine/core.py:519-632`).
+- The scheduler's stable interface is schedule/update at
+  `vllm/v1/core/sched/interface.py:51-107`. The base implementation builds the
+  output at `vllm/v1/core/sched/scheduler.py:1012-1100`, mutates request state
+  immediately afterward at lines 1130-1155, and consumes model results at
+  lines 1464-1803.
+- Worker execution wraps `gpu_model_runner.execute_model()` at
+  `vllm/v1/worker/gpu_worker.py:895-898`; the model runner's entry is
+  `vllm/v1/worker/gpu_model_runner.py:4055-4069`.
+
+The natural Layer-1 hooks are therefore in the base scheduler: snapshot after
+the `SchedulerOutput` is complete but before request state advances, then
+finalize with the matching model output. This covers the async subclass and
+avoids frontend/output aggregation. The only worker change is broadening the
+existing CUDA-graph-stat condition. Layer 2 uses the existing profile control
+path unchanged.
+
+### 2.6 Relevant changes from 0.20.0
+
+For comparison only, tag `v0.20.0` was fetched into the allowed clone without
+changing the pinned working tree. It resolves to commit
+`88d34c6409e9fb3c7b8ca0c04756f061d2099eb1`.
+
+- Async scheduling and the batch-queue architecture already existed in 0.20.0;
+  `SchedulerConfig` selected `AsyncScheduler` at
+  `v0.20.0:vllm/config/scheduler.py:146-176`, and the core schedule/future/update
+  paths were at `v0.20.0:vllm/v1/engine/core.py:402-431,443-533`. It is
+  incorrect to treat async scheduling as new in 0.24.0.
+- 0.24.0 adds DP prefill cadence/throttling and passes a prefill-throttle flag
+  into `schedule()` (`vllm/config/scheduler.py:140-161` and
+  `vllm/v1/engine/core.py:474-490`). It also hardens zero-token/dummy-iteration
+  handling (`vllm/v1/engine/core.py:433-472`) and changes when the batch queue
+  fills.
+- The 0.24 scheduler contains explicit scheduled/processed sequence fences and
+  snapshots state that must survive later async schedules, including routed
+  experts (`vllm/v1/core/sched/scheduler.py:293-322,1093-1165`). That is direct
+  evidence that an OpProf hook must not read mutable request/queue state only
+  when an older output returns.
+- `vllm/config/profiler.py`, `vllm/profiler/wrapper.py`, and
+  `vllm/entrypoints/serve/profile/api_router.py` are byte-identical between the
+  two tags (`git diff --quiet v0.20.0..v0.24.0` returned success). Profiler
+  endpoint semantics did not move; the scheduler/core line numbers and safe
+  composition hook did.
+
+Practical conclusion: do not port a 0.20 line-number patch into EngineCore.
+Hook the 0.24 base scheduler's schedule/update pair so sync, async, and batch
+queue paths share one implementation.
+
+## 3. Proposed dual-layer design
+
+Layer 1 emits one schema-versioned JSONL record per scheduler step from the
+EngineCore/scheduler process. It contains step/timestamps; scheduled and decode
+batch counts; scheduled prefill/decode tokens; aggregate first/middle/final
+chunk information; fixed-bucket context and chunk-size histograms; exact step
+preemptions; schedule-time running/waiting/deferred queues; total/free/used KV
+blocks and ratio; local/external prefix deltas; and CUDA-graph mode/bucket. No
+request IDs or raw context-length lists are written. MoE expert load is null and
+explicitly marked Layer-2-only.
+
+Records are encoded with the existing `msgspec` dependency
+(`requirements/common.txt:35`), placed into a
+bounded non-blocking queue, and drained by one background JSONL writer. It
+flushes every 1 MiB or one second and at clean shutdown, with explicit drop/gap
+counters and no per-step `fsync`. `VLLM_OPPROF_DIR` is the startup-only switch;
+unset is a true no-op. The unmeasured foreground estimate is 25-100
+microseconds per step, so Phase 2 must enforce the less-than-3% overhead gate
+with paired off/on serving runs.
+
+Layer 2 configures the already-built torch profiler and samples short windows,
+initially two warm-up plus eight active worker iterations. Set
+`max_iterations=8`; the wrapper stops the underlying profiler on the subsequent
+step after the counter exceeds the limit, as its existing test documents
+(`vllm/profiler/wrapper.py:83-114`;
+`tests/v1/worker/test_gpu_profiler.py:77-98`). The controller then calls
+`/stop_profile` to clear control state. Step-count/on-demand endpoint control is
+preferred to wall time. Layer 2 writes one trace per TP worker and joins to
+Layer 1 through the existing iteration annotation, order, token counts, and
+timestamps.
+
+Under CUDA-graph replay, the profiler is configured for CUDA activity, but the
+Python callable is not re-executed: vLLM calls `CUDAGraph.replay()`
+(`vllm/compilation/cuda_graph.py:233-360`). CUDA activity may therefore be
+visible through Kineto/CUPTI, while Python/PyTorch scopes cannot be assumed to
+provide the original per-layer attribution. Layerwise NVTX is explicitly
+unsupported with graphs (`vllm/config/observability.py:60-63`). Phase 2 must
+check whether the pinned torch/CUPTI combination exposes individual replayed
+kernels or a coarser graph launch. A matched eager window can identify kernel
+families but must not replace graph-enabled production timing.
+
+Layer-2 kernel time is grouped into attention, router/top-k, expert/dense GEMMs,
+normalization/activation, collectives, and sampling. Per-rank critical-path
+times calibrate a composition/graph-bucket-conditioned Layer-1 attribution;
+held-out profiler windows validate that mapping. Exact routed experts are
+enabled only in a separate sampled server run.
+
+### Patch surface
+
+| File | Approximate lines |
+|---|---:|
+| `vllm/envs.py` | 4 |
+| `vllm/v1/opprof.py` (new) | 220 |
+| `vllm/v1/core/sched/scheduler.py` | 32 |
+| `vllm/v1/worker/gpu_model_runner.py` | 4 |
+| `tests/v1/core/test_opprof.py` (new) | 190 |
+| **Total** | **5 files / about 450 lines** |
+
+Zero-patch wins: profiler config/endpoints/wrapper, `SchedulerOutput`, existing
+metrics plumbing, CUDA kernels, fused MoE/router code, Qwen3 model code, and
+frontend APIs. Supporting OpProf together with `--disable-log-stats` would add
+about 12 lines in `vllm/v1/core/kv_cache_manager.py`; the design recommends
+failing fast instead.
+
+### Main risks
+
+- Histogram construction and JSON encoding are the likely foreground overhead;
+  the 3% budget must be measured, not inferred.
+- Async scheduling can misassociate latest queue/cache state with an older
+  output; snapshot inside `schedule()` and pair by the exact output object.
+- Layer 1 must be scheduler-owned to avoid TP duplicates; Layer 2 remains
+  per-rank and needs offline aggregation.
+- At 500 steps/s and about 1 KiB/record, an uncompressed stream is roughly
+  44 GB/day per EngineCore. Runs must be duration-bounded, with queue/drop
+  counters and post-run compression.
+- Kineto perturbs execution, and graph replay loses Python-scope attribution;
+  it is a sampled calibration instrument, not an always-on latency source.
+
+## 4. Open design decisions
+
+1. Approve JSONL/`msgspec`, an 8192-record queue, context edges
+   `(128, 256, 512, 1K, 2K, 4K, 8K, 16K, 32K, 64K, 128K, +inf)`, and chunk-size
+   edges `(16, 32, 64, 128, 256, 512, 1K, 2K, +inf)`.
+2. Approve exact MoE expert load as Layer-2-only. The alternative is a new GPU
+   histogram path, with a larger patch and an overhead/graph-capture risk.
+3. Confirm checkpoint precision/quantization and intended TP/EP/DP topology.
+   The H20 Triton conclusion assumes unquantized BF16/FP16, auto backend, and no
+   LoRA.
+4. Choose the initial profiler window: recommended two warm-up plus eight
+   active iterations, or a longer active window.
+5. Decide whether the less-than-3% gate applies to point estimates or the upper
+   95% confidence bound for every primary metric.
+6. Confirm that OpProf campaign runs may reject `--disable-log-stats`; support
+   for it increases the patch to six files and about 462 lines.
+
+## Data sanity block
+
+- **Pinned root listing:** n=45 names; lexicographic min=`.buildkite`,
+  max=`vllm`; distinct=45; SHA-256=`75222e5d3bacdd043e421c417496aaf0fa5a9429b7244698b6d3ca2218b85b58`.
+- **Source versions compared:** n=2 tags; min=`v0.20.0`, max=`v0.24.0`;
+  distinct=2; pinned working-tree version remains exactly `v0.24.0`.
+- **Torch pin observations:** n=4 declarations
+  (`pyproject`, build requirements, CUDA runtime requirements, CMake);
+  min=max=`2.11.0`; distinct=1.
+- **Evidence-bearing source files inspected and cited:** n=44
+  distinct v0.24.0 paths; min/max not applicable to categorical paths;
+  v0.20.0 comparison touched n=5 distinct paths; min/max not applicable.
+- **Invariants checked:** exact tag equals `v0.24.0`; HEAD equals the recorded
+  commit; clone is clean; top-level names are unique; requested path was used;
+  torch pins agree; CUDA-graph modes are in `{NONE, PIECEWISE, FULL}`; KV usage
+  definition is bounded in `[0,1]`; scheduled-token total is defined as the sum
+  of per-request scheduled tokens; graph bucket is not smaller than unpadded
+  tokens; no remote/GPU/install/model-download action occurred.
+- **Benchmark/statistical conclusion check:** no performance benchmark was run,
+  so there are no per-configuration curves or measured overhead values on which
+  to perform identical-value, monotonicity, or continuity checks. All overhead
+  numbers in the design are labeled estimates.

docs/opprof/phase3-results.md

@@ -0,0 +1,262 @@
+# OpProf Phase 3 dash0 results
+
+Status: **FINAL A-P3-7 PARTIAL-MATRIX ANALYSIS — H1a INCONCLUSIVE; H1b PASS**.
+
+Date: 2026-07-12. The matrix is permanently frozen at 40/52 accepted measured
+runs and 20/24 complete pattern/config cells. A-P3-7 permits existential
+confirmation on complete comparisons but forbids refutation from incomplete
+coverage. The result is therefore **PARTIAL**: H1b is confirmed by five
+evaluable contrasts, while H1a cannot be evaluated on a valid pattern pair.
+
+The final machine result is
+`runs/opprof-phase3/phase3/metrics.json` (SHA-256
+`dcc44941bfe1eb56c7a069148e5565eddd1ccf966e6914aa16b64bd67d02f0f0`).
+The final analyzer SHA-256 is
+`483c170838a1e81a170d8425b4be22cc1be02ea4bc9492b474daf7ae86186d67`;
+7/7 no-GPU analysis tests passed, including ResourceWarning-as-error.
+
+## Data-sanity result first
+
+All **23/23** machine-checked invariants pass before inference: 40 unique
+accepted primary markers selected only through completed-stage records; exact
+240-second clean windows; zero clean failures; finite moderate rates within
+5%; balanced Layer-1 footer/sidecar accounting; ratios in range; 72 accepted
+trace files with the registered eight-file first-wave deviation; zero other
+GPU processes; exact 20-cell coverage; and the exact A-P3-7 missing-cell and
+missing-contrast sets.
+
+The accepted data contain 77,109 clean completions, 352,907 clean Layer-1
+steps, and 542,350 total Layer-1 records. Device-time classifiability is
+97.05–99.64%. No malformed trace, accounting mismatch, counter underflow,
+private-text leak, or unexpected identical-result family was found.
+
+Declared limitations are not hidden as sanity passes: eight accepted
+saturation trace files were lost during the registered first-wave controller
+cleanup; all four confirmation runs are absent; MoE per-layer CV is unavailable
+because layer scopes cover less than 80% of MoE GEMM time; and only one of nine
+completed C00-moderate patterns passes the Layer-2 inference gates.
+
+## Frozen coverage and A-P3-7 logic
+
+The four incomplete cells, each missing saturation and moderate, are exactly:
+
+- P03/C11;
+- P05/C00;
+- P10/C00-TP2; and
+- P11/C00.
+
+The four missing confirmations are P10, P06, P03, and P01 C00-moderate.
+Formal H1a coverage is therefore the nine completed C00 patterns P01, P02,
+P03, P04, P06, P07, P08, P09, and P10. Formal H1b has six evaluable and two
+non-evaluable frozen contrasts. A-P3-7's asymmetry is mandatory: one valid hit
+can confirm an existential hypothesis, but absence of a hit cannot support the
+original complete-matrix null.
+
+## H1a — operator composition
+
+**Verdict: INCONCLUSIVE; refutation is not allowed.**
+
+All moderate traces were parseable and classifiable, but only P04's two
+windows jointly passed classifiability, `abs(SMD)<=0.25` representativeness,
+and fixed recovery. P01/P02/P03/P09 failed representativeness in both windows;
+P06/P07/P08/P10 additionally failed at least one recovery. Thus only one of
+nine completed patterns is inferentially evaluable, leaving zero pattern pairs,
+zero ranking tests with data, and no Kendall tau-b pairs. No qualifying
+inversion can be accepted, and this is not evidence for a shared ranking.
+
+Descriptively, P04 is attention-led at both loads. P06 and P10 change from
+attention-led at saturation to MoE-GEMM-led at moderate load; the other
+available patterns remain MoE-GEMM-led. These are ranking changes in sampled
+windows, not H1a evidence because their window gates fail.
+
+### Per-pattern operator shares, both load points
+
+Values are the mean percentage across the two Layer-2 windows. `E` means the
+row is inferentially evaluable; `D` means data are available but descriptive
+only because a registered window/recovery gate failed; `NE-cell` means the cell
+is missing; `NE-trace` means the accepted run has no retained trace. `Other` is
+unclassified device activity. Collective, sampler, and dense-GEMM shares round
+to 0.00% in every available C00 row.
+
+| Pattern | Load | Use | Attention | MoE GEMM | Router | Collective | Sampler | Dense GEMM | Norm/elt | KV | Other | Top family |
+|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|
+| P01 | saturation | D | 16.32 | 77.81 | 0.85 | 0.00 | 0.00 | 0.00 | 3.90 | 0.10 | 1.02 | MoE GEMM |
+| P01 | moderate | D | 12.09 | 82.43 | 1.03 | 0.00 | 0.00 | 0.00 | 3.38 | 0.11 | 0.96 | MoE GEMM |
+| P02 | saturation | D | 32.67 | 62.39 | 1.05 | 0.00 | 0.00 | 0.00 | 2.98 | 0.16 | 0.74 | MoE GEMM |
+| P02 | moderate | D | 20.91 | 72.53 | 1.64 | 0.00 | 0.00 | 0.00 | 3.68 | 0.25 | 1.00 | MoE GEMM |
+| P03 | saturation | D | 39.11 | 56.60 | 0.44 | 0.00 | 0.00 | 0.00 | 2.98 | 0.03 | 0.84 | MoE GEMM |
+| P03 | moderate | D | 23.32 | 57.45 | 3.54 | 0.00 | 0.00 | 0.00 | 12.29 | 0.83 | 2.57 | MoE GEMM |
+| P04 | saturation | D | 65.88 | 29.77 | 1.16 | 0.00 | 0.00 | 0.00 | 2.30 | 0.17 | 0.72 | Attention |
+| P04 | moderate | **E** | 47.88 | 40.64 | 3.28 | 0.00 | 0.00 | 0.00 | 5.85 | 0.45 | 1.90 | Attention |
+| P05 | saturation | NE-cell | — | — | — | — | — | — | — | — | — | — |
+| P05 | moderate | NE-cell | — | — | — | — | — | — | — | — | — | — |
+| P06 | saturation | D | 56.59 | 39.35 | 1.04 | 0.00 | 0.00 | 0.00 | 2.25 | 0.15 | 0.62 | Attention |
+| P06 | moderate | D | 31.60 | 57.52 | 2.97 | 0.00 | 0.00 | 0.00 | 5.69 | 0.43 | 1.80 | MoE GEMM |
+| P07 | saturation | NE-trace | — | — | — | — | — | — | — | — | — | — |
+| P07 | moderate | D | 28.67 | 62.18 | 2.42 | 0.00 | 0.00 | 0.00 | 4.86 | 0.37 | 1.49 | MoE GEMM |
+| P08 | saturation | NE-trace | — | — | — | — | — | — | — | — | — | — |
+| P08 | moderate | D | 34.59 | 57.06 | 2.22 | 0.00 | 0.00 | 0.00 | 4.44 | 0.34 | 1.34 | MoE GEMM |
+| P09 | saturation | D | 26.23 | 69.03 | 0.36 | 0.00 | 0.00 | 0.00 | 3.43 | 0.00 | 0.95 | MoE GEMM |
+| P09 | moderate | D | 16.19 | 75.00 | 2.14 | 0.00 | 0.00 | 0.00 | 4.83 | 0.35 | 1.49 | MoE GEMM |
+| P10 | saturation | D | 67.80 | 28.90 | 0.76 | 0.00 | 0.00 | 0.00 | 1.87 | 0.10 | 0.57 | Attention |
+| P10 | moderate | D | 30.61 | 54.04 | 2.81 | 0.00 | 0.00 | 0.00 | 9.80 | 0.68 | 2.06 | MoE GEMM |
+| P11 | saturation | NE-cell | — | — | — | — | — | — | — | — | — | — |
+| P11 | moderate | NE-cell | — | — | — | — | — | — | — | — | — | — |
+
+### Clean CUDA-graph modes at moderate load
+
+| Pattern | FULL | PIECEWISE | NONE/eager |
+|---|---:|---:|---:|
+| P01 | 1.98% | 73.19% | 24.83% |
+| P02 | 83.16% | 13.00% | 3.84% |
+| P03 | 96.37% | 0.00% | 3.63% |
+| P04 | 98.32% | 0.02% | 1.66% |
+| P05 | — | — | — |
+| P06 | 99.10% | 0.05% | 0.85% |
+| P07 | 98.76% | 0.00% | 1.24% |
+| P08 | 98.64% | 0.01% | 1.36% |
+| P09 | 91.79% | 3.69% | 4.52% |
+| P10 | 98.33% | 0.12% | 1.55% |
+| P11 | — | — | — |
+
+P01 is the clear mode outlier: PIECEWISE covers 73.19% of clean moderate
+steps. Full per-mode operator segments and active-step counts remain in the
+machine result; no mode with fewer than eight sampled steps is summarized.
+
+## H1b — rectangular-benchmark miss
+
+**Verdict: PASS.** Five of six evaluable frozen contrasts pass at least one
+waste threshold with a simultaneous positive bound and at least 5% useful-token
+efficiency loss. The two P05 contrasts are `NOT EVALUABLE`; no value is
+imputed. Holm correction retains the original eight planned contrasts per
+metric. Reported simultaneous intervals below are percentage-point effects;
+passing bootstrap p-values are zero at 100,000-resample resolution.
+
+| Frozen contrast | Status | Passing evidence | Useful-token efficiency loss |
+|---|---|---|---:|
+| P05 vs P01 | **NOT EVALUABLE** | P05/C00 missing | — |
+| P05 vs P03 | **NOT EVALUABLE** | P05/C00 missing | — |
+| P06 vs P02 | **PASS** | R64 +23.015 pp, simultaneous 95% [22.345, 23.683] | 11.609% |
+| P06 vs P04 | **PASS** | R64 +35.440 pp [34.826, 36.059] | 22.846% |
+| P09 vs P01 | **PASS** | padding +6.673 pp [5.887, 7.492]; R64 +39.616 pp [39.065, 40.166] | 8.325% |
+| P09 vs P03 | EVALUABLE, NO HIT | padding +8.542 pp and R64 +52.041 pp are material, but association gate fails | **3.840%**, below 5% |
+| P10 vs P03 | **PASS** | padding +5.565 pp [2.010, 10.341]; R64 +44.787 pp [43.635, 45.971] | 44.693% |
+| P10 vs P04 | **PASS** | padding +5.399 pp [1.868, 10.213]; R64 +44.787 pp [43.638, 45.949] | 14.260% |
+
+This confirms the existential H1b claim on completed contrasts. It cannot
+support a null claim about P05 or the missing matrix.
+
+## Waste accounting
+
+The preregistered contrast thresholds are 5 percentage points for padding,
+10 points for graph miss/overflow, 0.15 absolute for R64, 0.10 for mixed-batch
+interference, and 0.15 for MoE layer CV, plus the 5% efficiency/residual
+association gate. Values below are clean C00-moderate results; efficiency is
+scheduled useful tokens per model-step millisecond.
+
+| Pattern | Padding | Graph miss | Overflow | R64 | Mixed interference | Efficiency | Supported/total mixed steps |
+|---|---:|---:|---:|---:|---|---:|---:|
+| P01 | 1.869% | 24.826% | 24.826% | 0.3687 | N/A | 4.9673 | 0/5,646 |
+| P02 | 1.998% | 3.837% | 3.837% | 0.3687 | N/A | 2.6664 | 0/1,536 |
+| P03 | 0.000% | 1.736% | 1.736% | 0.2444 | N/A | 4.7356 | 0/138 |
+| P04 | 0.167% | 1.301% | 1.301% | 0.2444 | N/A | 3.0547 | 0/141 |
+| P05 | — | — | — | — | — | — | — |
+| P06 | 0.276% | 0.830% | 0.830% | 0.5988 | N/A | 2.3568 | 0/152 |
+| P07 | 0.140% | 1.240% | 1.240% | 0.0000 | N/A | 2.7513 | 0/186 |
+| P08 | 0.005% | 1.357% | 1.357% | 0.0000 | N/A | 2.1478 | 0/178 |
+| P09 | **8.542%** | 4.518% | 4.518% | **0.7648** | N/A | 4.5537 | 0/1,054 |
+| P10 | **5.565%** | 0.921% | 0.921% | **0.6923** | N/A | 2.6191 | 0/92 |
+| P11 | — | — | — | — | — | — | — |
+
+No irregular-versus-control graph-miss or overflow contrast reaches the
+positive 10-point H1b threshold. P01 has the largest absolute miss rate but is
+a rectangular control; P09 actually reduces miss versus P01 by 20.308 points.
+
+The registered leave-one-pattern-out robust fits were applied for mixed-batch
+interference, but no pattern retained 30 supported mixed steps inside both
+pure-fit convex supports; every result is N/A rather than extrapolated. No
+separate LOAO operator-ranking procedure was preregistered, so none was added
+post hoc. The required two-window rule is used unchanged, and confirmation-run
+robustness is `NOT EVALUABLE` because all four confirmations are missing.
+
+### Top three waste findings
+
+1. **P10 real trace:** R64 is 44.787 points above both long rectangular
+   controls; padding is 5.565/5.399 points higher; efficiency is 44.693% worse
+   than P03 and 14.260% worse than P04.
+2. **P09 production-shaped mix:** versus P01, R64 rises 39.616 points and
+   padding 6.673 points with 8.325% lower efficiency. Its still-larger P03
+   contrast does not pass because efficiency loss is only 3.840%.
+3. **P06 bimodal long-output burst:** R64 rises 23.015 points versus P02 and
+   35.440 points versus P04, paired with 11.609% and 22.846% efficiency loss.
+
+## Pattern-conditioned operational findings
+
+- **P10/TP2 non-stabilization:** the fresh run completed 17 warm-up requests,
+  but trailing scheduled-token throughput fell 18,022.4 → 16,384.0 → 14,062.6
+  tokens/s. Its 36.76% fitted drift exceeds A-P3-6's 10% limit; same-wave
+  synthetic P11 and P03 completed 512 and 102 warm-up requests and passed their
+  applicable registered gates. The orchestrator accepts this as a
+  pattern-conditioned finding, not an accepted throughput measurement.
+- **Long-context drain:** P10/C01 saturation naturally drained for 288.619
+  seconds. The clean window and accounting were valid; the original 120-second
+  short-pattern watchdog was miscalibrated, while the amended 600-second P10
+  budget correctly retained the run.
+- **Failure boundary:** P01/C01 moderate had 5,828 clean successes and zero
+  clean failures. Two `ServerDisconnectedError` records occurred only in
+  excluded post-profile time; separating clean and auxiliary windows prevented
+  valid data from being discarded.
+- **Layer-2 perturbation:** despite 97%+ kernel classifiability, eight of nine
+  completed moderate patterns failed representativeness or recovery. Together
+  with the Phase-2 51.3% active-window perturbation, this shows that sparse
+  Kineto windows are operationally fragile for these serving patterns.
+
+These incidents consistently show long-context and mixed serving patterns
+breaking assumptions calibrated on shorter rectangular loads: fixed drain
+watchdogs, completion-count warm-up, global failure scope, and profile-window
+representativeness.
+
+## Matrix and GPU accounting
+
+| Item | Final value |
+|---|---:|
+| Planned measured runs | 52 |
+| Accepted measured runs | **40** |
+| Complete cells | **20/24** |
+| Accepted confirmations | 0/4 |
+| Accepted clean failures | 0 |
+| Drain-quarantined accepted runs | 0 |
+| Accepted Layer-2 trace files | 72 |
+| Registered missing trace files | 8 |
+| Other-user GPU processes | 0 |
+| Cumulative GPU use | **14.025875 H20-hours** |
+| Reserved/unused headroom | **1.974125 H20-hours** |
+
+No GPU work was run for A-P3-7 analysis. The reserved headroom remains unused,
+and all eight GPUs were at 0 MiB/0% before and after the CPU-only analysis.
+
+## Data sanity block
+
+| Numeric family | n | finite | missing | min | max | distinct | Check |
+|---|---:|---:|---:|---:|---:|---:|---|
+| Accepted throughput (req/s) | 40 | 40 | 0 | 0.445833 | 43.300000 | 37 | finite, positive, not identical |
+| Token efficiency (tokens/ms) | 40 | 40 | 0 | 2.147827 | 8.295849 | 40 | finite, positive, distinct |
+| Drain duration (s) | 40 | 40 | 0 | 0.305419 | 288.619108 | 40 | non-negative; no quarantine |
+| Clean Layer-1 steps/run | 40 | 40 | 0 | 718 | 20,171 | 40 | sum 352,907; continuous per run |
+| Operator classifiable fraction | 72 | 72 | 0 | 0.970458 | 0.996378 | 72 | all >=0.70 and within [0,1] |
+| Operator-family shares | 576 | 576 | 0 | 0.000000 | 0.873417 | 348 | within [0,1] |
+| Waste ratios | 240 | 240 | 0 | 0.000000 | 1.000000 | 168 | within [0,1] |
+| Waste-contrast effects | 36 | 24 | 12 expected | -0.203082 | 0.520413 | 17 | signed; N/A preserved |
+| Kendall tau-b | 0 | 0 | 0 | — | — | 0 | expected: only one valid pattern |
+| Accepted clean failures | 40 | 40 | 0 | 0 | 0 | 1 expected | sum zero |
+| Final GPU memory (MiB) | 8 | 8 | 0 | 0 | 0 | 1 expected | cleanup passed |
+
+Checked invariants: accepted IDs unique; exact 40-run/20-cell coverage; exact
+four missing cells and two missing contrasts; 240-second clean windows; zero
+clean failures; moderate rate within 5%; balanced Layer-1 accounting and zero
+drops; trace parsing and classifiability; ratios in range; exact 72+8 trace
+accounting; clock/load snapshots present; no other-user process; drain
+quarantine below 20%; non-identical pattern results; and exact reproduction of
+the 36.76% A-P3-6 operational result. **No data-sanity red flag remains.** H1a
+is inconclusive because of declared Layer-2 validity limits; H1b passes on five
+complete contrasts; the compound hypothesis is partial and cannot be refuted.

docs/opprof/phase4-optimization-plan.md

@@ -0,0 +1,306 @@
+# OpProf Phase 4 measured optimization plan
+
+Status: **PROPOSAL FROM ACCEPTED PHASE-3 DATA; NO UNMEASURED GAIN CLAIMS**.
+
+Date: 2026-07-12. This plan uses only the accepted Phase-3 40/52-run,
+20/24-cell dataset and the single optional Phase-4 capture-size validation.
+Phase-3 protocol and results are frozen. Bounds below are ceilings in the
+units actually measured; padding or raggedness percentages are not relabeled
+as end-to-end throughput gains.
+
+## Pinned implementation context
+
+- Model/hardware: Qwen3-30B-A3B BF16, one H20, TP1 primary.
+- vLLM source: accepted OpProf tip
+  `23450fb21ac255b0cf710f4ee965ee694921975d` on v0.24.0.
+- vLLM 0.24.0 exposes `--cudagraph-capture-sizes`,
+  `--max-cudagraph-capture-size`, `--max-num-seqs`, and
+  `--max-num-batched-tokens` (`vllm/engine/arg_utils.py:1390-1467`).
+- An explicit capture-size list replaces the inferred list. The default is
+  `[1,2,4]`, multiples of eight below 256, then multiples of sixteen through
+  the maximum, normally 512 (`vllm/config/vllm.py:1669-1792`).
+- Chunked prefill remains enabled. vLLM schedules decode first, then fills the
+  remaining MBT budget with prefill and chunks an over-budget prefill
+  (`docs/configuration/optimization.md:45-59`).
+
+The ranking is by measured opportunity, readiness, and downside together—not
+by a normalized composite score. Items that share the same raggedness bound
+are explicitly non-additive.
+
+## Ranked optimization list
+
+| Rank | Tier | Target and affected regime | Measured bound | Owner | Decision |
+|---:|---|---|---|---|---|
+| 1 | Config now / scheduler backlog | Preserve prefix affinity and prefix caching for P08-like shared-prefix traffic | P08 vs matched P07: **+82.14% saturation req/s**, 80% prefix-hit ratio, **62.29% fewer prefill tokens** | Serving/config owner now; cache-aware dispatch upstream | Deploy behind a workload classifier; do not apply to no-sharing traffic |
+| 2 | Scheduler | Length-aware cohorting/admission for ragged P10/P09/P06 | At most **44.79 pp** R64 contrast and **44.69%** measured efficiency gap on P10; 39.62 pp/8.32% on P09; 35.44 pp/22.85% on P06 | Upstream vLLM scheduler | Highest structural backlog item; preserve fairness and arrival semantics |
+| 3 | Config now | Add exact small CUDAGraph sizes for P09/P10-like moderate decode batches | Distributional bound **4.98 pp P09 / 5.26 pp P10** padding; P09 validation achieved **4.980 pp** | Serving/config owner | Mechanism confirmed; canary only because p95 was +3.01% in one pair |
+| 4 | Config now | Route P06/P10-like pools to MNS=64 | Saturation throughput **+3.37% P06 / +3.70% P10** versus C00 | Serving/config owner | Pattern-specific trial only; same setting was −24.27% on P01 |
+| 5 | Kernel backlog | Ragged-aware MoE GEMM and attention over the measured shape stream | Shares the rank-2 ceiling; no independent additive gain. Descriptive MoE share is 54.04–75.00% on P10/P09/P06 moderate | Engine/kernel colleagues | Optimize exact weighted shapes below; require serving confirmation |
+| 6 | Config guardrail | Keep MBT=8192 for short/high-throughput and long-prefill classes; do not globally set 2048 | Avoided saturation regressions up to **11.64% P03**, 6.53% P01, 5.82% P10 | Serving/config owner | Encode as a policy guardrail, not a positive optimization claim |
+
+## Tier A — configuration-level actions deployable today
+
+### A1. Prefix-affine routing with prefix caching
+
+**Measured regime.** P07 and P08 have the same 1,280-token prompt length,
+512-token output, burst-of-eight arrival, C00 config, and seed. P08 alone uses
+eight 1,024-token shared prefixes plus a unique 256-token suffix. With prefix
+caching already enabled:
+
+| Metric, saturation | P07 no sharing | P08 high sharing | Delta |
+|---|---:|---:|---:|
+| Prefix-query hit ratio | 0.00 | 0.80 | +0.80 |
+| Clean prefill tokens | 1,528,968 | 576,512 | **−62.29%** |
+| Completed throughput | 5.1083 req/s | 9.3042 req/s | **+82.14%** |
+
+**Root mechanism.** The matched cell changes only controlled prefix sharing;
+the observed hit and prefill-token changes directly identify cache reuse. The
+82.14% throughput delta is a point observation and the maximum evidence-backed
+gain for this exact regime, not a fleet-wide forecast.
+
+**Action now.** Keep `--enable-prefix-caching`; hash an application-known
+stable prefix or conversation identity to a replica so related requests do
+not destroy affinity through round-robin load balancing. Apply only when the
+online prefix-query hit ratio resembles P08, not P07.
+
+**Verification.** Interleave affinity ON/OFF on the same replicas and fixed
+request stream. Primary gates are prefix-hit ratio, prefill-token reduction,
+completed req/s, TTFT p95, per-replica queue imbalance, and KV occupancy. A
+throughput gain with queue/fairness or KV-capacity regression does not pass.
+
+### A2. Measured CUDAGraph capture sizes
+
+**Measured regime.** P09 moderate has decode-batch p50/p95/max 4/16/25. P10
+moderate has 1/4/7. Default captures skip sizes 3, 5, 6, 7, and 9, so those
+batches pad upward. Replaying the Phase-3 hit distribution predicts that
+adding exactly `{3,5,6,7,9}` to the complete default list can remove:
+
+- **4.9776 percentage points** of P09's 8.5421% graph-hit padding; and
+- **5.2579 points** of P10's 5.5652% padding.
+
+The list must be `default ∪ {3,5,6,7,9}`; passing only five sizes would replace
+and discard the rest of the default list.
+
+**Closed-loop result.** One P09 moderate ON/OFF pair used fresh servers, the
+same fixed seed and accepted saturation-rate source, 60 excluded warm-up
+seconds, and 240 clean seconds per arm. No Layer-2 profiler ran.
+
+| Metric | ON: exact sizes | OFF: default | ON relative to OFF |
+|---|---:|---:|---:|
+| Graph-hit padding | **3.5659%** | 8.5456% | **−4.9798 pp; −58.27%** |
+| Useful tokens/model-step ms | 4.56222 | 4.55405 | +0.179% |
+| Completed throughput | 4.9583 req/s | 4.9333 req/s | +0.507% |
+| Mean E2E latency | 1.6123 s | 1.6812 s | −4.10% |
+| p95 E2E latency | 3.9930 s | 3.8763 s | **+3.01%** |
+| Clean failures | 0 | 0 | equal |
+
+The observed padding reduction differs from the Phase-3 bound by only 0.0022
+percentage points, confirming the mechanism. It does **not** establish a broad
+performance win: token efficiency and throughput moved less than 1%, p95 moved
+the wrong way, and there is one ordered pair with no CI.
+
+Operational cost also matters: ON captured 56 FULL and 56 PIECEWISE sizes
+versus 51/51 OFF, estimated graph memory increased 0.64→0.68 GiB, and server
+ownership was 76.9 seconds longer. Deploy only as a pattern-specific canary;
+require interleaved replication with a p95 non-regression gate before rollout.
+
+### A3. Pattern-specific MNS pools
+
+The complete saturation comparisons show that `--max-num-seqs 64` is an
+interaction, not a globally better default:
+
+| Pattern | C10 MNS=64 vs C00 | Interpretation |
+|---|---:|---|
+| P01 short/short | **−24.27% req/s** | Reject for dense short traffic |
+| P03 long/short | −1.41% | No measured benefit |
+| P06 bimodal/long burst | **+3.37%** | Candidate pattern pool |
+| P10 real long-context | **+3.70%** | Candidate pattern pool |
+
+The evidence identifies the config×pattern interaction but not a lower-level
+cause. Do not attribute it to a particular kernel or queue effect without a
+new bisection. Verification is five interleaved saturation pairs per intended
+class plus TTFT, queue depth, preemption, KV usage, and exact-work checks.
+
+### A4. MBT policy guardrail
+
+`--max-num-batched-tokens 2048` versus the default 8192 changed saturation
+throughput by −6.53% P01, −11.64% P03, +0.35% P06, and −5.82% P10. The combined
+MNS64/MBT2048 setting was −30.22% on P01. Phase 3 therefore supports retaining
+MBT8192 for these classes and rejects a global MBT2048 rollout. The bound is
+an avoided regression, not new speedup.
+
+## Tier B — upstream scheduler changes
+
+### B1. Length-aware cohorting without starvation
+
+**Measured mechanism.** R64 is the rectangular padding fraction of the exact
+arrival-order prompt stream. It is 0.6923 for P10, 0.7648 for P09, and 0.5988
+for P06. Their passing control contrasts are:
+
+| Irregular pattern | Control | R64 excess | Useful-token efficiency loss |
+|---|---|---:|---:|
+| P10 | P03 | **44.79 pp** | **44.69%** |
+| P10 | P04 | **44.79 pp** | **14.26%** |
+| P09 | P01 | **39.62 pp** | **8.32%** |
+| P06 | P02 | **23.01 pp** | **11.61%** |
+| P06 | P04 | **35.44 pp** | **22.85%** |
+
+These are upper bounds on waste a length-aware path could avoid. R64 is not
+observed GPU time, and efficiency association is not causal. The scheduler
+change should maintain several ready queues by remaining prompt/context band,
+select a less-ragged cohort subject to the existing decode-first token budget,
+and impose a finite age/fairness bound. It must not rewrite request arrivals or
+drop long requests.
+
+**Verification.** Add a runtime per-step raggedness counter rather than using
+manifest R64 as a surrogate. Compare fixed-arrival ON/OFF runs for useful
+tokens/model-step ms, TTFT/E2E p95, queue age, starvation count, preemption,
+KV occupancy, and the full length histogram. The gain cannot exceed the
+corresponding R64/efficiency bounds above, and it is non-additive with a
+ragged-aware kernel.
+
+### B2. Automatic cache-aware dispatch
+
+The upstream form of A1 is a scheduler/replica dispatcher that chooses a live
+prefix-cache owner while respecting load. Its collaboration contract is the
+measured P07/P08 tuple: 1,024 shared + 256 unique prompt tokens, eight prefix
+IDs, burst size eight, output 512, target hit ratio 0.80. The load-balancing
+penalty and lost cache hits must be reported together; a synthetic cache hit
+increase without end-to-end balance is insufficient.
+
+### B3. Histogram-driven capture-list generation
+
+Static A2 proves that Layer-1 can choose useful sizes. An upstream controller
+could select a bounded number of exact sizes from padding contribution
+`count(size) * (next_bucket-size)`, while retaining the default list and a
+memory/startup budget. P09's top five `{3,5,6,7,9}` recovered 4.98 points at a
+0.04-GiB graph-memory and 76.9-second server-lifetime cost in this run. The
+selector must freeze its list before measurement and never continually tune on
+the scored window.
+
+## Tier C — kernel-level backlog and collaboration interface
+
+H1a is inconclusive, so Phase 3 does not prove a universal top operator. The
+available moderate windows are still useful shape inputs: descriptive MoE-GEMM
+shares are 57.52% P06, 75.00% P09, and 54.04% P10; attention shares are 31.60%,
+16.19%, and 30.61%. Only P04's operator windows pass inference gates, where
+attention is 47.88% and MoE GEMM 40.64%. Kernel work must therefore claim
+shape-local improvement, not a resolved global bottleneck.
+
+### Exact shape stream for kernel engineers
+
+`P`, `D`, and `N` below are per-step prefill tokens, decode tokens, and
+scheduled requests. Counts are from clean C00-moderate Layer-1 records.
+Context mix is the fraction of scheduled request-context observations in
+`<=1024 / 1025–8192 / 8193–32768 / >32768` bins.
+
+| Pattern | Model steps: decode / mixed / prefill | N p50 / p95 / max | Dominant exact `(P,D,N): count` | Context mix | Chunk signal |
+|---|---|---|---|---|---|
+| P01 | 5,760: 114 / 5,646 / 0 | 69 / 74 / 77 | `(0,66,66):18`, `(0,67,67):18`; mixed P p50=334, D p50=68 | 100.00 / 0 / 0 / 0% | 6,235 unsplit; sizes 129–512 dominate |
+| P04 | 12,455: 12,291 / 141 / 23 | 8 / 8 / 16 | `(0,8,8):12,129`, `(8191,1,3):23` | 0.03 / 94.62 / 5.35 / 0% | 238/319 chunks >2,048; first/final 122/123 |
+| P06 | 17,464: 17,310 / 152 / 2 | 8 / 16 / 16 | `(0,8,8):13,103`, `(0,16,16):4,098` | 55.11 / 42.93 / 1.96 / 0% | 174/422 >2,048; 136 in 257–512 |
+| P09 | 12,837: 11,783 / 1,054 / 0 | 4 / 16 / 25 | `(0,3,3):3,397`, `(0,2,2):1,549`, `(0,4,4):1,526`, `(0,5,5):1,050` | 51.70 / 48.26 / 0.04 / 0% | 228/1,193 >2,048; 345 in 1,025–2,048 |
+| P10 | 18,130: 17,941 / 92 / 97 | 1 / 4 / 7 | `(0,1,1):13,572`, `(0,2,2):2,675`, `(0,3,3):792`, `(0,4,4):524`, `(8192,0,1):38` | 12.25 / 39.94 / 47.76 / 0.05% | 138/191 >2,048; first/middle/final/unsplit 49/29/49/64 |
+
+The exported kernel-benchmark interface should be a prompt-free weighted table
+with `(P,D,N,context_bin,chunk_class,chunk_size_bin,runtime_mode,count)` plus
+step duration and useful tokens. Use the frozen histogram edges already emitted
+by Layer 1: context `128..131072` powers of two and chunk `16..2048` powers of
+two. Preserve the joint tuples; independent marginal sampling would erase the
+mixed-batch structure.
+
+### Kernel targets and acceptance
+
+1. **Ragged MoE GEMM:** accept variable token counts without padding every
+   expert/layer tile to the largest sequence. Weight microbenchmarks by the P06,
+   P09, and P10 tuples above. The ceiling is the same R64/efficiency opportunity
+   as B1, not an additional gain.
+2. **Attention:** retain P04 `(D,N)=(8,8)` as the valid long rectangular control
+   and test P10's mostly 1–4 decode batches plus 8,192-token chunks. Report
+   useful-token time, workspace, and graph compatibility.
+3. **Serving confirmation:** kernel time must improve on the exact weighted
+   stream, then pass a fixed-arrival serving A/B for throughput, TTFT/p95,
+   memory, and correctness. A rectangular-only kernel win does not close the
+   Phase-3 finding.
+
+`moe_expert_load` was unavailable in Phase 3. No expert-imbalance mechanism or
+gain is claimed; expert-specific packing requires a new low-overhead route
+histogram before implementation.
+
+## Honest limits and Phase-5 measurement requirement
+
+- H1a remains inconclusive: only P04 had two representative/recovered operator
+  windows. Eight other completed moderate patterns failed window validity even
+  though kernel classifiability was 97.05–99.64%.
+- A Phase 5 operator study needs longer, time-stratified samples that reproduce
+  clean scheduled-token, prefill-fraction, decode-batch, and graph-mode
+  distributions, plus a lower-perturbation per-op timer. It must demonstrate
+  overhead before using shares for optimization; the Phase-2 Kineto active
+  window perturbed throughput by 51.3%.
+- Confirmation runs are absent. The MNS and MBT config effects are single-run
+  point estimates and require replication before production decisions.
+- R64 is an offline rectangular-padding upper bound, not measured GPU idle
+  time. H1b's efficiency association does not establish causality.
+- Mixed-batch interference was N/A because no cell retained 30 supported mixed
+  steps inside both leave-one-pattern-out pure-fit supports.
+- Results cover one model, BF16, H20, mostly TP1, and 20/24 cells. P03/C11,
+  P05/C00, P10/C00-TP2, and P11/C00 are absent.
+- The capture validation is one ordered ON/OFF pair. Its padding endpoint is
+  mechanism-valid, but performance deltas have no CI and p95 regressed.
+- Layer 1 did not collect expert-route identities; kernel engineers cannot infer
+  routed-expert imbalance from these artifacts.
+
+## Verification and stop rules for Phase 4 work
+
+Every proposed experiment keeps the original fixed manifest/seed/work, excludes
+warm-up, records Layer-1 accounting, and changes one mechanism. A candidate
+stops on clean failure, footer imbalance/drop, output mismatch, GPU
+contamination, memory regression beyond its declared budget, or violation of
+the 16-H20-hour campaign cap. Throughput, latency, memory, and correctness are
+reported together; no metric shopping or silent pattern substitution is
+allowed.
+
+## GPU accounting
+
+The optional capture pair consumed **0.296389 H20-hours**, taking cumulative
+campaign use from 14.025875 to **14.322265 H20-hours**. Remaining headroom is
+**1.677735 H20-hours**. Both arms returned GPU0 to zero, all eight GPUs were
+0 MiB/0% at final inspection, and no other-user process appeared.
+
+Artifacts are under
+`runs/opprof-phase3/phase4/capture-p09/`; `result.json` SHA-256 is
+`5bb91df28790f6f3c34e4e9ed8e35a1cb8100f93086a4286689d587fd732f2a4`.
+
+## Final ranked one-liners
+
+1. **Prefix affinity (config now):** P08's measured ceiling is **+82.14% req/s** with 62.29% fewer prefill tokens versus matched P07.
+2. **Length-aware scheduler:** raggedness ceiling is **44.79 pp R64 / 44.69% efficiency gap** on P10; smaller confirmed bounds apply to P09/P06.
+3. **Exact capture sizes (config now):** ceiling **5.26 pp P10 / 4.98 pp P09 padding**; P09 validation removed 4.980 pp but did not prove p95 gain.
+4. **MNS64 pattern pools (config now):** measured ceiling **+3.70% req/s P10 / +3.37% P06**, with a −24.27% P01 counterexample.
+5. **Ragged kernels (kernel backlog):** share rank 2's bound; no additive E2E bound is supported while H1a is inconclusive.
+6. **MBT8192 guardrail (config now):** avoids measured regressions up to **11.64%**; MBT2048 has no general positive case.
+
+## Data sanity block
+
+| Numeric family | n | finite | missing | min | max | distinct | Invariant/result |
+|---|---:|---:|---:|---:|---:|---:|---|
+| Ranked items | 6 | 6 | 0 | rank 1 | rank 6 | 6 | Three tiers represented; bounds not summed |
+| Sentinel saturation config deltas | 11 | 11 | 0 | −30.216% | +3.704% | 11 | Both gains and regressions retained |
+| Passing R64 contrast effects | 5 | 5 | 0 | 0.230148 | 0.447872 | 4 | Ratios in [0,1]; duplicate P10 controls expected |
+| Capture-arm padding fraction | 2 | 2 | 0 | 0.035659 | 0.085456 | 2 | Non-negative; ON < OFF |
+| Capture-arm token efficiency | 2 | 2 | 0 | 4.554051 | 4.562222 | 2 | Positive; +0.179% ON |
+| Capture-arm throughput (req/s) | 2 | 2 | 0 | 4.933333 | 4.958333 | 2 | Same offered rate 4.920833 req/s |
+| Capture-arm clean failures | 2 | 2 | 0 | 0 | 0 | 1 expected | Exact 240 s and zero failures |
+| Capture-arm Layer-1 records | 2 | 2 | 0 | 16,491 | 17,579 | 2 | Every footer/sidecar invariant true; zero drops |
+| Optional validation GPU-hours | 1 | 1 | 0 | 0.296389 | 0.296389 | 1 | Positive; cumulative 14.322265 < 16 |
+| Final GPU memory (MiB) | 8 | 8 | 0 | 0 | 0 | 1 expected | Cleanup passed |
+
+Checked invariants: Phase-3 metrics remain frozen; every cited number resolves
+to accepted metrics or the checksum-recorded validation; config comparisons
+use saturation rather than normalized moderate throughput; padding/raggedness
+bounds are not presented as throughput; duplicate/non-independent bounds are
+not added; both validation arms use identical work and offered rate; clean
+failures are zero; output work, Layer-1 schema, step continuity, footer/sidecar
+balance, and zero drops pass; ratios lie in their declared domains; all GPU
+memory returned to zero; and cumulative GPU use stays below 16 H20-hours. No
+data-sanity red flag remains.

docs/opprof/phase5-protocol.md

@@ -0,0 +1,640 @@
+# OpProf Phase 5 pre-registered mechanism-decomposition protocol
+
+Status: **ACCEPTED FOR EXECUTION — ALL FIVE ORCHESTRATOR DECISIONS RESOLVED**.
+
+Date frozen: 2026-07-12 (Asia/Singapore). This document specifies Phase 5 only.
+It does not authorize a GPU launch, helper implementation, trace transfer, or
+change to the accepted vLLM patch series. Any change to an estimand, request
+set, service order, arrival transform, capture-size set, load, validity gate,
+or decision threshold requires a dated amendment before the affected run.
+
+## Approved dispositions (orchestrator; 2026-07-12)
+
+All five open decisions below are approved before execution. These dispositions
+are normative and close the protocol review gate without changing any frozen
+estimand, command delta, validity threshold, or budget:
+
+1. The recorded-arrival **bridge ledger** is approved. Every machine and human
+   output must state that it decomposes the recorded-arrival P5 gap anchored to
+   P3 controls, not literally P3's already-uniform P10 gap; A3 supplies the
+   explicit bridge back to P3.
+2. The dual P03/P04 control ledgers are approved. Both are always reported and
+   a dominant-mechanism call must pass the frozen rule under both denominators.
+3. A1 is approved exactly as specified: 142 requests, 32-request reorder
+   blocks, 16-request analysis cohorts, frozen bins, and a 64-second fairness
+   cap.
+4. Three P10 replicates per arm, P3 control reuse behind the 3% bridge gate,
+   the conditional control reruns, and optional-tier ordering under the 6.0
+   H20-hour hard cap are approved.
+5. Layer-1-only primary measurement is approved. Routed-expert telemetry is
+   analysis-only, private, optional, and never receives a causal ledger share.
+
+This approval authorizes the later execution turn only when its preflight,
+echo-before-launch, detached-controller, long-context, privacy, accounting,
+cleanup, and budget gates all pass.
+
+## Amendment A-P5-1 — rate-following cold-start gate (orchestrator; 2026-07-12)
+
+The first Phase-5 wave correctly hard-stopped because all four offered-load
+arms failed the inherited A-P3-6 throughput-drift gate: recorded base/A4 drift
+was approximately 216.5%, A1 was 190.6%, and uniform A3 was 13.23%, versus the
+frozen 10% limit. Their 240-second clean windows, output work, offered rates,
+Layer-1 accounting, and drains otherwise passed. The gate was semantically
+wrong for these arms: a rate-following run's scheduled-token throughput follows
+its arrival process by design, so recorded non-stationarity is treatment signal,
+not cold-start contamination. The large recorded-versus-uniform drift gap is
+retained as direct arrival-mechanism evidence. Throughput-drift stationarity
+remains appropriate and unchanged for saturation arms.
+
+For every **rate-following/offered-load** arm, A-P3-6 is replaced by all three
+of the following cold-start-artifact gates:
+
+1. Every logged torch.compile or CUDA-graph capture first-occurrence event must
+   precede the clean boundary. Match server-log event messages containing
+   `torch.compile took`, `Directly load AOT compilation`, `Compiling`, or
+   `Capturing CUDA graphs` (case-insensitive). Timestamped events are compared
+   against `t0_wall_ns + 60 s`. vLLM's capture progress lines have no timestamp;
+   they count as pre-client only when their log-line order precedes the server
+   ready/startup-complete marker, which itself precedes client `t0`. Any matching
+   event after readiness must have a parseable timestamp and precede clean;
+   otherwise the run is invalid.
+2. At least 16 requests must complete successfully in `[0,60 s)`, including at
+   least one request whose recorded `input_tokens >= 8192`.
+3. No first-occurrence capture event may appear inside `[60,300 s)`. Parse the
+   configured startup capture-size set and the completed FULL/PIECEWISE startup
+   capture passes from the server log. From Layer 1, define a captured replay
+   descriptor as `(runtime_mode,bucket_tokens)` for every model-executed
+   `cudagraph.hit=true` step. Every clean descriptor must be covered by the
+   startup-captured mode and bucket set, and no server-log compile/capture event
+   may occur inside clean. Warm and clean descriptor sets are both reported;
+   a descriptor's first **replay** in clean is not mislabeled as a first capture
+   when startup logs prove it was already captured. An uncovered descriptor or
+   clean-window capture/compile event invalidates that run only.
+
+The report records matched server-log events, warm-up completion/long-request
+counts, warm-up and clean descriptor sets, and clean-only descriptors per run.
+Absence of any required log timestamp, Layer-1 interval, or request record is a
+gate failure. The original 10% A-P3-6 drift criterion remains mandatory for
+closed-loop saturation arms. This amendment changes no request set, arrival
+transform, service order, server configuration, clean interval, metric,
+bootstrap, share estimator, dominance rule, control-reuse gate, or GPU budget.
+
+## Goal, system boundary, and success criterion
+
+Phase 3 measured a total useful-token-efficiency gap between irregular patterns
+and rectangular controls but did not causally allocate it. Phase 5 asks how much
+of the P10 gap is recovered when one treatment at a time removes:
+
+1. intra-cohort input-length raggedness;
+2. CUDA-graph decode-batch capture-bucket mismatch;
+3. recorded arrival burstiness; or
+4. usable natural-prefix structure.
+
+The primary system remains Qwen3-30B-A3B BF16, patched vLLM 0.24.0, C00, TP1,
+one H20 per server on dash0. The primary load is the P3 P10 rate
+`lambda = 0.60 * 0.7875 = 0.4725 request/s`; it is held fixed across arms so an
+ablation changes one treatment, not offered demand. The 240-second clean window
+and Layer-1 definition of
+
+```text
+E_token = sum(prefill_tokens + decode_tokens) / sum(model-step duration_ms)
+```
+
+are inherited unchanged. Layer 2 is not needed for the causal ledger and is not
+enabled in primary throughput runs.
+
+Success is a mechanism ledger with an absolute `E_token` for every arm, an
+un-normalized share and bootstrap confidence interval for every mechanism, and
+an explicit residual/interaction line. A null or negative share is a valid
+result. Merely recovering the expected sign is not success.
+
+## Pinned Phase-3 evidence and the arrival-estimand discrepancy
+
+The frozen P3 C00-TP1 moderate values are:
+
+| Cell | `E_token` (tokens/ms) | Role |
+|---|---:|---|
+| P10 | 2.6191132083 | irregular P3 base |
+| P03 | 4.7355997154 | long-input/short-output rectangular control |
+| P04 | 3.0547035940 | long-input/long-output rectangular control |
+
+P10 therefore lost 44.693% versus P03 and 14.260% versus P04. P3 used both
+controls, so Phase 5 reports two parallel ledgers, one per frozen control. It
+never selects the denominator that makes a mechanism look largest. A mechanism
+is called control-robust only when its decision agrees under both ledgers.
+
+There is one blocking provenance discrepancy. The private source contains
+`timestamp` and `source_timestamp`, but P3's materializer discarded them and
+set every P10 row to `arrival=steady`; the P3 client then admitted requests at
+exactly `1/lambda`. Thus P3 has no recorded-arrival burstiness to remove.
+
+The recommended resolution, pending orchestrator approval, is a **P5 bridge
+ledger**: the P5 base replays the same P10 requests at rate-normalized recorded
+timestamps, A3 uniformizes those timestamps, and P03/P04 remain the frozen P3
+controls. A3 also acts as a bridge back to the P3 steady workload. This meets
+the requested arrival ablation but decomposes a recorded-arrival P5 gap, not
+literally the already-uniform P3 gap. The report must show both
+`E_A3 - E_P3_P10` and its CI before relating the P5 ledger to P3.
+
+If the orchestrator rejects this rebase, the P3-exact alternative is mandatory:
+A3's share is `0 / N/A by construction`, and recorded arrival is reported only
+as a stress sensitivity, not as a mechanism share. It is scientifically invalid
+to call the reverse, burstiness-injecting treatment an ablation that removes
+arrival dynamics. No GPU work may begin before this decision is recorded.
+
+## Common request set and exact arrival transforms
+
+The primary request set is the first **142** rows of the frozen 4,011-row P10
+selection in original source order. This is exactly the number of admissions at
+`lambda=0.4725` over `[0,300 s)`: scheduled times are
+`0, 1/lambda, ..., 141/lambda`. All five arms contain the same 142 request IDs,
+prompts, per-request input/output lengths, and aggregate input/output token
+totals. There is no wrap, replacement, cancellation, or resampling.
+
+The Phase-5 materializer must preserve `timestamp` as private metadata. Let
+`z_i` be its stable source-order timestamp for request `i`, and let `N=142`.
+The two arrival vectors are frozen as:
+
+```text
+recorded-scaled: a_i = (z_i-z_0) * ((N-1) / (lambda*(z_(N-1)-z_0)))
+uniformized:     a_i = i / lambda
+```
+
+`z_(N-1)` must exceed `z_0`; timestamps must be finite and nondecreasing.
+Ties remain ties. Both vectors start at zero, end at `141/lambda`, have the same
+mean rate, and use the same request order except in A1. The client schedules
+against `a_i` directly and does not add jitter. This preserves the recorded
+inter-arrival shape while preventing mean-rate differences from masquerading as
+an arrival mechanism.
+
+The protocol requires a small `scripts/opprof_phase5_client.py` extension in a
+later, no-GPU implementation turn. Before execution, CPU-only tests must prove:
+
+- exact 142-row identity and token sums across all manifests;
+- timestamp normalization endpoints and nonnegative gaps;
+- uniform gaps equal `1/0.4725` within 1 microsecond;
+- the A1 fairness bound and deterministic ordering;
+- fixed 60+240-second timing, no wrap, exact output work, and text redaction.
+
+Its reviewed SHA-256 and all manifest SHA-256 values are frozen in the detached
+controller before GPU use.
+
+## Falsifiable mechanism estimands
+
+For arm `m` and rectangular control `c in {P03,P04}`:
+
+```text
+gap_c       = E_control,c - E_base
+delta_m     = E_ablated,m - E_base
+share_m,c   = delta_m / gap_c
+```
+
+The same base and same control are used for all four mechanisms within a
+ledger. No share is clipped to `[0,1]`. Shares may be negative, exceed one, or
+sum above/below one because single-factor interventions can overlap or interact.
+
+The arithmetic residual/interaction line is
+
+```text
+share_residual+interaction,c = 1 - sum_m share_m,c
+```
+
+with a joint bootstrap CI. This is bookkeeping, not proof that the remainder is
+one separable mechanism. It may contain unmeasured mechanisms, non-additivity,
+double-counting, MoE routing, chunked-prefill interference, and measurement
+error. Individual shares are never renormalized to total 100%; the residual is
+never clipped to make the table visually close.
+
+### A1 — length-binned service order
+
+**Hypothesis.** Length-homogeneous local cohorts reduce ragged-attention/SM
+imbalance, so `E_A1 > E_base`. The hypothesis is falsified if the registered
+manipulation check fails or the Holm-corrected efficiency contrast is not
+positive.
+
+Starting from consecutive **32-request reorder blocks** in original P10 order,
+assign each request to the fixed input-length bins
+
+```text
+[0,512], [513,1024], [1025,2048], [2049,4096],
+[4097,8192], [8193,16384], [16385,32768]
+```
+
+and stable-sort each block by `(bin_id, input_tokens, original_index)`. This
+creates two more homogeneous 16-request analysis cohorts per complete reorder
+block. Assign the sorted requests to the block's unchanged recorded-scaled
+arrival slots. If that assignment would delay any request by more than **64
+seconds** relative
+to its original slot, choose the earliest-deadline request first until all
+deadlines are feasible, then resume the length order. Early movement is allowed;
+late movement is capped. Ties are stable and no sorting crosses a 32-request
+block.
+
+On consecutive complete 16-request cohorts of evaluation-slice service order,
+define `R16 = 1 - sum(L_i) / sum(16*max_cohort(L_i))`; the incomplete final
+cohort is excluded and reported. Frozen pre-run values are base `R16=0.641744`
+and sorted `R16=0.473409`, a 0.168334 absolute (26.23% relative) reduction;
+plain sorting's maximum added delay is 62.744 seconds. The manipulation passes
+only if regenerated values match these within `1e-6`, `R16` falls by at least
+20% relative and 0.15 absolute, and no request violates the 64-second delay
+bound.
+Arrival-slot timestamps, request/content multiset, per-request output lengths,
+total tokens, server config, and prefix-caching setting are identical to base.
+
+A1 estimates the total effect of changing cohort composition. It does not
+claim to isolate a particular attention kernel: service order can mediate
+decode-batch composition, chunked-prefill mixing, cache locality, and
+content-bound MoE routing. If the prefix-query hit ratio changes by more than
+one percentage point, or the normalized inter-arrival vector changes at all,
+the arm is labeled confounded and has no publishable raggedness share.
+
+### A2 — measured decode-B capture sizes (config-tier deliverable)
+
+**Hypothesis.** Exact capture sizes for P10's observed pure-decode batch support
+remove decode-bucket slack, so pure-decode padding falls and `E_A2 > E_base`.
+No recovery falsifies the efficiency hypothesis; failure to remove the targeted
+padding invalidates the ablation rather than supporting a null mechanism.
+
+The P3 P10/C00/rho=0.60 clean Layer-1 stream has SHA-256
+`51ad4be12178da91d2af484d0946a2274afd3bcbbee33f37940cfe0ff2ea7fa7`.
+Its 17,941 pure-decode steps have the exact decode-B histogram:
+
+| B | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
+|---:|---:|---:|---:|---:|---:|---:|---:|
+| Steps | 13,572 | 2,675 | 792 | 524 | 161 | 202 | 15 |
+
+Defaults already contain 1, 2, 4, and 8. A2 therefore adds exactly
+`{3,5,6,7}` and freezes the complete server list as:
+
+```text
+1 2 3 4 5 6 7 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128
+136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256
+272 288 304 320 336 352 368 384 400 416 432 448 464 480 496 512
+```
+
+This covers 100% of P3's observed P10 pure-decode B support. The A2
+manipulation requires at least 99% support coverage in the new clean runs and a
+90% reduction in pure-decode padding tokens versus base. It targets decode
+capture-bucket mismatch; it does not remove prefill-token padding, eager
+overflow, graph launch overhead, or length raggedness itself. Capture startup
+time and memory are reported as config cost, not treated as free.
+
+### A3 — recorded arrival to uniform arrival
+
+**Hypothesis.** Uniformizing the same rate and request order reduces burst-driven
+decode-batch/queue variance, so `E_A3 > E_base`. The mechanism is falsified if
+the clean 5-second decode-B CV and waiting-queue CV do not fall, or if the
+Holm-corrected efficiency contrast is not positive.
+
+A3 changes only `recorded-scaled` arrival slots to `i/lambda`. Content, order,
+input/output lengths, aggregate tokens, server config, prefix caching, and
+capture sizes are unchanged. It estimates the total service effect of arrival
+shape, including its legitimate downstream changes to batching and queueing.
+It does not isolate a scheduler instruction cost. Under the P3-exact fallback,
+this arm is the base-equivalent bridge and its arrival share is N/A as described
+above.
+
+### A4 — natural prefix caching disabled
+
+**Hypothesis.** Direction is deliberately two-sided. Natural P10 reuse may
+increase `E_token` through KV reuse, in which case disabling it gives a negative
+share; alternatively, low-value/fragmented cache structure may impose overhead
+or alter batching, giving a positive share. Either direction is publishable.
+
+A4 omits only `--enable-prefix-caching`. Prompts, natural repeated-prefix
+structure, recorded arrival slots, request order, token totals, scheduler limits,
+and capture sizes remain identical to base. Required manipulation checks are
+zero local prefix cache hits/queries in the disabled arm and unchanged prompt
+hashes. This estimates the contribution of exploiting C structure, not the
+intrinsic content similarity of the prompts.
+
+### Mechanisms without a clean ablation
+
+MoE routing skew cannot be removed while preserving P10 content and model
+semantics: changing tokens, router weights, top-k, or expert placement changes
+more than routing skew. It receives no causal share.
+
+If budget remains, one **analysis-only**, separately started P10 sample may add
+`--enable-return-routed-experts`. It is excluded from every `E_token` clean
+window and ledger numerator. Offline analysis reports per-layer expert-count
+entropy, Gini coefficient, coefficient of variation, max/mean load, and their
+association with same-step token-normalized duration. The arrays remain private.
+This telemetry is sampled and perturbing, can consume a large scheduler-side
+buffer, and provides correlation rather than causal attribution; it can only
+help interpret the residual/interaction line. Phase 3's MoE layer-duration CV
+was N/A, so it cannot substitute for these routed-expert counts.
+
+## Exact commands and config deltas
+
+The following interface is normative for the later implementation. Variables:
+
+```bash
+P5C='python scripts/opprof_phase5_client.py'
+PRIVATE=/home/admin/cpfs/wjh/opprof-phase5-private/manifests
+P3PRIVATE=/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P10.jsonl
+P3SOURCE=/home/admin/cpfs/wjh/opprof-phase3-private/trace_windows/chat_w20260311_1000.jsonl
+MODEL=/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
+RATE=0.4725
+```
+
+Materialize the five private manifests without printing prompt text:
+
+```bash
+$P5C transform --in "$P3PRIVATE" --take-first 142 \
+  --timestamp-source "$P3SOURCE" --join-key source_index \
+  --timestamp-field timestamp --arrival recorded-scaled --target-rate "$RATE" \
+  --service-order original --out "$PRIVATE/P10-base.jsonl"
+$P5C transform --in "$P3PRIVATE" --take-first 142 \
+  --timestamp-source "$P3SOURCE" --join-key source_index \
+  --timestamp-field timestamp --arrival recorded-scaled --target-rate "$RATE" \
+  --service-order length-binned --reorder-block-size 32 \
+  --analysis-cohort-size 16 \
+  --length-bin-edges 512,1024,2048,4096,8192,16384,32768 \
+  --max-added-delay-seconds 64 --out "$PRIVATE/P10-A1.jsonl"
+$P5C transform --in "$P3PRIVATE" --take-first 142 \
+  --timestamp-source "$P3SOURCE" --join-key source_index \
+  --timestamp-field timestamp --arrival uniform --target-rate "$RATE" \
+  --service-order original --out "$PRIVATE/P10-A3.jsonl"
+ln -s P10-base.jsonl "$PRIVATE/P10-A2.jsonl"
+ln -s P10-base.jsonl "$PRIVATE/P10-A4.jsonl"
+```
+
+The symlinks make unchanged request bytes explicit; the controller hashes the
+resolved content and requires A2/A4 hashes to equal base. A1/A3 require equal
+sorted request-ID sets and equal input/output token sums.
+
+Common server command (`ARM` is `base`, `A1`, `A2`, `A3`, or `A4`):
+
+```bash
+taskset -c "$CPUSET" env CUDA_VISIBLE_DEVICES="$GPU" \
+  VLLM_OPPROF_DIR="$RUN_DIR/opprof" \
+  vllm serve "$MODEL" --host 127.0.0.1 --port "$PORT" \
+  --tensor-parallel-size 1 --enable-chunked-prefill \
+  --enable-prefix-caching --shutdown-timeout 600
+```
+
+- A2 adds
+  `--cudagraph-capture-sizes 1 2 3 4 5 6 7 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256 272 288 304 320 336 352 368 384 400 416 432 448 464 480 496 512`.
+- A4 removes `--enable-prefix-caching`.
+- Base, A1, and A3 have no server delta.
+
+Every moderate client command is:
+
+```bash
+taskset -c "$CPUSET" $P5C run \
+  --manifest "$PRIVATE/P10-$ARM.jsonl" \
+  --base-url "http://127.0.0.1:$PORT" --model "$MODEL" \
+  --load-point moderate --fixed-request-rate "$RATE" \
+  --max-concurrency 256 --ignore-eos --temperature 0 \
+  --warmup-seconds 60 --clean-segment-seconds 80 --num-clean-segments 3 \
+  --post-clean-seconds 0 --drain-timeout-seconds 600 \
+  --workload-seed 20260712 --server-seed 20260712 \
+  --result-dir "$RUN_DIR/client"
+```
+
+There are no profiler endpoint calls. Exact commands, effective config, startup
+capture list, compile-cache key, hashes, clocks, host load, and GPU process list
+are recorded per run.
+
+## Execution and validity discipline
+
+### Placement, order, and detached ownership
+
+A-P3-1 already rejected eight-way and authorized four-way placement. Phase 5
+therefore uses at most four simultaneous TP1 servers, GPU0--GPU3, with the
+frozen disjoint CPU masks `0-19`, `20-39`, `40-59`, and `60-79`. Topology must
+still match P3. A changed host topology, source/patch/runtime hash, or clock
+policy invalidates reuse of the placement gate and stops for review.
+
+Three independent replicates are run for each of the five primary arms (15
+measured runs). Sort all `(replicate,arm)` assignments by SHA-256 of
+`"20260715:<replicate>:<arm>"`, pack them into waves of 4, 4, 4, and 3, and
+rotate GPU assignments. The final three-target wave uses the frozen P06/C00
+saturation background in the fourth slot so measurements retain the validated
+four-way host regime. No wave contains two replicates of the same arm; if the
+hash order would do so, stable-swap the later item with the first legal item and
+record the resolved order before launch. Background data are not analyzed.
+
+As required by A-P3-2, a dash0-resident `setsid`/`nohup` controller owns every
+process, has `--resume`, atomically records state, and skips only fully validated
+runs with matching hashes. Interactive SSH never owns a server. Before each
+wave it logs one echo line with resolved arms, GPUs/CPUs, manifests, rate,
+paths, reserved H20-hours, expected duration, and disk headroom.
+
+Use three unmeasured 60-second burn-ins before measured arms: common C00,
+A2 capture-list C00, and prefix-cache-off C00. They warm compile/AOT artifacts
+but are not measurements. Per-run-unique OpProf directories must remain ignored
+by vLLM compile factors, preserving the accepted Phase-2 fix.
+
+### Long-context-safe warm-up and drain gates
+
+These gates are active from the first run; no short-context default is tried
+first.
+
+- Warm-up is exactly 60 seconds and excluded. P10 passes with at least 32
+  successful warm-up completions, **or** at least 16 completions plus the exact
+  A-P3-6 stabilization test: model-executed steps in `[45,50)`, `[50,55)`, and
+  `[55,60)`; at least 16 steps/bin; positive scheduled-token rates `R_j`; and
+  OLS drift `abs(slope)*15/mean(R_j) <= 0.10`. Missing bins, discontinuity, or
+  accounting failure invalidates the run.
+- Drain timeout is 600 seconds. A timeout marks the run drain-quarantined but
+  does not invalidate an otherwise valid clean window. More than 20% of primary
+  runs quarantined stops Phase 5 for review.
+- The clean window is exactly three contiguous 80-second segments. Admission
+  and completion accounting follows P3. Offered rate must be within 5% of
+  0.4725 request/s, with zero clean failures and exact output tokens.
+- Layer-1 JSONL/footer/sidecar accounting, zero drops, contiguous steps, no
+  profile leakage, clock/load capture, other-user-process absence, and final
+  zero GPU memory are hard gates inherited from P3.
+
+No semantic failure is retried with altered parameters. One exact retry is
+allowed for an infrastructure artifact failure, and both attempts remain in
+the operational findings.
+
+## Control reuse, secondary scope, and budget
+
+P03/P04 P3 controls may be reused because they already use the same model,
+patch, C00-TP1 config, normalized load, 240-second clean metric, placement
+regime, and Layer-1 schema. Reuse avoids spending GPU time without changing the
+denominator.
+
+Reuse is valid only if source, patch, model, runtime, clocks, and placement
+hashes match and fresh A3-uniformized P10 differs from frozen P3 P10 by at most
+3% in `E_token` with a bootstrap CI containing zero difference. If this bridge
+gate fails, temporal/runtime drift is plausible: rerun P03 and P04 under their
+exact P3 manifests and commands, three replicates each, before computing any
+share. A failed bridge never licenses rescaling old controls.
+
+After the complete P10 ledger, optional work is ordered as follows and starts
+only if the controller's conservative reservation remains below the 6.0
+H20-hour hard cap:
+
+1. one saturation run per P10 arm, using the full 4,011-row manifest, for
+   descriptive mechanism persistence only;
+2. one five-arm moderate ledger each for P09 and P06, reported as exploratory
+   within-run-bootstrap evidence, not equal replication to P10; and
+3. one routed-expert analysis-only sample.
+
+For secondary A2, the precomputed P3 pure-decode support additions are P09
+`{3,5,6,7,9,10,11,12,13,14,15,17,18,19,20,21,22,23,25}` and P06
+`{3,9,10,11,12,13,14,15}`; default sizes are retained. P09 A3 is a no-change
+steady negative control, while P06 A3 changes its registered burst-8 arrivals
+to uniform spacing at the same mean rate. P09/P06 conclusions are always
+labeled secondary.
+
+Expected accounting, including server-owned startup/shutdown time:
+
+| Tier | New measured runs | Expected H20-hours | Expected 4-way wall |
+|---|---:|---:|---:|
+| P10: 5 arms x 3 replicates | 15 | 1.6-1.9 | 35-55 min |
+| Three compile burn-ins | 0 | 0.1-0.2 | 3-6 min |
+| Conditional P03/P04 reruns | 6 | 0.6-0.8 | 15-25 min |
+| Optional P10 saturation | 5 | 0.5-0.8 | 15-25 min |
+| Optional P09/P06, 5 arms each | 10 | 0.9-1.2 | 20-35 min |
+| Optional routed-expert sample | 1 analysis-only | 0.1-0.2 | 5-10 min |
+| **Maximum planned** | **36 ledger + 1 analysis-only** | **3.8-5.1 expected** | **about 1.5-2.5 h** |
+
+The hard cap is **6.0 H20-hours new Phase-5 spend**, not a target. Actual time
+while a server owns GPU memory is charged. Optional tiers are skipped rather
+than overrunning the cap. The 600-second drain allowance is a watchdog, not an
+assumption in the expected estimate; repeated long drains consume the optional
+budget first.
+
+With no Kineto traces, primary P10 artifacts are estimated at 0.4-0.6 GB and
+all planned public artifacts below 1.5 GB. Stop if public Phase-5 artifacts
+exceed 3 GB or CPFS free space falls below 100 GB. Prompt-bearing manifests and
+routed-expert arrays remain in mode-0700 private storage and are not counted as
+public deliverables.
+
+## Statistical analysis and decision rules
+
+Use 5-second moving-block bootstrap over clean time, 100,000 resamples, seed
+20260716. For P10, first resample the three run IDs, then resample 5-second
+blocks within each selected run; arms and reused P3 controls are resampled
+independently. Every `E`, `delta`, share, share sum, and residual/interaction
+gets a percentile 95% CI. Absolute `E` and delta accompany every ratio.
+
+Bootstrap ratio draws are never deleted because their denominator is
+inconvenient. If either control gap has a point estimate `<=0`, its CI includes
+zero, or more than 5% of bootstrap denominator draws are `<=0`, that control's
+ledger is **INCONCLUSIVE (unstable denominator)**.
+
+The confirmatory family is the four two-sided tests of `E_ablated-E_base=0` on
+P10. Apply Holm correction at family-wise alpha 0.05 across A1--A4. The same
+corrected delta test serves both control ledgers; duplicating denominators does
+not create eight tests. A1--A3 only support the expected recovery claim when
+their corrected contrast is positive. A4 may be significant in either
+direction. Manipulation-check failure makes the corresponding share N/A even
+if efficiency changes.
+
+A mechanism is **dominant** only if, under both P03 and P04 ledgers:
+
+- point `share >= 0.30`;
+- the 95% share CI excludes 0.15 on the high side (`CI_low > 0.15`); and
+- its Holm-corrected efficiency contrast is significant in the expected
+  direction (two-sided for A4).
+
+Meeting the rule under only one control is reported as **control-sensitive**,
+not dominant.
+
+The primary ledger is **publishable** when all five P10 arms have three valid
+replicates, both control denominators are stable, all four manipulation checks
+are evaluable, every share/residual has a finite CI, the fresh A3/P3 bridge is
+reported, and no privacy/data-sanity red flag exists. It may be publishable
+with no dominant mechanism. It is **inconclusive** if any primary arm is
+missing, a denominator is unstable, the arrival-rebase decision is unresolved,
+or two or more share CIs have width greater than 0.50. One failed mechanism
+manipulation yields a publishable partial ledger only if that line is explicitly
+N/A and the headline claim excludes it.
+
+No additive causal claim is made from `sum(share_m)`. Pairwise and higher-order
+interactions are not identified by this one-factor-at-a-time matrix; a combined
+all-off arm would be a new experiment requiring amendment, not an improvised
+way to close the ledger.
+
+## Deliverable and artifact contract
+
+Execution, if later approved, must produce:
+
+- `docs/opprof/phase5-results.md`;
+- `runs/opprof-phase5/phase5/metrics.json` with schema, arm/run values,
+  bootstrap draws' seed and summary, Holm results, dual-control ledgers,
+  residual/interaction, gates, and GPU accounting;
+- per-run exact commands, environment/provenance, client records, Layer-1
+  stream/footer/sidecar, monitor data, and machine-readable sanity JSON; and
+- an **Operational findings** section covering warm-up stabilization, drains,
+  compile/capture startup cost, contamination, retries, and any mismatch between
+  expected and realized mechanism removal.
+
+Every raw-run summary, aggregate table, and final metrics file ends with the P3
+sanity schema: `n`, finite/missing, min, max, distinct count, and applicable
+invariants. Red flags are reported first and stop inferential analysis. Public
+artifacts may contain request IDs, hashes, lengths, counts, and timing only;
+prompt, messages, content, generated text, source substrings, and routed-expert
+arrays are forbidden.
+
+## Final ablation table
+
+| Mechanism | What changes | What is preserved |
+|---|---|---|
+| Base | Recorded P10 timestamps are rate-normalized and replayed in source order | Frozen 142 requests, content/tokens, C00-TP1, prefix cache on, default capture list, `lambda=0.4725` |
+| A1: length raggedness | Stable length-bin sort within 32-request reorder blocks into 16-request cohorts, 64-second late cap | Requests/content/token totals, arrival-slot vector, output lengths, server config, prefix setting |
+| A2: capture mismatch | Add exact P10 decode-B sizes `{3,5,6,7}` to the full default capture list | Manifest/order/arrivals/content/tokens, scheduler limits, prefix setting |
+| A3: arrival dynamics | Recorded-scaled slots become uniform `i/0.4725` slots | Request order/content/tokens, mean rate, server/capture/prefix config |
+| A4: prefix structure | Remove `--enable-prefix-caching` | Natural prompt structure, request order/arrivals/content/tokens, capture/scheduler config |
+| Residual + interactions | No extra run; joint arithmetic remainder `1-sum(shares)` | Raw unnormalized mechanism shares; no clipping or forced 100% allocation |
+
+## Final run count and GPU estimate
+
+Primary commitment: **15 new measured P10 runs plus three unmeasured burn-ins,
+1.7-2.1 expected H20-hours, 35-60 minutes four-way wall, and 0.4-0.6 GB public
+disk**. Conditional control reruns add 6 runs; all optional tiers bring the
+maximum to **36 ledger runs plus one analysis-only run, 3.8-5.1 expected
+H20-hours, about 1.5-2.5 hours wall, and less than 1.5 GB public disk**. New
+Phase-5 GPU use stops at **6.0 H20-hours** under all circumstances.
+
+## Resolved decisions from the orchestrator
+
+1. **Approved:** use the recommended recorded-arrival P5 bridge ledger, with
+   its explicit limitation that it is not a literal decomposition of P3's
+   already-uniform P10 gap; otherwise select the P3-exact A3=N/A fallback.
+2. **Approved:** dual P03/P04 control ledgers and the requirement that “dominant” hold
+   under both, rather than selecting one P3 denominator.
+3. **Approved:** the 142-request slice, 32-request reorder blocks, 16-request analysis
+   cohorts, fixed bins, and 64-second fairness cap as A1's isolation/latency
+   tradeoff.
+4. **Approved:** three P10 replicates, reuse of P3 controls behind the 3% bridge gate,
+   and the optional-tier order within the 6.0-H20-hour cap.
+5. **Approved:** Layer-1-only primary runs and treating routed-expert telemetry as
+   private analysis-only evidence with no causal share.
+
+## Protocol sanity block
+
+| Numeric family | n | Min | Max | Distinct | Checked invariant/result |
+|---|---:|---:|---:|---:|---|
+| P3 control/base `E_token` | 3 | 2.619113 | 4.735600 | 3 | Finite, positive, not identical |
+| P3 P10 control gaps | 2 | 0.435590 | 2.116487 | 2 | Positive; dual denominators retained |
+| P10 request rows/arm | 5 arms | 142 | 142 | 1 expected | Same IDs and input/output token sums required |
+| Selected source timestamps (s) | 142 | 0.014 | 21.445 | 142 | Finite, nondecreasing; gap min/max 0.002/0.851 s, 127 distinct gaps |
+| Primary offered rate (req/s) | 5 arms | 0.4725 | 0.4725 | 1 expected | Positive; achieved rate must be within 5% |
+| Warm-up / clean / drain gates (s) | 3 values | 60 | 600 | 3 | Clean is exactly `3*80=240`; long-context drain is 600 |
+| A1 input-length bins | 7 | 0 | 32768 | 7 intervals | Ordered, contiguous, cover frozen P10 range |
+| A1 frozen `R16` values | 2 | 0.473409 | 0.641744 | 2 | Sorted is lower by 0.168334 absolute / 26.23% relative; not identical |
+| A1 added-delay values (s) | 142 | 0 | 62.744 | >1 expected | Non-negative and all below 64-second cap |
+| P3 P10 pure-decode steps | 17,941 | B=1 | B=7 | 7 B values | Counts sum to 17,941; non-negative; stream SHA pinned |
+| P10 A2 added capture sizes | 4 | 3 | 7 | 4 | Exactly missing observed support `{3,5,6,7}`; 100% P3 support covered |
+| Primary measured runs | 15 | 3/arm | 3/arm | 1 expected | `5 arms * 3 replicates`; no arm omitted |
+| Maximum GPU analysis/ledger runs | 1 plan | 37 | 37 | 1 expected | `15+6+5+10+1=37`; burn-ins excluded |
+| Expected H20-hours | 1 plan | 3.8 | 5.1 | 2 bounds | Finite, non-negative, below hard cap 6.0 |
+| Share domain | 5 ledger lines/control | Unbounded | Unbounded | N/A | No clipping; ratios may be negative or >1 |
+| Bootstrap resamples | 1 setting | 100,000 | 100,000 | 1 expected | Seed 20260716; no denominator-draw deletion |
+| Phase-5 GPU runs in this protocol turn | 1 turn | 0 | 0 | 1 expected | Protocol-only requirement satisfied |
+
+Checked invariants: `0.60*0.7875=0.4725`; P3 control gaps are positive;
+the seven decode-B counts sum to 17,941; default plus `{3,5,6,7}` covers all
+observed P10 pure-decode B values; `5*3=15` primary runs; maximum planned count
+is `15+6+5+10+1=37`; expected GPU use remains below the 6.0-hour hard cap;
+ratios are not constrained to `[0,1]`; expected constants are labeled; and no
+GPU command, helper change, manifest transform, or experiment was executed in
+this protocol-only turn. The unresolved P3-arrival/P5-arrival estimand mismatch
+is reported first as a blocking decision rather than hidden in the ledger.

docs/opprof/phase5-results.md

@@ -0,0 +1,223 @@
+# OpProf Phase 5 dash0 results
+
+Status: **FINAL — INCONCLUSIVE MECHANISM LEDGER; NO DOMINANCE CALL**.
+
+Date: 2026-07-12. All 15 registered P10 primary runs completed with three
+replicates per arm and passed A-P5-1. The fresh A3 bridge passed, so the frozen
+P3 P03/P04 controls were reused. The ledger is nevertheless inconclusive:
+P04's reused-control denominator is unstable, and A2, A3, and A4 fail their
+predeclared manipulation checks. The protocol therefore permits one official
+share (A1 under P03), requires the other lines to be N/A, and forbids a
+dual-control dominance conclusion.
+
+The machine result is `runs/opprof-phase5/phase5/metrics.json` (SHA-256
+`fe16934866806ac542f7974cac061beeb1f1c58f5bfb36698efd8aeaa99b8cb7`).
+The final analyzer SHA-256 is
+`8b0ff8658614227ee00cf8d10c82c4b5edf6f6210db172fd6efc9cc149e51e10`;
+its no-GPU tools and analysis tests pass.
+
+This remains explicitly a **recorded-arrival P5 bridge ledger anchored to P3
+controls**, not a literal decomposition of P3's already-uniform P10 gap.
+
+## Data-sanity result first
+
+**RED FLAG: the P04 control denominator is unstable.** P5 recorded-arrival
+base has `E_token=3.013752`, while reused P3 P04 has `E_token=3.054704`, leaving
+only a `0.040952` gap with bootstrap 95% CI `[-0.649096, 0.752594]`; 45.353% of
+denominator draws are nonpositive. This violates every registered stability
+form: the CI contains zero and the nonpositive fraction exceeds 5%.
+
+P03 remains stable: its gap is `1.721848`, CI `[1.459092, 1.988772]`, with zero
+nonpositive draws. All execution-validity checks pass: 15/15 primary runs,
+three replicates per arm, exact 240-second clean windows, zero clean failures,
+offered rates within 5%, continuous/balanced Layer-1 accounting, identical
+request IDs and token sums, all A-P5-1 cold-start gates, no drain quarantine,
+and GPU spend below the cap. The red flag is reported before the ledger and no
+conclusion is built on P04's raw ratio draws.
+
+## Bridge and absolute efficiency
+
+A3 reproduces the P3 uniform-arrival P10 base: `delta_E=+0.007041`, 95% CI
+`[-0.329799, 0.354859]`, or 0.269% absolute relative difference. The CI contains
+zero and the point difference is below 3%, so the bridge passes and P3 control
+reuse is valid under the frozen rule. Conditional control reruns were not
+launched.
+
+| Arm | `E_token` | Bootstrap 95% CI | Delta from P5 base |
+|---|---:|---:|---:|
+| Base, recorded | 3.013752 | [2.754462, 3.267964] | — |
+| A1 length-binned | 3.078999 | [2.801659, 3.349307] | +0.065247 |
+| A2 capture sizes | 3.062128 | [2.802812, 3.314276] | +0.048376 |
+| A3 uniform arrival | 2.626154 | [2.451852, 2.792684] | -0.387598 |
+| A4 prefix cache off | 3.020959 | [2.760494, 3.274105] | +0.007207 |
+
+## Manipulation checks and Holm family
+
+| Arm | Manipulation result | Efficiency delta 95% CI | Raw p | Holm p |
+|---|---|---:|---:|---:|
+| A1 | **PASS:** R16 0.641744 -> 0.473409; max delay 62.743 s; prefix-hit-ratio delta -0.296 pp | [-0.309461, 0.442565] | 0.73108 | 1.00000 |
+| A2 | **FAIL:** support coverage 98.879% <99%; padding reduction 82.541% <90% | [-0.315361, 0.411446] | 0.79022 | 1.00000 |
+| A3 | **FAIL:** decode-B CV falls 0.8706 -> 0.6551, but waiting CV rises 6.0145 -> 6.8557 | [-0.693444, -0.078141] | 0.01384 | 0.05536 |
+| A4 | **FAIL:** disabled arm still reports 3,093,324 local prefix queries and 52,896 hits, not zero | [-0.357637, 0.369309] | 0.97230 | 1.00000 |
+
+Per protocol, a failed manipulation makes that mechanism's official share N/A
+even if its raw efficiency contrast is nonzero. A3's uncorrected contrast is
+negative, and it also misses family-wise significance after Holm correction.
+
+## A-P5-1 arrival evidence
+
+The four originally invalidated arms supplied the evidence for A-P5-1. Under
+the superseded throughput-drift calculation, recorded base/A1/A4 span
+190.55%-216.50% normalized drift, while uniform A3 is 13.23%. This gap is
+retained as direct evidence that the old gate measured arrival shape in a
+rate-following experiment; it is not converted into an efficiency share.
+
+All 15 replacement/continued primary runs pass the amended cold-start gate:
+25-28 warm-up completions, 12-16 completions with input at least 8192 tokens,
+all compile/capture first occurrences before clean, zero clean-window
+first-capture events, and zero uncovered Layer-1 graph descriptors. The 10%
+drift gate was retained unchanged for the P06 saturation background arm.
+
+## Operational findings
+
+- The detached controller completed three cache burn-ins, four four-way waves,
+  all drains, validation, budget accounting, and cleanup. The four invalidated
+  first-wave arms were rerun exactly; none of their superseded measurements
+  enters the ledger.
+- The first exact rerun produced complete clean artifacts but hit a post-hoc
+  validator `NameError` from a missing `math` import. Immutable artifacts were
+  CPU-re-adjudicated under A-P5-1, the import was repaired, and no third GPU run
+  was charged.
+- CPU analysis repairs admitted legitimate idle fixed-time blocks as `[0,0]`,
+  supported the older P3 control schema, converted one NumPy boolean at the
+  JSON boundary, and explicitly separated diagnostic from reportable shares.
+  These changes alter no GPU data, estimator, seed, threshold, or resample
+  count.
+- P10 drains range from 0.676 to 6.004 seconds. The separate P06 saturation
+  background run drained naturally in 87.903 seconds, below the 600-second
+  gate, with 564/564 clean completions and zero failures.
+- Public Phase-5 artifacts occupy 286,205,365 bytes (304 MiB), below the 3 GB
+  stop threshold; CPFS retained 1,286 GB free.
+- MoE routing skew remains content-bound and receives no causal share. The
+  optional routed-expert analysis-only run was not required for this primary
+  ledger and was not launched.
+
+## Launch echo
+
+```text
+LAUNCH_ECHO utc=2026-07-12T12:07:54Z host=dash0 gpus=0-3 cpus=0-79 source=/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0@4b253fd manifests=/home/admin/cpfs/wjh/opprof-phase5-private/manifests outputs=/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase5 runs=3burnin+15primary+1background rate=0.4725 warmup=60s clean=240s drain=600s est_wall=35-60min est_gpu=1.7-2.1_H20h hard_cap=6.0_H20h conditional_controls=6_if_bridge_fails
+```
+
+Resume after A-P5-1 reused the completed burn-ins and compile/cudagraph caches.
+The first launch through final controller completion took approximately 57.9
+minutes wall time, including the adjudication/resume interval.
+
+## Run completion stats
+
+| Item | Count/result |
+|---|---:|
+| Valid primary P10 runs | **15/15** |
+| Replicates | **3 per arm** |
+| Valid P06 background runs | 1 |
+| Completed burn-ins | 3 |
+| Superseded pre-amendment arms | 4 |
+| Conditional fresh controls | 0; bridge passed, P3 reused |
+| Optional secondary/routed-expert runs | 0 |
+| A-P5-1 primary passes | 15/15 |
+| Clean failures | 0 |
+| Drain quarantines | 0 |
+
+## Mechanism ledger
+
+Official shares follow the frozen N/A rules. Values in parentheses are raw
+diagnostic ratios retained for audit, not causal/reportable shares.
+
+| Mechanism | P03 share (95% CI) | P04 share (95% CI) | Disposition |
+|---|---|---|---|
+| A1 length raggedness | **0.0379 [-0.2011, 0.2344]** | N/A; unstable denominator (diagnostic 1.5933 [-6.8925, 7.0636]) | Manipulation passes; no significant recovery |
+| A2 capture mismatch | N/A; manipulation failed (diagnostic 0.0281 [-0.2049, 0.2170]) | N/A; unstable denominator (diagnostic 1.1813 [-6.2871, 6.4924]) | Config did not meet 99%/90% removal gates |
+| A3 arrival dynamics | N/A; manipulation failed (diagnostic -0.2251 [-0.4625, -0.0403]) | N/A; unstable denominator (diagnostic -9.4647 [-17.2450, 17.4102]) | Queue-variance gate failed; Holm p=0.05536 |
+| A4 prefix structure | N/A; manipulation failed (diagnostic 0.0042 [-0.2340, 0.1948]) | N/A; unstable denominator (diagnostic 0.1760 [-6.2590, 6.4925]) | Zero-query/hit gate failed |
+
+Shares are neither clipped nor renormalized. P04's extreme raw ratios are the
+registered consequence of dividing by a near-zero, sign-unstable denominator,
+not evidence of large mechanisms.
+
+## Dominance verdicts
+
+| Mechanism | P03 rule | P04 rule | Dual-control verdict |
+|---|---|---|---|
+| A1 | Fails share/CI/Holm thresholds | Not evaluable | **NOT EVALUABLE** |
+| A2 | Not evaluable: manipulation failed | Not evaluable | **NOT EVALUABLE** |
+| A3 | Not evaluable: manipulation failed | Not evaluable | **NOT EVALUABLE** |
+| A4 | Not evaluable: manipulation failed | Not evaluable | **NOT EVALUABLE** |
+
+No mechanism may be called dominant because the frozen rule requires both
+control denominators. This is an inconclusive verdict, not evidence that every
+mechanism is small.
+
+## Residual + interaction line
+
+The official residual is N/A under both controls because the official share
+ledger is incomplete. The unchanged arithmetic over all raw diagnostic shares
+is reported only for transparency:
+
+| Control | Diagnostic share sum | Diagnostic residual + interaction (95% CI) | Status |
+|---|---:|---:|---|
+| P03 | -0.1549 | 1.1549 [0.5536, 1.9397] | Diagnostic only |
+| P04 | -6.5142 | 7.5142 [-21.3965, 22.8767] | Diagnostic only; unstable denominator |
+
+Nothing is forced to 100%. The residual can include interactions, unablated
+MoE routing, chunked-prefill effects, double-counting, and measurement error.
+
+## Config-tier A2 measured delta
+
+A2 added the P3-derived sizes `{3,5,6,7}` to the full default capture list.
+Its measured `delta_E` is **+0.048376**, 95% CI
+`[-0.315361, 0.411446]`, or **+1.605%** relative to base. Pure-decode padding
+falls from 13.7405% to 2.3989%, an 82.541% reduction. New recorded-arrival
+runs expose decode-B support 1-13, so uncaptured 9-13 leave coverage at 98.879%;
+both this and the sub-90% padding reduction invalidate the causal A2 share.
+
+The config cost is reproducible: base graph capture is 11 seconds and 0.80 GiB
+in all three runs; A2 is 12-14 seconds and 0.84 GiB, a 1-3 second and 0.04 GiB
+increase. The measured config-tier result is therefore a small, statistically
+uncertain efficiency gain with a failed preregistered removal gate.
+
+## GPU total
+
+New Phase-5 spend is **2.4388167443 H20-hours** of the 6.0-hour cap, including
+the superseded first wave, exact reruns, continued primary matrix, burn-ins,
+and background arm. **3.5611832557 H20-hours remain unspent.** Final state on
+all eight H20s is 0 MiB and 0% utilization with zero compute processes.
+
+## Sanity block
+
+**Anomaly first:** P04's gap CI contains zero and 45.353% of its denominator
+draws are nonpositive; the dual-control ledger is inconclusive and inferential
+claims stop there.
+
+| Numeric family | n | Finite | Missing | Min | Max | Distinct | Checked invariant/result |
+|---|---:|---:|---:|---:|---:|---:|---|
+| Primary run `E_token` | 15 | 15 | 0 | 2.623534 | 3.083263 | 15 | Positive; per-arm results not all identical |
+| Clean Layer-1 steps | 15 | 15 | 0 | 11,178 | 18,400 | 15 | Non-negative; continuous/balanced per run |
+| Clean duration (s) | 15 | 15 | 0 | 240 | 240 | 1 expected | Exact `3*80`; zero clean failures |
+| Offered rate (req/s) | 15 | 15 | 0 | 0.470833 | 0.487500 | 2 | Every run within 5% of 0.4725 |
+| Warm-up completions | 15 | 15 | 0 | 25 | 28 | 2 | Every run >=16 |
+| Warm-up long completions | 15 | 15 | 0 | 12 | 16 | 3 | Every run >=1 with input >=8192 |
+| Clean first-capture events | 15 | 15 | 0 | 0 | 0 | 1 expected | Zero events and zero uncovered descriptors |
+| P10 drain duration (s) | 15 | 15 | 0 | 0.676181 | 6.003959 | 15 | Non-negative; all below 600 |
+| Control gap `E_token` | 2 | 2 | 0 | 0.040952 | 1.721848 | 2 | P03 stable; **P04 unstable red flag** |
+| Raw diagnostic shares | 8 | 8 | 0 | -9.464746 | 1.593268 | 8 | Finite; intentionally not constrained to [0,1] |
+| Diagnostic residuals | 2 | 2 | 0 | 1.154931 | 7.514211 | 2 | Finite; never forced to close |
+| New H20-hours | 1 | 1 | 0 | 2.438817 | 2.438817 | 1 | Non-negative; below 6.0 |
+
+Checked invariants: source/helper/protocol provenance; identical request IDs,
+content/token totals, and prompt-preserving manifests; C00-TP1 four-way
+placement; detached controller ownership; exact clean windows; zero failures;
+offered-rate tolerance; A-P5-1 compile/capture ordering and coverage; Layer-1
+footer/sidecar balance, continuous step indices, token composition, and zero
+drops; non-negative counters; drain gates; bridge decision; no public prompt or
+generated-text leakage; public-disk/free-space limits; GPU hard-cap compliance;
+and zero-process/zero-memory cleanup. Manipulation failures are A2, A3, and A4;
+the sole machine sanity red flag is P04 denominator instability.

docs/opprof/phase6-protocol.md

@@ -0,0 +1,638 @@
+# OpProf Phase 6 pre-registered cross-version churn protocol
+
+Status: **ACCEPTED FOR EXECUTION — ALL FIVE ORCHESTRATOR DECISIONS RESOLVED**.
+
+Date frozen: 2026-07-12 (Asia/Singapore). Phase 6 asks whether the vLLM 0.20.0
+best configuration and ranking on the C1
+`TP={1,2,4} x max-num-seqs={8,16,32,64}` surface survive an upgrade to patched
+vLLM 0.24.0. This is a paired historical-anchor experiment, not a fresh tuner
+search. It measures churn at the exact recorded C1 request sets and reports
+where the recorded local frontier moved.
+
+## Approved dispositions (orchestrator; 2026-07-12)
+
+All five open decisions are approved exactly as proposed, without changing an
+anchor, threshold, estimator, validity gate, or budget:
+
+1. The upgrade-path estimand is approved. Resolved default changes are part of
+   observed churn; dash1->dash0 and co-location remain explicit limitations and
+   the result is not described as a pure engine-version causal effect.
+2. The adaptive 25-anchor subset is approved: every cell receives its old peak
+   plus the direction-relevant adjacent anchor, and TP4/MNS16 receives both
+   neighbors.
+3. The 5% material-frontier threshold, `tau-b>=0.8` ranking-survival threshold,
+   and zero-anchored floor-bucket argmax/trap rules are approved.
+4. One primary observation per historical anchor plus the 0.35-H20-hour
+   confirmation reserve is approved.
+5. The A-P5-1-class warm-up long tier is approved as raw input `>4096` tokens
+   for this 0-8192 workload.
+
+This approval authorizes the later execution only after pinned-input, detached
+ownership, cold-start, echo-before-launch, placement, privacy, cleanup, and
+projected-budget gates pass. The 3.0-H20-hour cap remains absolute.
+
+## Amendment A-P6-1 — checkpoint-sidecar accounting and conditional cap
+
+The first Phase-6 W1 request replays were incorrectly rejected by requiring a
+clean in-stream footer and `final=true` sidecar after a non-graceful shutdown.
+That contradicts the already accepted A-P3-5/P5 accounting rule. Before any
+resume, all four W1 cell streams and their eight measured anchor intervals must
+be CPU-re-adjudicated against the latest atomic checkpoint sidecar. With no
+in-stream footer, the sidecar is authoritative when every complete JSONL line
+decodes, step indices are contiguous, on-disk data-record count equals
+`written_records`, final data-line step equals `last_step_index`,
+`encoded_records = written_records + dropped_records`, `dropped_records=0`, and
+the checkpoint is within the configured one-second flush interval (plus the
+accepted 100-ms scheduling tolerance) of the recorded shutdown boundary. At
+most one post-checkpoint flush interval may be absent. A balanced stream accepts
+all covered W1 replay intervals without a GPU rerun.
+
+All subsequent waves use the graceful P5 path: start `vllm serve` with positive
+`--shutdown-timeout`, signal each API parent with SIGINT, wait up to 150 seconds
+for EngineCore drain/finalization, and use process-group SIGTERM/SIGKILL only as
+fallback. The preferred result is one in-stream footer plus an agreeing
+`final=true` sidecar; the atomic checkpoint rule remains the crash fallback.
+
+Only if any W1 sidecar fails the CPU balance/coverage gate does A-P6-1c activate:
+the hard cap becomes **3.5 H20-hours**, the activation is logged before launch,
+and W1 is rerun under graceful shutdown. If all W1 sidecars balance, A-P6-1c is
+not activated, W1 is accepted in place, its already charged 0.299393 H20-hours
+is retained, and execution resumes at W2 under the original 3.0-hour cap.
+
+## Amendment A-P6-2 — authoritative solo frontier tier and 6.0-hour cap
+
+The user raises the cumulative Phase-6 hard cap from **3.0 to 6.0 H20-hours**.
+This amendment responds to the W2/W3 confirmation evidence: when neighboring
+wave clients became idle, pass rate changed from `0.412 -> 1.000` and `0.028 ->
+0.957` at TP2/MNS32 and from `0.036 -> 1.000` at TP4/MNS16, while Layer-1
+waiting means collapsed from `1.22 -> 0.00`, `8.50 -> 0.35`, and `28.33 ->
+0.06`. Because the v0.20 C1 baseline ran one cell at a time on dash1,
+co-located SLO feasibility is not a valid authoritative tier for the paired
+frontier. Co-location remains an explicitly reported, metric-dependent
+methodological ablation; its throughput-only observations are indicative.
+
+Every SLO-frontier-decisive Phase-6 anchor is therefore run **solo**: exactly
+one vLLM server and one replay client are active on dash0, no other Phase-6
+server/client is resident, and only the cell's `TP` GPUs are allocated. Each
+solo cell gets a fresh server process, the frozen exact-selection client, an
+independent 16-request long-tier warm-up, A-P5-1 cold-start gates, Layer-1
+telemetry, and graceful A-P6-1 shutdown. The solo result is authoritative for
+feasibility, frontier, floor bucket, argmax, tau-b, and trap decisions. A
+co-located-only result may not fill a missing solo frontier or support a paper
+claim.
+
+The mandatory solo set has no cell exclusions:
+
+| Cell set | Mandatory solo anchors | Inclusion reason |
+|---|---|---|
+| TP1/MNS8 | existing L+P | co-located peak failed |
+| TP1/MNS16,32,64 | existing P+U | co-located U passed and censored the frontier |
+| TP2/MNS8,16 | existing L+P | co-located peak failed; L distinguishes bracket vs left censoring |
+| TP2/MNS32 | existing L+P | old argmax; both anchors split primary/confirmation |
+| TP2/MNS64 | existing L+P | the indicative `>29.4% down` bound is unquotable until solo-confirmed |
+| TP4/MNS8 | existing L+P | co-located peak failed; trap-neighbor comparison |
+| TP4/MNS16 | existing L+P+U | named trap; peak split and L failed co-located |
+| TP4/MNS32,64 | P plus direction-relevant L/U | W4 completion; unmeasured under the 3.0-hour cap |
+
+After these 25 anchors, the controller may crawl only the pinned 92-anchor
+history: upward while the highest solo anchor passes or downward while the
+lowest solo anchor fails, stopping at the first opposite-feasibility anchor.
+There is no interpolation and no unrecorded request set. A pass rate in
+`[0.93,0.97]`, or a solo/co-located disagreement that changes argmax/trap or a
+material-frontier claim, triggers a same-solo-placement confirmation; a split
+gets a third trial for the frozen 2-of-3 rule if projected budget permits.
+Unresolved or history-edge frontiers remain explicitly censored.
+
+Priority is W4, TP2/MNS32, TP2/MNS64, TP4/MNS16, then the remaining failed or
+censored cells, so a budget stop preserves the most decision-relevant evidence.
+All twelve cells are nevertheless included. The planning envelope is 3.44 new
+H20-hours for serialized startup, warm-up, 25 mandatory anchors, bounded crawl,
+graceful drain, and a 0.20-hour safety reserve. Added to the already charged
+2.291173 hours, the launch projection is **5.731173/6.0 H20-hours**. Before
+each one-cell wave, the controller echoes cell, anchors, GPU placement, input
+paths, charged spend, remaining projection, and cap. Projected or actual spend
+reaching 6.0 stops further GPU work.
+
+Final analysis preserves the complete W1-W3 co-located attempt history and
+reports a solo-versus-co-located delta table for every exact anchor pair.
+ARGMAX, RANKING, and TRAP use solo values only. RANKING remains inconclusive
+and tau-b non-evaluable unless all 12 solo frontiers are bounded under the
+pinned history.
+
+## Headline claim and falsifiable outcomes
+
+The paper-facing question is:
+
+> Does the C1 vLLM 0.20.0 per-GPU SLO-feasible ranking, especially global best
+> TP2/MNS32 and local trap TP4/MNS16, remain valid on vLLM 0.24.0 when the exact
+> historical request-selection and SLO semantics are replayed?
+
+Phase 6 can produce four distinct outcomes:
+
+1. **SURVIVES:** TP2/MNS32 remains in the top floor bucket, the full-surface
+   Kendall tau-b is at least 0.8, and the TP4/MNS16 trap persists.
+2. **ARGMAX MOVED:** TP2/MNS32 is not in the top v0.24 measured-frontier bucket.
+3. **RANKING MOVED WITHOUT ARGMAX MOVEMENT:** the old best survives but tau-b is
+   below 0.8 or the trap changes status.
+4. **INCONCLUSIVE/PARTIAL:** a decision-critical frontier is unbracketed, an
+   anchor is not reproducible, repeated boundary verdicts disagree, the 12-cell
+   surface is incomplete, or the 3.0-H20-hour cap stops required work.
+
+The experiment quantifies observed upgrade churn. Because engine version,
+host, resolved defaults, and co-location differ together, it does not identify
+a pure causal effect of one vLLM commit.
+
+## Frozen vLLM 0.20.0 surface
+
+The historical objective is the maximum measured SLO-feasible offered rate
+divided by TP:
+
+```text
+f20(c) = max feasible selected_request_count / 60 seconds / TP
+```
+
+| TP | MNS8 | MNS16 | MNS32 | MNS64 |
+|---:|---:|---:|---:|---:|
+| 1 | 2.1000 | 2.3500 | 2.2833 | 2.2833 |
+| 2 | 2.2750 | 2.2750 | **3.2833** | 3.2583 |
+| 4 | 1.2833 | **2.4417** | 2.4417 | 2.4417 |
+
+Values are request/s/GPU. The global best is TP2/MNS32. TP4/MNS16 is the
+registered local trap: it is the first point on a TP4 plateau, has no improving
+adjacent MNS move, but is below the global best.
+
+The pinned ground-truth asset is
+`/home/gahow/phd/replayserve/docs/assets/simfid_s2r/ground_truth.json`, SHA-256
+`23ff3e6f6b88df8632bf37d37c0ff87bfef9d1676db81592948c08e898f7a670`.
+It contains 92 probe observations: eight anchors for each TP1/TP2 cell and seven
+for each TP4 cell.
+
+## Comparability contract
+
+### Exact workload and selection semantics
+
+The private materialized workload is
+`trace_windows/traces/chat_w20260311_1000.jsonl`, 32,606 rows, SHA-256
+`f539f38eb0ee0f750e3c23ff47df6eed3faf723a25f1444d55665a85871750b9`.
+Its window record is `trace_windows/windows.json`, SHA-256
+`23c432e7439508b07991cabfe1db9977ebb55f8ddc39ca214a627fc5f5ae4725`:
+window `[0,600]` seconds becomes `[0,60]` under replay scale 0.1.
+
+The later helper must call or byte-for-byte reproduce the following pinned
+AITuner implementations, rather than use the Phase-3/5 client:
+
+| Source | SHA-256 | Frozen role |
+|---|---|---|
+| `src/aituner/trace.py` | `6bb19f0e3eb42b1aecc6969a5f20d65eb505a91564dd117dd149155a6114eb35` | load, stable time sort, cap/downsample, threshold selection |
+| `src/aituner/worker.py` | `18337fe273a48d1cbb2bc364f774d100f8ffa2be5f9cd6a7c58c8ed4717c5e48` | rate-following submit loop, early stop, denominator accounting |
+| `src/aituner/slo.py` | `1d2e5b1ed9d0d6d6c07f8b5b6fa56d36ed9bb62710be54c125e9b2890f4f96d5` | stepped SLO evaluation |
+| `src/aituner/http_client.py` | `213b80d5b37cc74333517f46ab4e8cdba1a274bdbfabdfc14207b314f61799ac` | urllib/SSE TTFT, TPOT, usage accounting |
+
+The exact order is normative:
+
+1. Read all JSONL rows in file order. Retain raw `input_length` in `[0,8192]`.
+2. Build the same chat body and stable-sort by scaled arrival time only. Stable
+   ties retain source order; sorting by `(timestamp,sampling_u,...)` is forbidden.
+3. For TP1/TP2, even-downsample the full filtered, time-sorted list to 512
+   entries **before** threshold selection, using index
+   `floor(i*N/512)` for `i=0..511`. TP4 is uncapped.
+4. Select exactly the requests with `sampling_u <= anchor`. Preserve their
+   selected service order. Do not resample, wrap, shuffle, or interpolate.
+5. Submit at each row's materialized `timestamp*0.1`; do not uniformize,
+   rebase individual gaps, or replace the recorded burst process.
+6. Send the original messages through `/v1/chat/completions` with `stream=true`,
+   `stream_options.include_usage=true`, and the row's temperature when present.
+   Set both `min_tokens=128` and `max_tokens=128`.
+7. Use maximum concurrency 64. Client request timeout remains 900 seconds.
+8. Verify `usage.completion_tokens==128`; missing usage, a different token count,
+   HTTP failure, timeout, and every request not submitted because the SLO verdict
+   became unrecoverable are failures in the original selected-request denominator.
+9. Preserve original early-stop logic: after more than
+   `N-ceil(0.95*N)` failed evaluations, synthesize failures for the remaining
+   selected requests. Disable adaptive Stop-A; it was absent in C1.
+
+An offline, prompt-free preflight must regenerate all 92 historical request
+counts exactly. The local reconstruction already yields 17,710 filtered rows,
+512 capped rows, and **92/92 count matches**. Any future mismatch in count,
+request-ID hash, arrival hash, raw-length hash, or requested-token sum stops the
+GPU launch.
+
+### Exact SLO and score
+
+Request feasibility uses the original raw trace input length, not tokenizer
+usage after chat templating:
+
+```text
+raw input <= 4096:  TTFT <= 2000 ms
+raw input <= 32768: TTFT <= 4000 ms
+otherwise:          TTFT <= 6000 ms
+all requests:       TPOT <= 50 ms
+anchor feasible:    SLO-passing selected requests / all selected requests >= 0.95
+```
+
+Although the C1 filter limits inputs to 8192, the complete three-step rule is
+retained. TTFT is client wall time from request start to first nonempty content.
+TPOT is `(last_content_time-first_content_time)/(usage_completion_tokens-1)`.
+The paired surface score is always selected count divided by 60 seconds and TP;
+completed throughput is a diagnostic and never replaces the historical score.
+
+### Serving configuration
+
+The v0.24 command repeats the explicit C1 launch surface:
+
+```bash
+MODEL=/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
+VENV=/tmp/wjh-opprof-phase2-dash0-20260711/.venv
+RUN_ROOT=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6
+
+taskset -c "$CPUSET" env CUDA_VISIBLE_DEVICES="$GPU_IDS" \
+  VLLM_OPPROF_DIR="$RUN_DIR/opprof" \
+  "$VENV/bin/vllm" serve "$MODEL" \
+  --host 127.0.0.1 --port "$PORT" \
+  --served-model-name qwen3-30b-a3b-community \
+  --max-num-batched-tokens 8192 \
+  --max-num-seqs "$MNS" \
+  --tensor-parallel-size "$TP" \
+  --shutdown-timeout 120
+```
+
+`--shutdown-timeout` is operational and outside the measured interval. As in
+C1, do **not** explicitly set block size, GPU-memory utilization, prefix
+caching, chunked prefill, CUDA-graph capture sizes, DP, or EP. Their resolved
+v0.24 values must be parsed from startup logs and reported. Allowing version
+defaults to change is intentional: this experiment asks whether an operator's
+old explicit configuration remains good after upgrading, not whether internal
+defaults can be manually made identical.
+
+Preflight records `vllm.__version__`, source commit, patch checksums, Python,
+PyTorch, CUDA, NCCL, Triton, driver, model/tokenizer file hashes, MoE backend,
+resolved scheduler/cache/graph config, compile-cache key, and exact command.
+The expected source is the existing Phase-3/5 patched vLLM 0.24 environment;
+any missing OpProf patch or unexpected model path is a stop condition.
+
+### Client equivalence, not a new client covariate
+
+The later `scripts/opprof_phase6_client.py` is permitted only as a thin anchor
+driver around the four pinned AITuner modules. Its interface is normative:
+
+```bash
+P6C='python scripts/opprof_phase6_client.py'
+
+$P6C preflight \
+  --trace trace_windows/traces/chat_w20260311_1000.jsonl \
+  --windows trace_windows/windows.json --window-id chat_w20260311_1000 \
+  --ground-truth /home/gahow/phd/replayserve/docs/assets/simfid_s2r/ground_truth.json
+
+$P6C run-anchor --cell "$CELL" --anchor "$U" \
+  --base-url "http://127.0.0.1:$PORT" \
+  --model qwen3-30b-a3b-community --completion-tokens 128 \
+  --replay-time-scale 0.1 --max-concurrency 64 \
+  --max-requests "$CAP_OR_NONE" --target-pass-rate 0.95 \
+  --ttft-step-ms '4096:2000,32768:4000,inf:6000' --tpot-ms 50 \
+  --request-timeout-seconds 900 --probe-deadline-ceiling-seconds 900 \
+  --result-dir "$ANCHOR_DIR"
+```
+
+Before GPU execution, a no-server golden test must prove identical selected IDs,
+order, arrivals, raw input lengths, serialized bodies excluding model/URL, and
+SLO decisions for synthetic outcomes. The wrapper may add interval markers and
+sanity JSON, but may not change HTTP transport, scheduling, completion checks,
+early stop, or latency formulas.
+
+### Covariate ledger and estimand limit
+
+| Covariate | C1 v0.20 | Phase 6 v0.24 | Treatment/handling |
+|---|---|---|---|
+| Engine | community vLLM 0.20.0 | patched vLLM 0.24.0 | Primary intended churn dimension; exact builds recorded |
+| Host | dash1 | dash0 | Both 8x H20 96 GB class; CPU/NUMA/clocks/driver/kernel recorded, not assumed identical |
+| GPU visibility | TP1 C1 env exposed GPUs `0,1`, TP1 consumed one | expose exactly TP GPUs | Declared operational covariate; H20-hour charge follows TP |
+| Trial placement | recovered C1 harness trials | up to four co-located servers | CPU/NUMA pinned; reject outside GPU processes and report co-location limitation |
+| HTTP client | pinned AITuner urllib/SSE | same code paths through thin wrapper | Eliminated as a semantic covariate by golden tests |
+| Workload | materialized C1 chat window | byte-identical file/hash | Fixed |
+| Selection/cap | cap-before-threshold; TP4 uncapped | exact same | Fixed and 92-count preflighted |
+| SLO length | raw input length | raw input length | Fixed; server usage length is diagnostic only |
+| Explicit flags | TP, MNS, MBT8192 | same | Fixed |
+| Unspecified defaults | v0.20 defaults | v0.24 defaults | Intentionally allowed to churn; resolved values reported |
+| Telemetry | no Layer 1 | zero-overhead OpProf Layer 1 | Added measurement; overhead evidence -0.04196%, CI [-0.17443%, 0.04550%] |
+
+Therefore “version churn” means the observed paired upgrade path under these
+documented platform covariates. It must not be rewritten as a same-host causal
+microbenchmark.
+
+## Budget-constrained anchor subset
+
+### Adaptive adjacent-anchor design
+
+The primary subset improves on measuring next-infeasible anchors only for a
+chosen top four:
+
+1. Measure the recorded v0.20 peak anchor for all 12 cells.
+2. For every non-trap cell, select exactly one adjacent recorded anchor by a
+   result-independent rule fixed now:
+   - if the v0.20 peak remains feasible on v0.24, measure the nearest higher
+     recorded anchor, which was infeasible on v0.20;
+   - if the peak becomes infeasible, measure the nearest lower recorded anchor,
+     which was feasible on v0.20.
+3. For TP4/MNS16, measure lower, peak, and higher anchors unconditionally. This
+   directly brackets the registered trap.
+
+Thus exactly 25 primary anchors are measured: TP1 eight, TP2 eight, and TP4
+nine. The design covers every cell and spends the second anchor in the direction
+where its frontier could have moved. It also includes the suggested top-four
+test. Under the floor-bucket rule, the nominal top-four cutoff expands to five
+cells because TP4/MNS16/32/64 share one bucket; all five receive an upward test
+whenever their old peak still passes.
+
+If an adjacent anchor does not bracket the transition—higher also passes or
+lower also fails—the cell is censored. Remaining budget may crawl one recorded
+anchor farther outward, in this priority order: old global best TP2/MNS32,
+TP2/MNS64, the TP4 plateau MNS16/32/64, then other cells by old score. Crawling
+uses only that cell's recorded history and stops at the first transition. No
+new `sampling_u` value may be invented. Failure to close a decision-critical
+bracket before the budget stop makes the applicable argmax/ranking verdict
+inconclusive.
+
+### Boundary confirmations
+
+The original surface has one observation per anchor. Phase 6 keeps that paired
+unit but reserves at most 0.35 H20-hours for decision-triggered confirmation.
+Repeat an anchor if its v0.24 pass rate lies in `[0.93,0.97]`, or if its single
+verdict alone changes argmax or trap status. Priority is argmax, trap, material
+frontier calls, then other cells. Agreement across two runs freezes the verdict;
+disagreement requests a third run if budget permits and uses 2-of-3. Without a
+third run, that anchor and every dependent headline verdict are inconclusive.
+Request outcomes are arrival-correlated, so a binomial CI is descriptive only
+and cannot replace these repeat rules.
+
+## Execution plan
+
+### Preflight and detached ownership
+
+The Phase-6 controller follows A-P3-2 detached ownership: one controller PID,
+per-stage atomic state, process-group/environment markers, resumable completed
+anchors, exact command logs, and cleanup restricted to its own markers. Before
+launch it must verify dash0 has eight H20s, no outside compute processes on
+assigned GPUs, at least 100 GB CPFS free, pinned source/workload/helper hashes,
+all 92 selection counts, legal TP placement, and a projected total below 3.0
+H20-hours.
+
+The autonomous launch log must echo one resolved line before starting:
+
+```text
+LAUNCH_ECHO host=dash0 engine=vllm-0.24.0-patched model=Qwen3-30B-A3B trace_sha=f539f38e... cells=12 primary_anchors=25 waves=4 gpus=0-7 warmup=16 clean_horizon=60s drain=derived<=120s est_wall=25-35min est_gpu=2.65_H20h confirm_reserve=0.35_H20h hard_cap=3.0_H20h
+```
+
+### Rate-following cold-start and drain gates
+
+The throughput-drift stationarity gate is forbidden for these rate-following
+anchors. A-P5-1-class cold-start gates are applied instead:
+
+1. Every torch.compile/CUDA-graph first-occurrence event must precede the first
+   measured anchor, verified from server logs and Layer 1.
+2. A deterministic unmeasured warm set must complete at least 16 exact-output
+   requests, including at least one request with raw input `>4096` tokens. The
+   peak-selected sets contain 43-184 such requests, so this gate is feasible.
+3. No first-occurrence compile/capture event may occur in any measured anchor;
+   all graph-hit `(runtime_mode,bucket_tokens)` descriptors must be covered by
+   startup capture logs. A violation invalidates that anchor only.
+4. Before the next anchor, all selected requests must be accounted for and three
+   consecutive controller samples must show running/waiting/deferred queues at
+   zero. Prefix cache is not explicitly enabled; nevertheless a full drain is
+   required.
+
+The probe deadline reproduces `_probe_drain_deadline`: last selected arrival +
+raw-length TTFT budget + `128*50 ms` + 30 seconds, capped by the historical
+900-second ceiling. With raw inputs <=8192 this is about 100.4 seconds from
+probe start; the controller has a 120-second class watchdog for cleanup. A
+deadline miss is an SLO-infeasible anchor, not permission to omit denominator
+requests. The server then shuts down through the official 120-second path.
+
+### Four-wave packing
+
+| Wave | Cells | Placement | Per-server measured anchors | Planned wall |
+|---|---|---|---:|---:|
+| W1 | TP1/MNS8,16,32,64 | four servers on GPUs 0,1,2,3; GPUs 4-7 idle | 2 each | 4-6 min |
+| W2 | TP2/MNS8,16,32,64 | four servers on GPU pairs 0-1,2-3,4-5,6-7 | 2 each | 4-6 min |
+| W3 | TP4/MNS8 and TP4/MNS16 trap | GPU quads 0-3 and 4-7 | 2 and 3 | 5-7 min |
+| W4 | TP4/MNS32 and TP4/MNS64 | GPU quads 0-3 and 4-7 | 2 each | 4-6 min |
+
+Each server starts once, passes warm-up once, runs its peak first, drains, then
+runs the predeclared adjacent anchor. Trap lower/upper order after peak is
+counterbalanced by execution seed 20260717. Anchor confirmation occurs before
+that server is released when possible. The controller records GPU clocks,
+power, utilization, memory, CPU load, NUMA placement, and outside processes
+through every wave. Any contamination invalidates the whole co-located wave.
+
+Actual H20-hours are charged from server launch through verified cleanup for
+every allocated GPU, including idle time while a paired TP4 cell finishes.
+Before each wave or confirmation, the controller recomputes
+`spent + current + conservative_remaining`; it stops before launch if the
+projection can reach 3.0. Optional crawling and confirmations are sacrificed
+before any hard-cap violation.
+
+## Measurement and mechanism notes
+
+For every anchor, report selected/admitted/completed/exact-output/SLO-pass
+counts; early-stop reason; offered and completed req/s; TTFT/TPOT mean and
+p50/p90/p95/p99; failures by raw-length TTFT bucket; elapsed/drain time; and
+peak GPU memory/utilization. The surface objective remains offered req/s/GPU at
+the highest measured feasible recorded anchor.
+
+Layer 1 is active over the complete anchor service interval, from the first
+submission through the last selected-request accounting event. Aggregate:
+
+- prefill, decode, and mixed-step counts and token shares;
+- decode-batch distribution and 5-second CV;
+- FULL/PIECEWISE/NONE graph-mode shares, bucket padding, and uncovered support;
+- running/waiting/deferred queues and queue CV;
+- preemption count/rate;
+- KV blocks used/total, mean/max usage, and allocation pressure; and
+- prefix queries/hits as a resolved-default diagnostic.
+
+Mechanism notes compare v0.24 cells/anchors and may relate a frontier change to
+preemptions, KV pressure, batching, or graph modes. C1 lacks Layer-1 telemetry,
+so these notes cannot claim that a measured composition field itself changed
+from v0.20 or caused the version drift. Historical v0.20 TTFT/TPOT failure
+reasons may be paired directly; device-mechanism attribution remains
+descriptive.
+
+## Statistical and ranking analysis
+
+### Floor buckets
+
+For each unrounded score vector `s`, use the SimFid inventory rule:
+
+```text
+tol(s) = max(1e-9, 1e-6 * max_c(abs(s(c))))
+bucket_s(c) = floor(s(c) / tol(s))
+```
+
+Two scores tie only when their integer buckets match. Bucketing occurs before
+display rounding, pair signs, tau-b, top selection, or trap evaluation. Report
+`tol`, bucket integers, and occupied intervals `[j*tol,(j+1)*tol)`. For v0.20,
+`tol20=3.283333333333333e-6`; the TP4 MNS16/32/64 tie makes nominal top-4 an
+effective top-5 set.
+
+### Per-cell frontier
+
+For each cell, define the v0.24 measured recorded-anchor frontier:
+
+```text
+f24(c) = max_{measured u with accepted feasible verdict} N_c(u) / 60 / TP
+drift(c) = (f24(c) - f20(c)) / f20(c)
+```
+
+The predeclared material threshold is **X=5% relative**, large enough to avoid
+calling one-request anchor quantization a paper-level churn effect. Report two
+orthogonal labels:
+
+- **BOUNDARY DOWN/UP:** old peak flips feasible->infeasible, or the old adjacent
+  infeasible anchor flips to feasible, respectively;
+- **MATERIAL FRONTIER MOVED:** `abs(drift)>5%`; otherwise report the boundary
+  flip as sub-material.
+
+If peak passes and higher fails, and `abs(drift)<=5%`, the local recorded
+frontier is **STABLE**. If peak and lower both fail or peak and every affordable
+higher anchor pass, report a directional bound and **UNBRACKETED**, never an
+invented peak.
+
+### Argmax, ranking, and trap
+
+- **ARGMAX MOVED** when TP2/MNS32 is not in the top `bucket_f24` among all 12
+  decision-bounded cells. **ARGMAX SURVIVED** requires it to share the top
+  bucket and no censored competitor capable of overtaking it. Otherwise it is
+  inconclusive.
+- Compute Kendall tau-b over all 12 floor-bucketed `f20/f24` scores and report
+  concordant, discordant, v0.20-tied, v0.24-tied, and comparable-pair counts.
+  **RANKING SURVIVES** requires an evaluable 12-cell surface, argmax survival,
+  `tau-b>=0.8`, and no >5% top-bucket pair reversal. `tau-b<0.8` is **RANKING
+  MOVED**. A partial surface receives no suite-level tau claim.
+- **TRAP PERSISTS** when TP4/MNS16 is in a bucket at least as high as both
+  adjacent TP4 cells MNS8 and MNS32, yet below the global-best bucket. It
+  **ESCAPES** if an adjacent TP4 move is strictly better, and **CEASES TO BE A
+  TRAP** if it joins the global-best bucket. Missing/unbracketed inputs make trap
+  status inconclusive.
+
+No arithmetic average of normalized scores is reported. Absolute rates,
+relative drift, the paired table, and every excluded/censored anchor accompany
+all ranking summaries.
+
+## Later deliverables and artifact contract
+
+Execution, after approval, must produce:
+
+- `docs/opprof/phase6-results.md`;
+- `runs/opprof-phase6/phase6/metrics.json` containing the complete paired
+  surface, anchor/run values, selection hashes, feasibility, floor buckets,
+  frontier/argmax/ranking/trap verdicts, confirmation history, Layer-1 notes,
+  covariates, and GPU accounting;
+- exact commands, controller state, environment/machine fingerprints, server
+  logs, client outcomes, Layer-1 JSONL/footer/sidecar, monitor samples, and
+  per-anchor sanity JSON; and
+- an operational-findings section covering startup/capture gates, drains,
+  early stops, co-location, default changes, retries, contamination, and budget.
+
+The results begin with data sanity and stop inferential claims on any red flag.
+Every numeric family ends with `n`, finite/missing, min, max, distinct count,
+and applicable invariants. Prompt text, messages, generated content, source
+substrings, and private request bodies stay in mode-0700 storage. Public files
+contain only hashes, IDs, lengths, timestamps, outcomes, counters, and summaries.
+
+## Anchor-subset table
+
+Notation is `u / selected N / recorded req/s/GPU`. `P` is always measured. For
+non-trap cells, measure `U` if P passes on v0.24, otherwise `L`. TP4/MNS16
+measures all `L+P+U`. Planning cost per anchor includes the 60-second replay and
+20 seconds expected drain/accounting; shared startup/warm-up is budgeted below.
+
+| Cell | L: nearest recorded feasible below P | P: recorded peak | U: nearest recorded infeasible above P | Primary selection | H20-hour cost each |
+|---|---|---|---|---|---:|
+| TP1/MNS8 | 0.21875 / 121 / 2.0167 | 0.2265625 / 126 / 2.1000 | 0.23046875 / 130 / 2.1667 | P + (U if pass else L) | 0.0222 |
+| TP1/MNS16 | 0.2421875 / 137 / 2.2833 | 0.24609375 / 141 / 2.3500 | 0.25 / 143 / 2.3833 | P + (U if pass else L) | 0.0222 |
+| TP1/MNS32 | 0.234375 / 132 / 2.2000 | 0.2421875 / 137 / 2.2833 | 0.24609375 / 141 / 2.3500 | P + (U if pass else L) | 0.0222 |
+| TP1/MNS64 | 0.234375 / 132 / 2.2000 | 0.2421875 / 137 / 2.2833 | 0.24609375 / 141 / 2.3500 | P + (U if pass else L) | 0.0222 |
+| TP2/MNS8 | 0.4921875 / 269 / 2.2417 | 0.49609375 / 273 / 2.2750 | 0.5 / 276 / 2.3000 | P + (U if pass else L) | 0.0444 |
+| TP2/MNS16 | 0.4921875 / 269 / 2.2417 | 0.49609375 / 273 / 2.2750 | 0.5 / 276 / 2.3000 | P + (U if pass else L) | 0.0444 |
+| TP2/MNS32 | 0.75 / 391 / 3.2583 | 0.75390625 / 394 / **3.2833** | 0.7578125 / 396 / 3.3000 | P + (U if pass else L) | 0.0444 |
+| TP2/MNS64 | 0.5 / 276 / 2.3000 | 0.75 / 391 / 3.2583 | 0.75390625 / 394 / 3.2833 | P + (U if pass else L) | 0.0444 |
+| TP4/MNS8 | 0.016055910008 / 301 / 1.2542 | 0.016591107009 / 308 / 1.2833 | 0.017126304009 / 317 / 1.3208 | P + (U if pass else L) | 0.0889 |
+| TP4/MNS16 trap | 0.033182214016 / 575 / 2.3958 | 0.033717411016 / 586 / **2.4417** | 0.034252608017 / 600 / 2.5000 | **L + P + U** | 0.0889 |
+| TP4/MNS32 | 0.033182214016 / 575 / 2.3958 | 0.033717411016 / 586 / 2.4417 | 0.034252608017 / 600 / 2.5000 | P + (U if pass else L) | 0.0889 |
+| TP4/MNS64 | 0.033182214016 / 575 / 2.3958 | 0.033717411016 / 586 / 2.4417 | 0.034252608017 / 600 / 2.5000 | P + (U if pass else L) | 0.0889 |
+
+## Total GPU estimate versus 3.0 cap
+
+The 25 primary anchors contain 8 TP1, 8 TP2, and 9 TP4 replays. At the
+80-second per-anchor planning charge they consume `0.1778 + 0.3556 + 0.8000 =
+1.3334` H20-hours. Four shared startup/warm-up/cleanup envelopes plus runtime
+variance are budgeted at 1.3166 hours, for a **2.65-H20-hour primary plan**.
+The remaining **0.35 H20-hours** is reserved for boundary confirmations or
+decision-critical outward crawling. The absolute launch cap is **3.0
+H20-hours**; projected or actual spend reaching the cap stops all further work.
+Expected wall time is 25-35 minutes over four waves. Expected public disk is
+0.3-0.6 GB and must remain below 3 GB.
+
+## Decision rules
+
+1. Anchor feasibility is exact-output request pass rate `>=0.95` using raw-length
+   stepped TTFT and 50-ms TPOT; absent/early-stopped requests remain failures.
+2. Boundary movement is an old peak feasible->infeasible flip or an old adjacent
+   infeasible->feasible flip. Material frontier churn is `abs(drift)>5%`.
+3. Apply zero-anchored floor buckets with set-wide `tol=max(1e-9,1e-6*max|s|)`
+   before every tie/rank decision.
+4. ARGMAX MOVED iff TP2/MNS32 is outside the v0.24 top bucket on a bounded
+   12-cell subset; censored contenders make survival inconclusive.
+5. RANKING SURVIVES only with all 12 cells, argmax survival, tau-b >=0.8, and no
+   >5% top-bucket pair reversal. Tau-b <0.8 is RANKING MOVED.
+6. TP4/MNS16 trap persists only when it is no worse than adjacent MNS8/MNS32 and
+   remains below the global best; an improving neighbor means escape.
+7. Boundary `[0.93,0.97]` or verdict-changing anchors require confirmation when
+   budget permits. Disagreement without a 2-of-3 resolution is inconclusive.
+
+## Open decisions for orchestrator
+
+1. Approve the **upgrade-path estimand**: resolved default changes are part of
+   churn, while dash1->dash0 and co-location remain explicit limitations rather
+   than pretending this is a pure engine-version causal effect.
+2. Approve the **adaptive 25-anchor subset**, which gives every cell a
+   direction-relevant neighbor and the trap both neighbors, instead of spending
+   next-infeasible anchors only on a tie-ambiguous top four.
+3. Approve **5% material frontier drift**, `tau-b>=0.8`, and the stated
+   argmax/trap floor-bucket rules.
+4. Approve one primary observation per historical anchor with the 0.35-hour
+   reserve for `[0.93,0.97]`/verdict-changing confirmations, rather than reducing
+   surface coverage to replicate every anchor.
+5. Approve the A-P5-1-class warm-up adaptation from `input>=8192` to
+   **raw input >4096**, the longest SLO-relevant tier available reliably in this
+   0-8192 C1 workload.
+
+## Sanity block
+
+| Numeric family | n | Min | Max | Distinct | Checked invariant/result |
+|---|---:|---:|---:|---:|---|
+| C1 cells | 12 | TP=1,MNS=8 | TP=4,MNS=64 | 12 | Exact 3x4 Cartesian surface |
+| Historical probe observations | 92 | 7/cell | 8/cell | 2 per-cell counts | `8*8 + 4*7 = 92` |
+| Materialized trace rows | 1 file | 32,606 | 32,606 | 1 expected | SHA pinned; prompt content not emitted |
+| Raw-length-filtered rows | 1 reconstruction | 17,710 | 17,710 | 1 expected | Raw length 22-8192; non-negative |
+| Capped TP1/TP2 source rows | 1 set | 512 | 512 | 1 expected | Stable time sort, then even downsample |
+| Reconstructed historical counts | 92 | 66 | 600 | 34 | 92/92 match ground truth |
+| C1 peak scores (req/s/GPU) | 12 | 1.283333 | 3.283333 | 8 | Finite, positive, not all identical |
+| v0.20 floor tolerance | 1 vector | 3.283333e-6 | 3.283333e-6 | 1 | Applied before display rounding |
+| Nominal top-4 effective size | 1 cutoff | 5 | 5 | 1 | TP4 three-way cutoff tie retained |
+| Primary peak anchors | 12 | 1/cell | 1/cell | 1 expected | No cell omitted |
+| Realized adjacent/trap anchors | 13 | 1/non-trap | 2/trap | 2 | Predeclared direction rule; total 25 |
+| Primary anchors by TP | 25 | 8 at TP1/TP2 | 9 at TP4 | 2 counts | `8+8+9=25` |
+| Selected requests in anchor candidates | 36 candidates | 121 | 600 | >1 expected | TP1/2 cap-before-threshold; TP4 uncapped |
+| Per-anchor planning cost (H20-hour) | 3 TP classes | 0.0222 | 0.0889 | 3 | Proportional to TP for 80 seconds |
+| Primary planned H20-hours | 1 plan | 2.65 | 2.65 | 1 | Non-negative; below 3.0 |
+| Confirmation/crawl reserve | 1 plan | 0.35 | 0.35 | 1 | Primary plus reserve equals hard cap |
+| GPU work in this protocol turn | 1 turn | 0 | 0 | 1 expected | Protocol-only requirement satisfied |
+
+Checked invariants: `12=3*4`; `92=8*8+4*7`; all 92 local selection counts
+match; cap precedes threshold; arrivals span the fixed 60-second replay; raw
+input length, not retokenized usage, selects TTFT thresholds; completion is
+exactly 128; SLO denominator includes all selected requests; surface scores use
+60 seconds and divide by TP; floor buckets precede ranking; the top-four cutoff
+expands to five; `25=12+11+2`; planned `2.65+0.35=3.0`; ratios and counters have
+their declared domains; and no GPU run, helper, manifest, remote artifact, or
+results file was created in this protocol-only turn.

docs/opprof/phase6-results.md

@@ -0,0 +1,226 @@
+# OpProf Phase 6 dash0 results
+
+Status: **COMPLETE, DATA-VALID A-P6-2 SOLO TIER; ARGMAX/RANKING/TRAP INCONCLUSIVE UNDER THE FROZEN RULES**.
+
+Phase 6 completed all 12 C1 cells on patched vLLM
+`0.24.1.dev3+g668cfb7e2` (`4b253fd`). A-P6-2 makes the serialized solo tier
+authoritative because the v0.20 baseline was solo and the original W2/W3
+confirmations exposed large pass-rate and queue changes after neighboring wave
+clients became idle. The authoritative campaign contains 37 primary anchors,
+29 same-placement confirmations, and 12 valid Layer-1 streams.
+
+The machine-readable result is
+`runs/opprof-phase6/phase6/metrics.json`, SHA-256
+`290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16`.
+Co-located W1-W3 artifacts are preserved beside the solo tree rather than
+overwritten. Public artifacts contain no prompt or generated text.
+
+## Data sanity first
+
+There are **no data-sanity red flags**. All 37 authoritative primary anchors
+reproduce the pinned v0.20 request count exactly; all client invariants pass;
+all 12 solo cells have one graceful footer and agreeing final sidecar; Layer-1
+indices are contiguous with zero drops; every anchor interval is represented;
+and all cold-start compile/capture events precede measurement.
+
+The scientific decision gate is separate from data validity. Eight cells have
+bounded monotonic solo frontiers. TP2/MNS16, TP4/MNS32, and TP4/MNS64 pass the
+highest recorded request set and are right-censored at the pinned-history edge.
+TP4/MNS16 is non-monotonic: L resolves feasible, P resolves infeasible, and U
+resolves feasible. These four cells prohibit a full-surface floor-bucket rank,
+tau-b, and trap decision; no value is imputed beyond the recorded history.
+
+## Launch echo
+
+```text
+A-P6-2_LAUNCH_ECHO host=dash0 engine=vllm-0.24.1.dev3+g668cfb7e2@4b253fd authoritative=solo cells=12 mandatory_anchors=25 crawl=recorded-only confirmations=2of3 prior_h20h=2.291173 new_est_h20h=3.440 cumulative_est_h20h=5.731173 cap_h20h=6.0 est_wall=45-70min
+SOLO_WAVE_ECHO order=TP4/MNS32,TP4/MNS64,TP2/MNS32,TP2/MNS64,TP4/MNS16,TP2/MNS8,TP2/MNS16,TP4/MNS8,TP1/MNS8,TP1/MNS16,TP1/MNS32,TP1/MNS64 placement=one_server_one_client
+FINAL_ACCOUNTING status=complete cells=12 primary=37 confirmations=29 solo_h20h=3.353371 campaign_h20h=5.644544 cap_h20h=6.0
+```
+
+The exact per-cell echoes, GPU placement, trace and ground-truth paths, spend,
+and projection are retained in
+`runs/opprof-phase6/phase6/solo-authoritative/launch-echo.log`.
+
+## Run completion stats
+
+| Item | Result |
+|---|---:|
+| Historical selections reconstructed | 92/92 exact |
+| Authoritative solo cells | 12/12 |
+| Solo primary anchors | 37 |
+| Solo confirmations | 29 |
+| Solo anchor trials | 66 |
+| Solo warm-ups | 12; exactly 16 completions each |
+| Long prompts per warm-up | 2-8 raw inputs `>4096` |
+| Bounded solo frontiers | 8/12 |
+| History-edge right-censored cells | 3 |
+| Non-monotonic cells | 1 |
+| Prior co-located primary/confirm trials | 21 + 3 |
+| Total accepted anchor trials across attempts | 90 |
+| Co-located spend before A-P6-2 | 2.291173 H20-hours |
+| A-P6-2 solo spend | 3.353371 H20-hours |
+| Total campaign spend | **5.644544 H20-hours** |
+| Artifact footprint | 467 MiB, below 3 GiB |
+
+## Final paired surface
+
+`A*` marks the authoritative A-P6-2 solo tier. Exact values are bounded
+frontiers. `>=` is a recorded-history lower bound. The TP4/MNS16 value is not a
+frontier because its feasibility sequence is non-monotonic.
+
+| Cell | Tier | v0.20 frontier | v0.24 authoritative result | Drift | Status |
+|---|---|---:|---:|---:|---|
+| TP1/MNS8 | A* solo | 2.1000 | **2.3833** | **+13.49%** | bounded up |
+| TP1/MNS16 | A* solo | 2.3500 | 2.3833 | +1.42% | bounded up |
+| TP1/MNS32 | A* solo | 2.2833 | 2.3833 | +4.38% | bounded up |
+| TP1/MNS64 | A* solo | 2.2833 | 2.3833 | +4.38% | bounded up |
+| TP2/MNS8 | A* solo | 2.2750 | 2.2417 | -1.47% | bounded down |
+| TP2/MNS16 | A* solo | 2.2750 | >=2.3000 | >=+1.10% | right-censored history edge |
+| TP2/MNS32 | A* solo | **3.2833** | **3.2583** | -0.76% | bounded down |
+| TP2/MNS64 | A* solo | 3.2583 | **2.3000** | **-29.41%** | bounded down |
+| TP4/MNS8 | A* solo | 1.2833 | 1.3208 | +2.92% | bounded up |
+| TP4/MNS16 | A* solo | 2.4417 | N/E; observed max feasible 2.5000 | N/E | **non-monotonic: L pass, peak fail, U pass** |
+| TP4/MNS32 | A* solo | 2.4417 | >=2.5000 | >=+2.39% | right-censored history edge |
+| TP4/MNS64 | A* solo | 2.4417 | >=2.5000 | >=+2.39% | right-censored history edge |
+
+The solo tier validates two material frontier changes under the predeclared 5%
+threshold: TP1/MNS8 improves by 13.49%, while TP2/MNS64 declines by 29.41%.
+The previous TP2/MNS64 decline is now quotable because both its passing 2.3000
+anchor and failing 3.2583 anchor were repeated solo.
+
+## Frozen decision-rule verdicts
+
+- **ARGMAX: INCONCLUSIVE.** TP2/MNS32 is the highest bounded observed frontier
+  at 3.2583 req/s/GPU, but the frozen rule requires a bounded 12-cell subset.
+  TP2/MNS16 and TP4/MNS32/64 terminate at passing history-edge anchors, and
+  TP4/MNS16 has no coherent frontier. The top floor bucket is therefore not
+  defined for all contenders.
+- **RANKING: INCONCLUSIVE.** Only 8/12 frontiers are bounded. Per protocol,
+  Kendall tau-b is **not evaluable** (`tau_b=null`) rather than computed from
+  censored lower bounds. The `tau-b>=0.8` survival rule cannot be applied.
+- **TRAP: INCONCLUSIVE.** TP4/MNS16 is non-monotonic and its TP4/MNS32 neighbor
+  is right-censored. Neither trap persistence nor escape satisfies the frozen
+  floor-bucket rule.
+
+## Solo versus co-located exact-anchor deltas
+
+The co-located column is the simultaneous-wave primary. The solo column is the
+accepted median after same-placement 2-of-3 adjudication. Delta is percentage
+points. `P/F` are SLO feasible/infeasible at 0.95.
+
+| Cell | Anchor | Co-located primary | Solo authoritative | Delta | State change |
+|---|---:|---:|---:|---:|---|
+| TP1/MNS8 | 0.21875000 | 0.992 | 0.992 | +0.00pp | P->P |
+| TP1/MNS8 | 0.22656250 | 0.206 | 0.992 | **+78.57pp** | F->P |
+| TP1/MNS16 | 0.24609375 | 1.000 | 1.000 | +0.00pp | P->P |
+| TP1/MNS16 | 0.25000000 | 1.000 | 1.000 | +0.00pp | P->P |
+| TP1/MNS32 | 0.24218750 | 1.000 | 1.000 | +0.00pp | P->P |
+| TP1/MNS32 | 0.24609375 | 1.000 | 1.000 | +0.00pp | P->P |
+| TP1/MNS64 | 0.24218750 | 1.000 | 1.000 | +0.00pp | P->P |
+| TP1/MNS64 | 0.24609375 | 1.000 | 1.000 | +0.00pp | P->P |
+| TP2/MNS8 | 0.49218750 | 0.691 | 1.000 | +30.86pp | F->P |
+| TP2/MNS8 | 0.49609375 | 0.092 | 0.581 | +48.90pp | F->F |
+| TP2/MNS16 | 0.49218750 | 1.000 | 1.000 | +0.00pp | P->P |
+| TP2/MNS16 | 0.49609375 | 0.088 | 1.000 | **+91.21pp** | F->P |
+| TP2/MNS32 | 0.75000000 | 0.412 | 1.000 | +58.82pp | F->P |
+| TP2/MNS32 | 0.75390625 | 0.028 | 0.575 | +54.70pp | F->F |
+| TP2/MNS64 | 0.50000000 | 0.460 | 1.000 | +53.99pp | F->P |
+| TP2/MNS64 | 0.75000000 | 0.141 | 0.561 | +42.07pp | F->F |
+| TP4/MNS8 | 0.016055910008 | 1.000 | 1.000 | +0.00pp | P->P |
+| TP4/MNS8 | 0.016591107009 | 0.071 | 1.000 | **+92.86pp** | F->P |
+| TP4/MNS16 | 0.033182214016 | 0.609 | 1.000 | +39.13pp | F->P |
+| TP4/MNS16 | 0.033717411016 | 0.036 | 0.238 | +20.22pp | F->F |
+| TP4/MNS16 | 0.034252608017 | 0.847 | 1.000 | +15.33pp | F->P |
+
+The deltas range from 0 to +92.86pp across 21 exact pairs. Four TP1/MNS16-64
+pairs are unchanged, while several TP2/TP4 and TP1/MNS8 anchors flip. Thus
+co-location validity is **metric- and cell-dependent**, not a universal offset.
+Throughput-only values may co-locate, but SLO-frontier feasibility cannot be
+mixed across placement tiers.
+
+## Layer-1 mechanism notes for material drift
+
+Layer-1 is mechanism context, not a matched v0.20 causal decomposition. P3 P10
+is TP1 at a different workload/output contract. All 37 solo primaries have
+zero preemptions.
+
+| Cell/anchor | State | Waiting mean/max | Decode-B mean | KV mean | `NONE` graph share | Padding |
+|---|---|---:|---:|---:|---:|---:|
+| TP1/MNS8, 0.25 | frontier pass | 0.74 / 8 | 4.25 | 0.0407 | 3.26% | 0.34% |
+| TP1/MNS8, 0.50 | next fail | 11.66 / 27 | 7.19 | 0.0644 | 5.04% | 0.02% |
+| TP2/MNS64, 0.50 | frontier pass | 0.00 / 0 | 3.28 | 0.0084 | 0.00% | 18.88% |
+| TP2/MNS64, 0.75 | old-peak fail, primary | 0.00 / 0 | 18.05 | 0.0448 | 9.47% | 0.52% |
+| TP2/MNS64, 0.75 | old-peak fail, confirm | 0.0002 / 1 | 11.52 | 0.0285 | 4.57% | 1.13% |
+
+For TP1/MNS8, the new frontier remains feasible despite a larger decode batch
+and modest queueing; failure at 0.5 coincides with a 16x waiting-mean increase,
+higher KV usage, and more eager/non-full graph execution. For TP2/MNS64, the
+passing lower anchor has small decode batches and full/piecewise graphs despite
+high padding, whereas both old-peak trials fail with much larger decode batches
+and graph fallback. Zero preemptions and near-zero queueing rule out preemption
+and server waiting as the main TP2/MNS64 explanation, but the unmatched v0.20
+telemetry prevents assigning causality uniquely to graph mode or batch shape.
+
+## Operational findings and full attempt history
+
+1. The initial ownership-monitor false start charged 0.132918 H20-hours without
+   measurement. The repaired W1 charged 0.299393; A-P6-1 later accepted its
+   eight anchors from balanced checkpoint sidecars without rerun.
+2. Co-located W2/W3 completed 13 primary anchors and three confirmations but
+   exposed placement-sensitive SLO results. The 3.0-hour projection stopped W4
+   at 2.291173 total; those values remain indicative and preserved.
+3. A-P6-2 completed W4 and remeasured the full surface solo. Every solo server
+   used `--shutdown-timeout 120`, emitted a graceful footer, and passed Layer-1
+   accounting.
+4. Placement was not the only transient. TP4/MNS32 and TP4/MNS64 first full
+   replays failed at 0.031/0.114 but subsequent identical solo trials passed;
+   TP4/MNS16 produced an order-dependent non-monotonic sequence. The 16-request
+   A-P5-1 gate removes compile/capture cold start but not necessarily the first
+   full-trace transient. The predeclared 2-of-3 rule prevented single-trial
+   conclusions, but a future protocol should use a full-trace burn-in or
+   randomized/reversed anchor order.
+5. Two cleanup checks briefly observed 1-4 MiB at 0% with zero compute PIDs.
+   Memory decayed to 0 MiB within seconds, graceful footers validated, and no
+   rerun was needed. Cleanup now polls for 30 seconds rather than treating
+   transient driver bookkeeping as a failed experiment.
+
+## GPU total
+
+The complete campaign charged **5.6445441643 H20-hours** against the
+user-approved **6.0** cap:
+
+- original co-located attempts: 2.2911728495;
+- A-P6-2 authoritative solo tier: 3.3533713148;
+- remaining headroom: 0.3554558357 H20-hours.
+
+Final dash0 state is 0 MiB and 0% utilization on all eight H20s, with zero
+controller, client, vLLM, EngineCore, or worker processes.
+
+## Sanity block
+
+| Numeric family | n | Min | Max | Distinct | Checked invariant/result |
+|---|---:|---:|---:|---:|---|
+| Historical selection checks | 92 | 66 requests | 600 requests | 34 | 92/92 exact |
+| Solo primary pass rates | 37 | 0.0307 | 1.0000 | 17 | Ratios in `[0,1]`; not identical |
+| Solo selected requests | 37 | 121 | 600 | 18 | Exact v0.20 count per anchor |
+| Layer-1 steps per primary | 37 | 343 | 12,103 | 37 | Contiguous; non-negative; zero drops |
+| Layer-1 records per cell | 12 | 14,174 | 58,725 | 12 | Footer/sidecar balanced |
+| Solo preemptions | 37 | 0 | 0 | 1 expected | Zero throughout |
+| Reported v0.24 frontier values/bounds | 11 | 1.3208 | 3.2583 | 6 | TP4/MNS16 omitted as non-monotonic |
+| Frontier drift values/bounds | 12 | -29.41% | +13.49% | 9 | 11 finite; one non-monotonic missing |
+| Bounded/censored cells | 12 | 8 bounded | 4 unbounded | 2 | Censoring never imputed |
+| Exact solo/co-located deltas | 21 | +0.00pp | +92.86pp | 13 | Same request set and anchor |
+| Solo confirmation pass rates | 29 | 0.4130 | 1.0000 | 10 | Frozen 2-of-3 applied |
+| Solo cell H20-hours | 12 | 0.0845 | 0.5534 | 12 | Non-negative; serialized placement |
+| Total campaign H20-hours | 1 | 5.644544 | 5.644544 | 1 | Below 6.0 hard cap |
+| Final GPU memory/utilization | 8 | 0 MiB / 0% | 0 MiB / 0% | 1 expected | Zero compute processes |
+
+Checked invariants: exact cap-before-threshold selection; raw-length SLO
+evaluation; exact 128-token completions or counted failures; nondecreasing
+arrivals; all selected outcomes represented in the denominator; one solo
+server/client at a time; compile/capture outside measured intervals; 12/12
+graceful footer balances; non-negative counters; pass ratios in range; no prompt
+text in public artifacts; complete attempt-history retention; authoritative
+tier separation; explicit censoring/non-monotonicity; tau-b suppression when
+the full surface is unbounded; hard-cap compliance; and complete GPU cleanup.

docs/opprof_campaign_state.md

@@ -0,0 +1,80 @@
+# OpProf campaign state — pattern-conditioned operator profiling (vLLM 0.24.0 / Qwen3-30B-A3B / H20)
+
+Started 2026-07-11. Orchestrator: Claude (session a42d23fb). Workers: codex via companion.
+Discipline inherited from SimFid campaign (replayserve/docs/simfid_campaign_state.md):
+pre-registered acceptance gates per phase, codex implement + independent review,
+orchestrator verifies, echo-before-expensive-action, GPU actions logged.
+
+## User decisions (2026-07-11)
+- vLLM pinned to **0.24.0** (latest community); record commit + sha256 at clone time.
+- GPU: **dash0 8×H20, approved for the whole campaign** (Phases 2-4). Orchestrator
+  still echoes load/duration before each launch. Single H20 sufficient for TP1.
+- Observation patch: **dual-layer** — always-on lightweight per-iteration composition
+  telemetry (<3% overhead budget) + sampled heavy torch.profiler/CUPTI windows.
+- L-C-A is REFERENCE ONLY in this campaign — pattern axes defined operationally
+  (input-len dist / output len / arrival shape / prefix sharing), no L-C-A dependency.
+
+## Phase plan (pre-registered)
+- P0 recon (local): clone v0.24.0, inventory built-in observability, patch design doc.
+  Gate: USER reviews patch design before P1 implementation.
+- P1 patch dev + no-GPU tests (local). Gate: strict review + orchestrator verify.
+- P2 single-H20 smoke on dash0: artifacts complete; measured overhead (on/off same
+  load) within budget; op-time sums ≈ iteration time; composition matches request-level
+  ground truth. Hard gate before P3.
+- P3 pattern matrix (~8-12 patterns × 2-3 configs, short replays, dash0): per-pattern
+  operator time breakdown + composition distributions + waste accounting
+  (padding% / graph-miss / ragged imbalance / mixed-batch interference).
+  Hypothesis go/no-go: does operator bottleneck ranking change across patterns?
+- P4 optimization proposal (ranked, measured bounds) + one cheap config-level
+  closed-loop validation (e.g., cudagraph capture sizes vs measured B distribution).
+- Non-goals: no new kernels; no L-C-A ablation (separate track); no production clusters.
+
+## Artifact layout
+- vLLM source: /home/gahow/phd/vllm-v0.24.0 (clone, pinned)
+- Patches: aituner patches/vllm-0.24.0-opprof/ (patch files + apply script + no-GPU tests)
+- Docs: aituner docs/opprof/ (phase0-recon.md, patch-design.md, later phase reports)
+
+## Job log
+- P0 recon dispatched: task-mrg3nm1f-spyl85 (2026-07-11T08:25:42Z). Scope: clone v0.24.0 to ~/phd/vllm-v0.24.0, observability inventory with file:line, dual-layer patch design doc. Gate: user reviews design before P1.
+- P0 ACCEPTED (task-mrg3nm1f, 34m54s): pin ee0da84a=v0.24.0 verified; VLLM_TORCH_PROFILER_DIR-absent claim verified; profiler moved to ProfilerConfig + entrypoints/serve/profile/api_router.py (verified); design doc 421 lines. USER APPROVED design with 5 delegated decisions (JSONL+msgspec+8192 queue; expert loads L2-only; 2+8 profiler window; 3% gate on 95% CI upper bound; reject --disable-log-stats) + checkpoint decision: community BF16, TP1 primary, TP2/4 counterpoints.
+- P1 dispatching: implement per approved design; branch in ~/phd/vllm-v0.24.0 + exported patches/vllm-0.24.0-opprof/ in aituner; no-GPU tests must pass locally (opprof core import-light).
+- P1 completed (task-mrg588j2, 26m36s): branch opprof tip c60e7eeb, +560/-1 across 5 files (within ±30% of design). Orchestrator pre-checks: pytest 11/11 reproduced; patch series applies cleanly on pristine v0.24.0 worktree and applied tree is byte-identical to branch (diff=0); scheduler/model_runner integration diff reviewed — surgical, matches design hooks. Strict review dispatching.
+- Strict review (task-mrg68puk, 14m59s): FAIL — 0 Blocking / 3 Major / 1 Minor. M1 writer-thread silent death + shutdown deadlock (opprof.py:106/145); M2 apply.sh already-applied bypasses base/dirty checks; M3 foreground raw lists violate no-raw-list hotspot contract (opprof.py:182). Minor: missing zero-token+None-stat test; true Scheduler integration test DEFERRED to P2 smoke gate (no local vllm install). Reviewer verified OK: fail-fast log_stats ValueError, pairing across sync/batch-queue/PP/spec-decode, schedule-time snapshots, DP-unique paths, patch-id equality 3/3. Orchestrator adjudication: ACCEPT all; fix worker dispatching (fresh task — resume-last would hit reviewer thread).
+- P1 fix round CLOSED (task-mrg6tnum, 13m34s): branch tip 668cfb7e (3 clean commits on ee0da84a). Orchestrator closure: 14/14 tests reproduced; M3 raw-lists-gone verified (bisect accumulation); BOTH apply.sh refusal cases independently reproduced (dirty exit=1; wrong-base-with-identical-patches exit=1); M1 writer failure handling inspected (one-time log, liveness check, bounded close, drop visibility). P1 ACCEPTED.
+- ECHO P2 (GPU, dash0): deploy patched vLLM 0.24.0 (python-only patch, VLLM_USE_PRECOMPILED path) on dash0, serve Qwen3-30B-A3B BF16 TP1 on ONE free H20; artifacts smoke + overhead gate (interleaved 5xON/5xOFF identical load, 3% gate on 95% CI upper bound) + one 2+8 Layer-2 profiler window + integration assertions (records==steps, no pending leak, footer accounting). Est 1.5-2.5h wall, single GPU. Deferred review item (real Scheduler integration) is covered by these assertions.
+- P2 dispatched: task-mrg7erq5 (dash0 smoke; ssh dash0 ONLY authorized; hard gates: artifacts/schema/footer/no-pending-leak, overhead 95%CI-upper <=3%, Layer-2 window loadable). Est 1.5-2.5h.
+- P2 first run (task-mrg7erq5, 1h37m): overhead gate FAIL honored (13.11%, CI [6.24,19.53]) — but measurement judged INVALID by orchestrator: OFF-arm spread 29% across identical runs, one pair NEGATIVE overhead, 10s window with cold-start per run, ON-first order confound (run 1 = coldest = ON). Physical bound: per-step cost is bisect+encode+enqueue at ~11 steps/s. Artifacts gate passed except PIECEWISE not exercised by smoke load. 14 remote tests pass; TRITON backend 12/12; cleanup verified. AMENDMENT A-P2-1 (pre-registered before rerun): warmup excluded, >=120s steady-state window, ignore_eos fixed output lengths, ABBA counterbalanced order + discard first pair, host load recorded, recorder microbenchmark as secondary evidence, one PIECEWISE-exercising artifact run. GATE UNCHANGED (3% on 95% CI upper).
+- P2 A-P2-1 rerun (task-mrgdbfju + predecessor, valid measurement): overhead gate REAL FAIL — 4.1816%, CI [3.1364, 4.7117], gate 3%. ON tight (28.18-28.26), OFF (28.80-29.64), pairs 2.07-4.74%. Artifacts PASS incl. PIECEWISE addendum (129 records, accounting balanced). Layer-2 correctly not run. LOCALIZATION PUZZLE: producer microbenchmark 29.1µs/step x 9.7 steps/s ≈ 0.04s/run vs observed ON-OFF gap 5.9s — direct recording cost explains <1% of overhead; structural suspect (capture scan @200 concurrency, writer-thread GIL, queue locking, flush I/O locus). GPU spend so far 117 H20-min. Diagnostic+fix round dispatching (stage-bisection first, minimal fix, gate rerun).
+- Bisection round (task-mrge67ke, 34m9s): recorder pipeline exonerated — all 4 stages ~0.5% vs stage-off; BUT all stages kept VLLM_OPPROF_DIR set → gpu_model_runner CUDAGraphStat construction/propagation path active in all; cross-attempt (attempt-2 true-OFF 29.45 vs bisection off 28.39) localizes ~3.6% to that path. Worker honored outside-OpProf stop boundary; temp diffs reversed; 14/14 tests both sides. Orchestrator hypothesis for next round: CUDAGraphStat in ModelRunnerOutput poisons msgspec IPC fast path (pickle fallback for whole output) OR upstream cudagraph_metrics feature carries inherent cost. Next: STATIC analysis first, then 3-trial confirm (true-baseline / upstream-metrics-only / env-set-recorder-off), then targeted fix.
+- P2 CLOSED: ALL GATES PASS (task-mrgfgedp, 2h17m). ROOT CAUSE of the 4.18%: VLLM_OPPROF_DIR entered compile_factors() (envs.py:2011-2105) → hashed into torch.compile cache path (backends.py:1024-1065) → every unique ON dir = cold graph/AOT cache (compile 36.41s vs 6.07s) and ~4% slower artifacts. Recording itself was NEVER the cost (recorder 29.1µs/step; serializer-poisoning hypothesis WRONG — TP1 in-process executor, msgspec handles CUDAGraphStat natively). 3-trial confirmation: baseline 29.72 / upstream-metrics-only 29.64 (0.25%) / env-set 28.45 (4.25%). Fix: +1 production line (ignore-list) + tests; tip bbfa717; 5-patch series; 15/15 tests x3 environments. FINAL GATE: overhead -0.042%, CI [-0.174, +0.046] — PASS. Layer-2 PASS (16.4MB Kineto, 2+8 window, 790,843 events, 7,367 kernel events; active-window perturbation 51.3% — confirms sampled-only design). Artifacts PASS (PIECEWISE 129/139). Orchestrator verified: fix commits, ignore-list diff, 15/15 local rerun, 5 patches. Caveat noted: attempt-4 JSONLs footerless (simultaneous shutdown) — excluded from accounting. UPSTREAM FINDING worth reporting: any per-run-unique env var hashed into compile factors silently causes cold-compile + slower artifacts.
+- P3 dispatching: protocol-first (worker drafts docs/opprof/phase3-protocol.md, STOPS for orchestrator review before execution).
+- USER DIRECTIVE (2026-07-12): parallelize P3 across 8xH20, one experiment per GPU (~8x speedup). Orchestrator caveat: no GPU contention but shared host CPU/memory-bandwidth (API server/tokenizer/scheduler/bench are host-side; P2 showed host-side effects can fabricate %-level differences; saturation-point calibration depends on absolute throughput). P3 protocol must include: pre-registered co-location validity check (same cell solo vs alongside 7 neighbors; throughput + op-share deltas <2-3% to authorize 8-way; fallback 4-way + revalidate), CPU affinity pinning per server/client pair, and host load recording per run.
+- P3 protocol (task-mrh6vfqp, 27m13s, 835 lines): ACCEPTED with orchestrator amendments. Dispositions on 6 open decisions: (1) fixed-duration client build+test APPROVED; (2) P10 private-trace transfer to dash0 CPFS wjh APPROVED (sampling_u<=0.125, input<=32768, output cap 256; no prompt text in artifacts; data returns to same org cluster it came from); (3) MNS{64,1024}xMBT{2048,8192} sparse factorial APPROVED; (4) P10/C00 sole TP2 counterpoint APPROVED; (5) 60% load / burst-8 / 240s / 4 windows APPROVED; (6) kernel mapping + 70% classifiability gate + thresholds APPROVED. AMENDMENT A-P3-1 (user directive): 8-way GPU parallelism, one cell per GPU; pre-registered co-location validity check (same cell solo vs with 7 neighbors, throughput AND op-share deltas <3% to authorize; fallback 4-way + revalidate); CPU affinity pinning (disjoint core sets per server+client); saturation calibration under the SAME co-location regime as measurement; host load + clocks per run; revised wall estimate ~1.5-2h. AMENDMENT A-P3-2 (E2 lesson): execution via detached resumable controller script on dash0 (setsid, --resume, state file) so runs survive codex turn deaths.
+- P3 E-a ACCEPTED (task-mrh7wd5x, 1h27m, 3.96 H20-h): co-location gate → 8-way FAIL (throughput Δ=0.000% but op-share shifts up to 4.2pp > 3% threshold), 4-way PASS (shares ≤0.075pp) — AUTHORIZED 4-WAY. Client+controller 12/12 no-GPU tests; provenance hashed. P10 transfer verified (4,011 rows, tokenizer parity 4011/4011 exact, 0600/0700 perms, no prompt text public). Orchestrator decisions: (1) 4-way verdict accepted despite validation-stream footer absence (shares from Kineto, unaffected); (2) shutdown-fix (API-parent-first) GPU quick-verify REQUIRED before E-b matrix; (3) A-P3-3: per-class drain budget — 240s for output-512 burst cells (P04-class), 120s others; (4) A-P3-4: P3 hard stop raised 8→16 H20-hours (user blanket dash0 approval; E-a consumed 3.96).
+- E-b step1 STOP (task-mrhb2dxw, 10m42s): shutdown-fix verification FAILED hard footer gate — vLLM 0.24.0 API shutdown selected mode=abort timeout=0s and SIGTERMed EngineCore before scheduler/opprof teardown; finally-footer never runs. Run itself healthy (5268 reqs, 43.9 req/s, 1701 records, 0 drops, step range contiguous). Matrix NOT launched (correct). Orchestrator decision: footer must not depend on graceful shutdown — (a) 10-min static check for an official graceful path in 0.24.0; if unreliable, (b) checkpoint-footer sidecar in opprof.py (atomic write each flush cadence; accounting gate accepts sidecar when stream footer absent; bounded loss = last flush interval), new commit + re-export + tests, GPU re-verify incl. abort-kill case, then matrix.
+- E-b round 2 (task-mrhbhj9j, ~1h5m): sidecar fix DONE (commit f8b68f24, +195/-7, patch tip 23450fb2, 7 patches, 18/18+12/12 tests; graceful path found: vllm serve --shutdown-timeout N; dual GPU verification PASS incl. SIGKILL with 24ms checkpoint sidecar balancing). Matrix launched, STOPPED at pre-registered drain gate: P10/C01 saturation drain 288.6s > 120s budget (clean window + accounting VALID; drain is post-measurement hang-watchdog only; 0.74 req/s saturated 32k-context queue drains slowly by nature). 3/4 first-wave runs PASS with plausible ordering. GPU 5.47/16. AMENDMENT A-P3-5: P10-class drain budget 600s; drain violations = quarantine-run-and-continue (stop only if >20% of runs violate); P10/C01 existing data re-adjudicated under amended rule if accounting balanced.
+- NETWORK OUTAGE (~19:2xZ 2026-07-12 onward): dash0 AND dash1 unreachable from local (pre-auth timeout) — ingress/network fault, not host failure. Detached matrix controller on dash0 unaffected by design (A-P3-2); worker holds 1/min read-only reachability probe with retained artifact hashes for post-recovery integrity check. On reconnect: verify controller state (expect matrix advanced or complete), resume worker thread if its turn died.
+- E-b round 3 blocked cleanly on ssh outage (task-mrhdu37w): resumable handoff at runs/opprof-phase3/phase3/access-blocker-20260712.json; last durable 8/52 accepted, 0 quarantine, 0 clean-failures, 6.82 H20-h; controller PID 2237019 untouched, likely still advancing locally. Also delivered mid-flight: P03 profile-control starvation client fix (13/13 tests) + frozen analyzer analyze_phase3.py (4/4 tests). Reconnection watcher armed.
+- NETWORK RESTORED (~21:1xZ per user). Controller advanced 8→12/52 during outage then FAILED ~2.2h ago (idle since). GPU 8.13/16. E-b worker resumed (task-mrhkbq8s): integrity check vs retained hashes → diagnose controller failure → minimal fix → --resume matrix (40 runs remain) → frozen analysis.
+- E-b round 4 (task-mrhkbq8s, ~2h): 40/52 accepted (20/24 cells), 0 clean-window failures, 542,350 L1 records, 72 L2 traces, GPU 13.31/16. Two aux-gate fixes shipped (+58/-11, 15/15+4/4 tests): profiler-control connection, clean-window failure boundary (post-clean disconnects wrongly counted), 3-sample GPU preflight. FINAL BLOCKER: P10/TP2 warm-up gate demands 32/32 completions; long-context prompts cannot finish in budget (15/32, 21/32) though clean windows are valid. Same failure class as drain budget: auxiliary gate miscalibrated for long-context pattern. AMENDMENT A-P3-6: P10-class warm-up gate = 32 completions OR (>=16 completions AND Layer-1-verified throughput stabilization over trailing warm-up steps), with mandatory post-hoc stability evidence in the report. Finish remaining cells within 16h cap; if cap hits, stop and analyze with documented gaps.
+- E-b round 5 (task-mrhnw1rm, 23m46s): P10/TP2 FAILED frozen A-P3-6 stabilization on fresh data (drift 36.76% vs 10%; bins 11/10/16 vs 16) — the cell genuinely does not stabilize; clean window itself valid. Worker correctly refused to relax. GPU 14.03/16. ORCHESTRATOR DECISION: freeze matrix at 40/52 runs / 20/24 cells; abandon remaining 12 runs. AMENDMENT A-P3-7: per-contrast evaluation — each frozen contrast evaluates iff both cells complete; missing → NOT EVALUABLE (never imputed); H1a/H1b use existential logic on completed cells (CONFIRM possible, REFUTE NOT possible at 20/24 — must be stated explicitly). P10/TP2 non-stabilization recorded as a pattern-conditioned operational FINDING.
+- PHASE 3 FINAL (task-mrhos386, 19m37s): **H1a INCONCLUSIVE / H1b PASS (5 of 6 evaluable contrasts) / compound PARTIAL** under A-P3-7. H1b effects large + Holm-corrected p≈0: P10 real trace R64 +44.79pp vs both long rectangular controls (efficiency −14.3%/−44.7%); P09 production mix R64 +39.62pp, padding +6.67pp (−8.3%); P06 bimodal burst R64 +23.0/+35.4pp (−11.6%/−22.8%). H1a inconclusive because only 1/9 patterns Layer-2 windows passed representativeness gates (descriptive: P06/P10 flip attention-led↔MoE-GEMM-led across load points). Orchestrator verified metrics.json per-contrast records match report exactly. GPU final 14.03/16 (1.97 unused). P4 dispatching.
+
+## Phase 5 — mechanism-decomposition ablations (USER-APPROVED 2026-07-12)
+- Motivation (user challenge): the sign of H1b is obvious; the contribution is the mechanism LEDGER. vLLM does no request-level padding (continuous batching); measured bucket padding (+5.4-6.7pp) explains ~1/6 of the 14-45% E_token gap; the remaining ~30pp attribution (ragged-attention SM imbalance / cudagraph bucket mismatch / chunked-prefill mix interference / MoE routing skew / scheduler batch-variance) is unmeasured in the literature.
+- Design skeleton (protocol to formalize): controlled ablations each isolating ONE mechanism on P10 (primary) + P09/P06 (secondary): (i) length-sorted/binned replay [kills intra-cohort raggedness, keeps content+totals]; (ii) capture sizes matched to measured decode-B distribution [kills bucket mismatch; doubles as the config-tier deliverable]; (iii) arrival-shape ablation [steady vs burst, same lengths]; (iv) prefix-caching on/off [C structure]. Accounting must include an explicit interaction/residual term — shares are NOT forced to sum to 100%.
+- Metric: E_token under the P3 discipline (C00-TP1, rho=0.60, 240s clean windows). New GPU cap: +6 H20-hours for P5 (P3 cap closed at 14.03/16).
+- Gates: protocol-first (orchestrator review before any GPU run); P4 acceptance re-scoped — speculative recovery claims in P4 must defer to P5 measured shares.
+- P5 protocol (task-mrhq3ud3, 13m49s): ACCEPTED. Data-grounded specifics verified by worker from P3 raw Layer-1: P10 decode-B support is {1..7} (17,941 pure-decode steps) → A2 capture sizes {3,5,6,7}; A1 = 32-request reorder blocks (R16 0.6417→0.4734, −26.2% rel), 142-request slice @0.4725 req/s, 64s fairness cap. Orchestrator dispositions on 5 open decisions: (1) BLOCKING → approve recorded-arrival BRIDGE ledger (production-faithful estimand; explicit not-literal-P3-decomposition limitation); (2) dual P03/P04 control ledgers, dominant must hold under both — approved; (3) A1 parameters — approved; (4) 3 replicates/arm, P3-control reuse behind 3% bridge gate, optional tier within 6.0 H20-h — approved; (5) Layer-1-only primary, routed-expert telemetry analysis-only no causal share — approved. Execution authorized.
+- P5 wave 1 (detached exec, 30min, 0.65 H20-h): HARD STOP correct — all 4 attempted arms failed frozen warm-up stabilization (drift 190.6-216.5% recorded arms, 13.2% uniform arm, gate 10%; completions 25-28/32). ROOT CAUSE: semantic mismatch — throughput-drift gate designed for saturation/steady loads is wrong for rate-following arms whose throughput follows a non-stationary recorded arrival BY DESIGN. Purpose of warm-up = no cold-start contamination, not stationarity. Bonus signal: 190% vs 13% drift gap is itself direct arrival-mechanism evidence. AMENDMENT A-P5-1: rate-following arms use cold-start-artifact gates (all compile/capture events before clean window per Layer-1 graph-mode series + >=16 warm-up completions incl >=1 long-context + zero first-occurrence capture events inside clean window); drift gate retained for saturation arms only. Budget 5.35 remaining.
+- P5 round 2 (detached, ~70min, 2.44/6.0 H20-h): 15/15 primary valid under A-P5-1, bridge PASS (0.269%). OFFICIAL ledger INCONCLUSIVE per frozen rules (only A1-under-P03 evaluable: share 0.038, CI [-0.20,0.23]; A2/A3/A4 manipulation-failed; P04 denominator unstable 45% nonpositive draws). REAL FINDINGS: (1) P3 P10-vs-P04 gap was largely a MATERIALIZATION ARTIFACT — P5 recorded-arrival base E=3.014 ≈ P04 control 3.055 (gap 0.041 CI spans 0), while uniformizing arrival (as P3 did) drops E to 2.626 (−12.9%, raw p=0.014, Holm 0.055): burstiness HELPS at low rate via batch formation (decode-B CV 0.87→0.66 under uniform but waiting CV rises); P5 A3 reproduces P3 base within 0.27% — bridge design caught our own artifact. (2) All four intuitive mechanisms ≈0 at rho=0.60: raggedness 3.8% n.s., capture-fix +1.6% n.s. (P4 independent validation +0.18%, padding bound matched within 0.002pp), prefix ~0. Remaining P10-vs-P03 gap (~36%) explained by NONE — workload physics, not recoverable waste at this regime. (3) P4 ranked list: #1 prefix affinity +82.14% ceiling (P08 vs P07), #4 pattern-specific MNS64 has a −24.27% P01 counterexample (sign flips by pattern). OpProf campaign phases P0-P5 all complete; ~16.5 H20-h total.
+
+## Phase 6 — cross-version churn quantification (USER-APPROVED 2026-07-13 "请你测试一下")
+- Question: does the vLLM 0.20.0 C1 12-cell TP×MNS optimum/ranking survive on 0.24.0? Paired-surface churn evidence for the paper motivation (matrix × real-eval cost × churn).
+- Ground truth: C1 recovered stores (verified): TP1 [2.10,2.35,2.283,2.283], TP2 [2.275,2.275,3.283,3.258], TP4 [1.283,2.442,2.442,2.442] req/s/GPU; best TP2/MNS32=3.2833; trap TP4/MNS16; per-cell recorded anchors + request counts in replayserve docs/assets/simfid_s2r/ground_truth.json; workload = materialized chat_w20260311_1000 (local + dash0 CPFS), output override 128, scale 0.1, pass>=0.95, stepped TTFT 2/4/6s, TPOT 50ms; caps 512 TP1/TP2, uncapped TP4.
+- Budget: 3.0 H20-hours hard cap → anchor-subset design required (TP4 cells cost 4 GPU-h per wall-hour). Covariate to note: C1 ran on dash1, P6 on dash0 (same H20 class).
+- Gate: protocol-first, orchestrator review before GPU.
+- P6 protocol ACCEPTED: 92/92 historical anchor counts reproduced locally pre-GPU (comparability verified); adaptive 25-anchor design (12 peak + 11 adjacent + 2 trap-bracket), 2.65 primary + 0.35 confirmation reserve = 3.0 cap exactly. Decision rules: feasibility flip / >5% drift / floor-bucket argmax / tau-b>=0.8 ranking survival / trap escape rule / 2-of-3 confirmation. All 5 open decisions APPROVED incl. upgrade-path estimand (resolved defaults = churn; dash1→dash0 + colocation = stated limitations) and warmup long-tier adaptation raw>4096. Execution dispatched.
+- P6 W1 hard stop (0.43 H20-h spent): 8 replays REJECTED for missing in-stream footers (wrong shutdown path) + redo projection 3.03 > 3.0 cap. Orchestrator diagnosis: P6 validator failed to apply the P5-ACCEPTED sidecar-footer accounting rule (records == sidecar counters within one flush interval). AMENDMENT A-P6-1: (a) sidecar footers are valid accounting per P5 rule — re-adjudicate W1 artifacts WITHOUT rerun if sidecars balance; (b) controller must use the graceful path (vllm serve --shutdown-timeout, found in P5) for subsequent waves; (c) IF W1 salvage fails, cap raised 3.0→3.5 as fallback (worker recommendation, within user posture).
+- P6 r2 partial (2.29/3.0 H20-h, hard stop before W4): 21/25 anchors, 10/12 cells. Signals: TP2/MNS64 >=29.4% DOWN (left-censored), TP1 cells right-censored UP, TP2 family down — surface shape moved opposite directions. CONFOUND (worker confirmation runs): co-location flips SLO-frontier pass rates (0.41→1.00, 0.03→0.96 solo; waiting 8.5→0.35) — co-location valid for throughput/op-shares (P3) but NOT for pass-rate cliffs; 0.20 baseline was solo. USER DECISION: cap raised to 6.0; A-P6-2 = frontier anchors SOLO placement; finish W4 + solo re-confirm key cells.
+- P6 FINAL (A-P6-2 solo tier complete, 5.64/6.0 H20-h, 66 authoritative trials, 12/12 cells): formal verdicts INCONCLUSIVE per frozen letter (4 cells censored/non-monotonic prevent bounded 12-cell surface; tau_b=null, never manufactured). SUBSTANTIVE PAIRED SURFACE (all solo-authoritative): TP2/MNS64 −29.41% CONFIRMED (3.2583→2.3000 bounded; mechanism: decode-B 11.5-18 + 4.6-9.5% NONE-graph at old anchor); TP1/MNS8 +13.49% and TP1 plateau converges to 2.3833; old best TP2/MNS32 −0.76% (held, highest bounded); TP4 family right-censored UP >=2.50; TP4/MNS16 frontier became NON-MONOTONIC in load. CO-LOCATION DELTA TABLE: up to +92.86pp pass-rate flips (21 exact-request anchor pairs) — co-location validity is metric-dependent (standalone methodological finding). Orchestrator verified −0.2941 drift in metrics.json. Campaign total incl. P6: ~22.2 H20-h.

docs/simulator-fidelity-frontier-20260711.md

@@ -0,0 +1,134 @@
+# Simulator-based tuning 保真度实验总结（Frontier / H20）— 供 review
+
+日期：2026-07-11。目的：判断现有实验数据是否**严格**支撑"simulator-based
+tuning 会导致错误的 rank 排序和最终性能 gap"这一论文主张。全部原始材料在
+`~/phd/replayserve`（见文末指针）；本文只做总结，不新增任何数字。
+
+## 一句话结论
+
+在给足模拟器一切有利条件（同模型同硬件的 H20 算子 profile、逐 token 精确的
+真实 workload、在独立数据上冻结的逐 TP 吞吐校准）之后，Frontier 在预注册判定
+规则下仍未通过：在真实调优器实测过的 12-cell TP×MNS 面上，模拟器选出的配置
+比真实最优差 **30.46%**（top-1 real-evaluated regret），关键交互关系只复现
+3/6。**若按模拟器结果部署，你会选 TP1/MNS64（真实 2.283 req/s/GPU），放弃
+TP2/MNS32（真实 3.283）——这就是"最终 gap"的操作性含义。**
+
+## 实验设置（三个不可辩驳性设计）
+
+- **Ground truth 是真实调优研究本身**：C1 交互研究（dash1 恢复的 store，
+  vLLM 0.20.0，H20，`chat_w20260311_1000` 窗口，scale 0.1），12 个
+  TP{1,2,4}×MNS{8,16,32,64} cell 的 SLO-feasible peak（pass≥0.95，阶梯
+  TTFT 2/4/6s，TPOT 50ms）。每个数字从原始 state.json/engine.log 独立复算。
+- **EXACT workload**：materialized JSONL（32,606 行，含真实 prompt 文本）
+  逐 token 重建——tokenize 后逐行断言等于 `input_length`（17,710/17,710 全
+  过），chat 模板开销恒为 +8 token 并分解到具体 token ID，block-16 前缀 hash
+  按 vLLM 0.20.0 的 `hash_block_tokens` 源码（vendored、逐字节比对）计算。
+  每个 cell 用它自己实测过的锚点（92 个锚点，选中请求数 92/92 与真实记录
+  相等），不插值、不补网格。
+- **预注册**：判定规则在任何模拟跑之前冻结于
+  `replayserve/docs/simfid_s2r_protocol.md` §6——(1) 最坏 top-1 regret ≤5%；
+  (2) TP4/MNS16 陷阱 6 条关系全复现；(3) 92 次留一锚点复检全过。四种读法中
+  只有"冻结校准 + 吞吐代理"承载结论，其余为诊断。校准系数
+  a_tp = {TP1: 0.7235, TP2: 0.4681, TP4: 0.3521} 来自独立的 3 配置切片
+  （S2-E），S2-R 数据上不重拟合。
+
+执行：184 次 Frontier CPU run（92 锚点 × 未校准/校准），184/184 通过、
+0 失败，sanity 无红旗。
+
+## 核心证据：12-cell 分数表（req/s/GPU）
+
+| Cell | 真实 SLO peak | 校准后 sim 吞吐 | 校准后 sim SLO |
+|---|---:|---:|---:|
+| tp1_mns8  | 2.100 | 2.173 | 1.717 |
+| tp1_mns16 | 2.350 | 3.242 | 2.383 |
+| tp1_mns32 | 2.283 | 4.297 | 2.383 |
+| tp1_mns64 | 2.283 | **4.357** ← sim 判为全局最优 | 2.383 |
+| tp2_mns8  | 2.275 | 2.039 | 1.742 |
+| tp2_mns16 | 2.275 | 2.244 | 2.300 |
+| tp2_mns32 | **3.283** ← 真实全局最优 | 3.650 | 3.750 |
+| tp2_mns64 | 3.258 | 3.650 | 3.750 |
+| tp4_mns8  | 1.283 | 1.545 | 1.321 |
+| tp4_mns16 | 2.442 | 2.437 | 2.500 |
+| tp4_mns32 | 2.442 | 2.462 | 2.500 |
+| tp4_mns64 | 2.442 | 2.462 | 2.500 |
+
+**失败机理**（不是尺度错，是形状错）：真实 TP1 在 MNS16 后随 SLO 边界饱和
+（2.35→2.28），sim 认为吞吐随 MNS 单调上升（3.24→4.30→4.36）。逐 TP 校准
+已消掉尺度误差，负载响应形状仍然错——来源是排队/尾延迟/调度开销，不是算子
+时间表。佐证：sim TTFT p95 仅为真实的 0.30–0.38，TPOT p95 0.63–0.79
+（S2-E 持出集）。
+
+排序质量：校准吞吐读法 τ-b = 0.448（未校准 0.236），成对方向正确率
+68–73%。远低于可用水平。
+
+## 什么已被严格证明（本文主张的边界内）
+
+1. **在被测面上，simulator-only 的吞吐排序会给出 30.46% 的真实 gap**——
+   端到端、预注册、workload 逐 token 精确、同硬件 profile、校准冻结。链条
+   里没有"我们没给模拟器机会"的空隙。
+2. **未做延迟校准的模拟器做 SLO 判断不可信**：S2-E 直接反例（sim TPOT p95
+   46.43ms vs 真实 71.38ms，横跨 50ms SLO → sim 判可行、真实不可行）；
+   S2-R-b 校准 SLO 读法在 92 锚点上有 21 假可行 / 7 假不可行。
+3. **覆盖缺口独立于精度成立**：MoE EP>1 无 profile 支持、GMU 惰性、
+   CUDA-graph 未接、scheduler-delay 不可表达——这些 knob 模拟器根本无法
+   评估（replayserve/docs/simfid_inventory.md 旋钮矩阵）。
+
+## 什么还没被严格证明（review 时请重点判断这三条）
+
+1. **跨引擎版本 profile**（最大攻击面）：H20 profile 来自 vLLM 0.11.1 时代
+   的对齐工作，ground truth 是 0.20.0。防守方可以说"按 0.20.0 重新 profile
+   就好了"。我们的回应有二：(a) 冻结校准已吸收逐 TP 尺度误差，剩余的是形状
+   误差，其来源（排队/调度/CUDA-graph）不在算子表里；(b) 反身性论证——若
+   每个引擎版本 × 硬件都要重 profile + 重校准，profiling 本身占用同款 GPU
+   跑真实负载，模拟器的成本优势即被churn吃掉。但 (a) 目前是机理论证 +
+   间接证据，不是实验闭环。
+2. **SLO 门控读法的意外成功**：校准 + SLO 门控（预注册为仅诊断）达到
+   regret 0–0.76%、τ-b 0.967、陷阱 6/6。审稿人可主张"你选错了读法"。
+   我们的回应：该读法建立在错误的逐锚点判定上（21 假可行/7 假不可行），
+   正确性来自误差在逐 cell 峰值处的部分抵消，无法保证泛化；且它是事后
+   观察，预注册规则不允许换读法。**但要诚实：这条把可主张的结论从
+   "模拟器必然错排"弱化为"模拟器排序不可信、必须真实验证"。**
+3. **单面、单模型、单硬件、单 workload 窗口**：无跨 workload 复制。且论文
+   审稿人点名的多半是 Vidur，我们测的是 Frontier（同类：算子 profile +
+   事件驱动调度模拟；需论证类代表性或列为 limitation）。
+
+## 建议的论文主张口径（可辩护版本）
+
+> 不主张"simulator 总是错排"，主张：在我们的真实调优器必须绕开局部陷阱的
+> 那个交互面上，一个被给足条件的模拟器（同硬件 profile + 吞吐校准 + 精确
+> workload）在其预注册的最优读法下产生 30% 的部署 gap，且其 SLO 判定在
+> 无延迟校准时存在跨边界的假可行；因此 simulator-only tuning 的结果不经
+> 真实评估不可信——而真实评估的成本控制正是 AITuner 的贡献。混合设计
+> （模拟器粗筛 + 真实终判）是被我们的诊断数据支持的未来方向，且"何时必须
+> 真实评估"仍由相似度度量回答。
+
+## 可选补强（按闭环价值排序）
+
+1. **负载响应形状分析**（零新模拟，复用 184 run + 真实 probe history）：
+   逐 cell 对比归一化吞吐/延迟-锚点曲线。若形状系统性不匹配，则证明任意
+   逐配置尺度校准（= 任意精度的算子 profile）原理上无法恢复排序，直接
+   封死上面第 1 条攻击面的一半。
+2. **vLLM 0.20.0 重 profile**（需 GPU 时间 + 审批）：实验性封死跨版本
+   质疑，但注意这同时演示了 churn 成本，输赢都有叙事价值。
+3. **第二个 workload 窗口**（如 coder 或 2200 slot）：复制性。
+
+## 数据 sanity block
+
+- 12-cell 真实向量：n=12，min/max=1.283/3.283，8 个 distinct 值。
+- 校准 sim 吞吐向量：n=12，min/max=1.545/4.357，10 个 distinct。
+- 执行：n=184，失败 0，请求数 min/max=66/600（34 distinct）。
+- regret=30.456853% 由 1−2.2833/3.2833 独立复算精确一致；a_tp 三值与
+  S2-E 冻结清单一致；所有比率在 [0,1]；无逐配置全同向量。
+- 已知异常（保留未修饰）：真实 TP2/MNS32 与 MNS64 的 pass-rate 非单调，
+  但 0.95 可行性截断有序。
+
+## 材料指针（全部在 ~/phd/replayserve）
+
+- 最终综合报告：`docs/simfid_s3_fidelity_report.md`
+- 12-cell 结果与全部指标：`docs/simfid_s2rb_results.md`、
+  `runs/simfid_s2rb/results/metrics.json`
+- 预注册协议 + 修正案：`docs/simfid_s2r_protocol.md`
+- 吞吐校准与延迟反例：`docs/simfid_s2e_report.md`
+- 数据清单与旋钮覆盖矩阵：`docs/simfid_inventory.md`
+- 全程决策/验收台账：`docs/simfid_campaign_state.md`
+- 恢复的 EXACT workload：`~/phd/aituner/trace_windows/traces/chat_w20260311_1000.jsonl`

docs/telemetry-residual-tuning-roadmap-20260714.md

@@ -0,0 +1,286 @@
+# Telemetry-conditioned residual tuning roadmap
+
+Status: **R0 COMPLETE / FAILED; R1 AND R2 CLOSED FOR THIS MODEL**.
+
+Date: 2026-07-14 (Asia/Singapore).
+
+## Research question and claim boundary
+
+The question is whether a small number of real engine observations can correct
+a simulator's task-specific error over **unmeasured configurations**, and
+whether that correction reduces the real-GPU cost of finding a high
+SLO-goodput serving configuration.
+
+The intended headline claim, if the evidence supports it, is:
+
+> An engine-state-conditioned residual model turns a simulator prediction into
+> a task-specific posterior over unmeasured serving configurations, allowing a
+> sequential tuner to reach near-oracle SLO-goodput with materially fewer
+> H20-hours than simulator-only and outcome-only tuning.
+
+Classification accuracy, simulator-error diagnosis, and telemetry overhead are
+supporting evidence.  None is an end-to-end tuning contribution by itself.
+
+The following method is closed and will not be revived under another name:
+per-candidate five-second accept/reject as the headline contribution.  The P1
+result showed only 1.426% cost reduction in the frozen `k=2` workflow.
+
+## Two models, one evaluation
+
+Both branches use the same legal candidate set, real measurements, task split,
+cost accounting, and acquisition function.
+
+### Simulator-residual branch (primary)
+
+For measured anchor `c_t` and unmeasured candidate `c'`:
+
+```text
+y_hat(c') = y_real(c_t)
+          + [y_sim(c') - y_sim(c_t)]
+          + f(state_real(c_t) - state_sim(c_t), c' - c_t, workload, SLO)
+```
+
+The simulator delta is the prior.  The learned model may correct it only with
+training-supported state/config transitions; uncertainty or distribution shift
+must shrink the correction back toward the simulator prior.
+
+### Telemetry-only branch (mandatory)
+
+```text
+y_hat(c') = y_real(c_t)
+          + g(state_real(c_t), c' - c_t, workload, SLO)
+```
+
+This branch tests whether the simulator is actually necessary.  It does not
+use a hand-authored bottleneck-to-knob rule.
+
+### Search policy
+
+Legal configurations are enumerated independently of telemetry.  A generic
+cost-aware acquisition rule ranks candidates from predicted improvement,
+uncertainty, and measured H20 cost.  The current production harness's
+bottleneck scores, topology-first ordering, and hand-set relief constants are
+not consumed by either branch.  The validator may enforce legality,
+full-config no-repeat, failure accounting, and resource caps only.
+
+## Hypotheses
+
+| ID | Hypothesis | Direct test | Failure meaning |
+|---|---|---|---|
+| H0 | Existing artifacts can express a common, direct-measurement state without heuristic labels. | Engine/simulator extractor coverage and invariants. | Route is not currently implementable. |
+| H1 | Simulator errors are predictable from engine/simulator state discrepancy at measured anchors. | Task-held-out pairwise inversion correction and new-inversion rate. | Telemetry is diagnostic but cannot correct the surface. |
+| H2 | Telemetry alone predicts useful config transitions beyond outcome-only history. | Telemetry-only versus real-outcome-only sequential replay. | Direct telemetry-guided tuning has no independent value. |
+| H3 | Residual correction changes actual tuning decisions and cost. | H20-hours to 95% oracle and regret AUC against the strongest safe baseline. | No system contribution even if H1/H2 prediction metrics improve. |
+
+## Common-state contract
+
+Only directly observed or exactly reconstructed quantities are admitted.
+
+| Quantity | vLLM Layer-1 | Frontier | R0 status |
+|---|---|---|---|
+| Scheduled requests / batch size | Per scheduler step | Existing per-batch metric, disabled in P1 output | Common after CPU replay |
+| Scheduled prefill/decode tokens | Per scheduler step | Existing per-batch metrics | Common after CPU replay |
+| Scheduler/batch rate | Monotonic step timestamps | Batch count / simulated duration | Common after CPU replay |
+| Waiting queue area | Time-weighted queue gauge | Sum of request waiting times | Common aggregate |
+| Running request area | Time-weighted running gauge | Sum of E2E minus waiting time | Common aggregate, semantics audited |
+| Preemption count | Per step | Per request | Common |
+| KV usage/headroom | Exact blocks and ratio | Not in committed output | Engine-only until exact reconstruction exists |
+| CUDA graph mode/padding | Exact per step | Not modeled | Engine-only omitted-mechanism signal |
+| Request TTFT/TPOT/pass rate | Exact real outcomes | Exact simulated request metrics | Common outcome, not state |
+
+Unavailable fields remain null.  They cannot be imputed from a human
+`prefill/decode/queueing` label.
+
+Frontier already contains the required detailed batch and timestamped
+stage-batch ledger output.  P1 disabled it for artifact size.  R0 replays the
+same immutable fixtures with the existing output flags enabled; it does not
+change the simulator model or calibration.
+
+## Data separation
+
+- Phase 6 / `chat_w20260311_1000`: development only.
+- P1 / `chat_w20260312_1000`: development only.
+- R1 / `chat_w20260313_1000`: new development surface.
+- R2: trace windows not used for feature, model, threshold, candidate-space,
+  cutoff, or acquisition decisions.
+- Splits are by complete workload/SLO task.  Anchor- or pair-level random
+  splits are prohibited.
+- Sequential-policy seeds measure algorithmic variability; they are not
+  counted as independent system tasks.
+
+The two existing development tasks have an important limitation: the now-
+available SLO-gated simulator reading already retains the real oracle at its
+top rank/tie.  They therefore cannot establish a positive end-to-end ranking
+claim.  They are used for plumbing, known false-feasible cases, and negative
+evidence.  R1 must be run as an unbiased complete surface, not selected after
+observing simulator success or failure.
+
+## Step-by-step roadmap
+
+### R0.1 — Inventory and roadmap
+
+Deliverables:
+
+- this roadmap;
+- rolling untracked `ONGOING.md`;
+- exact engine/simulator field and artifact inventory.
+
+Gate: every claimed input has an authoritative file path and provenance.
+
+### R0.2 — Common-state plumbing
+
+Deliverables:
+
+- `runs/telemetry-residual/common_state.py`;
+- synthetic correctness tests;
+- one exact P1 Frontier replay with individual batch metrics and the full
+  stage-batch ledger enabled;
+- paired engine/simulator state summary for the same fixture.
+
+Gate:
+
+- replay request count and SLO scorer exactly agree with the committed replay;
+- batch/ledger outputs are non-empty;
+- all counters are non-negative, ratios bounded, times monotonic;
+- no GPU is visible to Frontier;
+- output volume is practical before expanding to twelve replays.
+
+### R0.3 — Development residual/headroom audit
+
+Use all frozen P1 primary fixtures and corresponding engine intervals.  Produce:
+
+- common-state residuals per anchor;
+- simulator-error labels and continuous SLO/goodput residuals;
+- ordered source/target diagnostic that removes both config identities from
+  both roles in every training fold;
+- oracle upper bound for cross-candidate correction;
+- explicit comparison with simulator+outcome and telemetry-only features.
+
+R0 is a feasibility gate, not headline evidence.  Proceed to R1 only if:
+
+1. state features are collected with the measured source anchor, vary across
+   cells, and are available before any target config is evaluated;
+2. at least one known simulator error has a state discrepancy not exposed by
+   the matched external prefix outcome;
+3. a prior-preserving model can correct development errors without introducing
+   a larger number of new errors under regularization sensitivity;
+4. an oracle cross-candidate correction has at least 15% sequential tuning-cost
+   headroom under full startup/warm-up accounting.
+
+### R0 result and decision
+
+R0 completed without a data-validity red flag, but failed condition 3.  The
+decision is **STOP_BEFORE_R1**; no H20 job was launched for this route.
+
+- All 12 detailed Frontier CPU replays exactly reproduced their committed SLO
+  scorers.  Runtime was 23.943--54.786 seconds per replay, detailed artifacts
+  were 4.12--13.53 MB, CUDA visibility was empty, and there were zero failures.
+- The paired surface contains 12 real/sim anchors, two known simulator
+  false-feasible anchors, and 120 legal cross-config ordered transitions.  A
+  fold removes both the source and target TP/MNS identity from source and
+  target roles; the two offered-load anchors remain part of the same task.
+- Raw Frontier feasibility is 83.33% on the repeated transition view.  The
+  structurally correct hybrid model uses
+  `r_target = r_source + delta_r`; the direct model uses
+  `y_target = y_source + delta_y` and never reads simulator fields.
+- Direct telemetry is not robust relative to real-outcome-only: its accuracy
+  delta over L2 `{0.1,1,10,100}` is `{-0.83,+1.67,0,-4.17}` percentage points,
+  and its best absolute accuracy is 54.17%, below the raw simulator's 83.33%.
+- Hybrid telemetry raises classification accuracy over the corresponding
+  simulator+outcome transition regression by 1.67--4.17 percentage points,
+  but worsens pass-rate RMSE by 0.141--0.201 and MAE by 0.084--0.125.  Its full
+  correction reaches only 46.67--53.33% absolute accuracy.
+- Across 24 nonzero `(L2, raw-simulator-prior weight)` combinations, no model
+  both corrects an existing simulator error without more new errors and avoids
+  worsening RMSE/MAE.  Whenever a correction fixes at least one error, it
+  corrupts at least 11 previously correct transitions.
+- A perfect correction could skip the frozen simulator rank-2 real final and
+  save 0.043469 H20-hours: 15.45% of the prospective online `k=2` cost, or
+  14.40% when the prior failed launch is charged.  On this development task the
+  simulator top-1 already is the real oracle with zero regret, so headroom
+  versus the observed-safe top-1 baseline is 0%.
+
+The result does not prove that engine telemetry is useless.  It shows that the
+current one-task anchor-transition evidence cannot support either a safe
+simulator-residual tuner or a simulator-free telemetry tuner.  A larger model
+or an R1 run would add capacity/data after a failed gate and is therefore not
+authorized under this roadmap.
+
+### R1 — New development surface
+
+Status: **NOT LAUNCHED; CLOSED BY R0**.
+
+Frozen starting setup:
+
+- host: dash0, eight NVIDIA H20 GPUs;
+- cells run solo; no co-location for SLO verdicts;
+- patched vLLM 0.24.1.dev3, Qwen3-30B-A3B BF16;
+- trace: `chat_w20260313_1000`;
+- output tokens: exactly 128;
+- SLO: stepped TTFT 2/4/6 seconds, TPOT 50 ms, pass rate at least 0.95;
+- config surface: TP `{1,2,4}` × MNS `{8,16,32,64}`;
+- hard campaign cap: 4 H20-hours.
+
+The load ladder, repetitions, randomized order, exact commands, expected wall
+time, and artifact paths are frozen only after R0.  A resolved echo is required
+before launch.
+
+R1 passes only if a frozen sequential replay shows at least 15% E2E H20-hour
+headroom over the strongest safe baseline with final regret at most 5%.  R1 is
+development evidence and cannot be reported as the held-out result.
+
+### R2 — Held-out sequential tuning
+
+Status: **NOT LAUNCHED; CLOSED BY R0**.
+
+Required baselines:
+
+1. random search;
+2. real-outcome-only Bayesian/sequential search;
+3. Frontier ranking plus real top-k final;
+4. simulator plus real-outcome residual;
+5. telemetry-only transition tuner;
+6. simulator plus telemetry residual tuner;
+7. complete real surface as oracle, not as a cost competitor.
+
+Primary metric: end-to-end H20-hours to first reach 95% of the real full-surface
+SLO-goodput oracle.  Secondary metrics are cost-normalized regret AUC, final
+regret at fixed budgets, oracle false-prune, wall time, and per-task regressions.
+
+The route is successful only if the winning telemetry method reduces the
+primary cost by at least 20% versus the strongest safe baseline and ends within
+5% regret on every headline task.  If hybrid beats telemetry-only by at least
+10%, simulator residual correction is the primary method.  If telemetry-only
+is within 5% or better, the simulator dependency is removed.  If neither clears
+the contribution bar, the route is closed and telemetry remains a diagnostic
+facility only.
+
+## Cost discipline
+
+- R0 simulator work is CPU-only and must set empty CUDA visibility.
+- R1 cannot exceed 4 H20-hours.
+- R2 receives no budget until R1 passes.
+- Startup, warm-up, burn-in, failed launches, real probes, continuation, and
+  final validation are charged.  Benchmark-only annotation repeats are
+  reported separately and cannot disappear from campaign accounting.
+
+## Final R0 sanity block
+
+| Data | n | Min | Max | Distinct | Checked invariant |
+|---|---:|---:|---:|---:|---|
+| Phase 6 cells | 12 | TP1/MNS8 | TP4/MNS64 | 12 | Surface not identical; solo SLO tier authoritative |
+| Phase 6 Layer-1 primary steps | 37 streams | 343 | 12,103 | 37 | Contiguous; zero drops |
+| P1 primary anchors | 12 | infeasible | feasible | 2 labels | 7 feasible / 5 infeasible |
+| P1 Frontier runtime | 12 | 24.093 s | 54.575 s | 12 | CPU-only; zero failures |
+| Detailed Frontier replay runtime | 12 | 23.943 s | 54.786 s | 12 | Exact committed scorers; CUDA hidden |
+| Detailed artifact bytes | 12 | 4,123,724 | 13,527,776 | 12 | Non-negative; practical CPU replay size |
+| Cross-config transitions | 120 | real pass 0.1067 | real pass 1.0 | 6 outcomes | Both endpoint config identities held out |
+| State residual vectors | 12 | 16 fields | 16 fields | 12 vectors | Finite; no missing common field |
+| R0 E2E cost values | 4 | 0.237914 | 0.301935 H20-h | 4 | Non-negative; `k=1/2`, online/conservative |
+
+Checked invariants: non-negative counts and costs; pass rates in `[0,1]`;
+simulator results not all identical; exact request count/hash agreement; Layer-1
+step continuity and zero drops; no co-resident SLO measurements; no calibration
+or evaluation split reuse for a future headline claim.  No current red flag
+invalidates R0 plumbing.  The R0 tuning gate itself failed because safe
+prior-preserving correction was absent.

docs/tuning-core-challenges-cost-audit-20260715.md

@@ -0,0 +1,391 @@
+# AITuner tuning：核心挑战、统一成本口径与研究路线
+
+日期：2026-07-15（Asia/Singapore）
+
+状态：**问题定义与历史成本审计完成；新的 tuner 贡献尚未建立。**
+
+## 结论先行
+
+我们不应该把 tuning 定义成“根据当前 telemetry 判断哪个 cap 满了，再调对应 knob”。这个定义同时遗漏了 knob interaction、反事实识别、实验成本和跨任务失配。更准确的问题是：
+
+> 给定模型、engine version、hardware、workload、SLO 和一个声明好的合法配置空间，tuner 如何用最少的真实 GPU 成本，依次选择可能包含多个 knob 的 intervention，找到 SLO-goodput regret 不超过 `epsilon` 的配置？
+
+AITuner 可以形成的系统贡献应当是：
+
+> **一个 intervention-calibrated、action-conditioned、cost-aware 的 tuner：它从真实 engine trajectory 和已测 intervention 中学习联合 config action 的反事实收益分布，并以 cost-to-oracle 而非规则命中率作为目标。Harness 只负责实验语义、合法性、配对、记账和可复现性，不负责用人工 bottleneck rule 决定 action。**
+
+现有结果支持这个问题值得做，但不支持宣称它已经解决：
+
+- 在真实 `TP x MNS` surface 上，one-knob-at-a-time 会停在比 oracle 低 **25.6%** 的 coordinate-wise local optimum。
+- 在 action-aware pilot 中，增加 MBBT 在“几乎从未独占打满 MBBT cap”的情况下仍把 source goodput 提高 **48.0%--77.1%**；因此 `cap -> knob` 不是完整模型。
+- 同一 dash0 任务上，当前 guided harness 到 5% empirical regret 只比纯 LLM 少 **5.85%** H20-hours；到 2% regret 则少 **61.09%**。这说明必须比较完整 cost--regret curve，不能只比较最终最好值。
+- Frontier 的 decision-bearing throughput top-1 在 12-cell surface 上有 **30.46%** real regret。Simulator 本身的边际 GPU cost 是 0，但通过 real-final 恢复 oracle 需要 tie-expanded 4 个真实 cell，即 **0.7828 reconstructed H20-hours**。
+
+## 1. Tuning 问题和成功标准
+
+固定 task context：
+
+```text
+T = {model, engine build, hardware, workload/trace, SLO, legal config space C}
+```
+
+每个完整配置 `c in C` 的目标为：
+
+```text
+f_T(c) = max request_rate_per_gpu
+         subject to request SLO pass rate >= target
+```
+
+有限空间 oracle 为：
+
+```text
+f*_T = max_{c in C} f_T(c)
+regret(c) = 1 - f_T(c) / f*_T
+```
+
+顺序 tuner 在第 `t` 步基于历史 `D_t` 选择一个完整 config intervention：
+
+```text
+a_t = c_t -> c_{t+1}
+```
+
+成功不是“最后找到一个不错的值”，而是同时满足：
+
+1. `regret(best_t) <= epsilon`；
+2. 达到该点之前的 all-in H20-hours 最小；
+3. launch、correctness、SLO 和失败率约束不退化；
+4. 结论在 held-out task 上成立，而不是在用于设计规则的 task 上成立。
+
+### 1.1 GPU cost 的统一定义
+
+未来实验的 task-marginal cost 应定义为：
+
+```text
+C_task = sum_j allocated_GPU_count_j
+                 * (GPU_idle_or_release_time_j - allocation_start_time_j)
+```
+
+它包括 method 实际触发的 startup、warm-up、prefix/full replay、confirmation、failure、cleanup；如果 LLM 思考期间 GPU 仍被占用，也计入。Simulator/模型的一次性 onboarding 成本单独报告：
+
+```text
+C_e2e(N tasks) = C_profile_or_training / N + C_task
+```
+
+另外报告 CPU-hours、LLM API latency/cost，但不把它们伪装成 GPU-hours。构建 benchmark oracle 的 exhaustive annotation cost 是公共评测成本，单独报告，不计入任何方法；同时可给一个将其等量加回所有方法的 conservative view。
+
+历史记录没有 allocation start/release timestamp。本次只能从每个 `engine.log` 的首末时间戳重建：
+
+```text
+C_engine_lower_bound = parallel_size * engine_log_span / 3600
+```
+
+因此下面所有历史 H20-hour 数字都是 **engine-lifetime lower bound**，不是 all-in cost。尤其 simulator 的一次性 H20 operator profiling 成本没有记录，不能称为完全免费。
+
+### 1.2 两种 oracle 必须分开
+
+- **Exact finite-surface oracle**：声明好的 12-cell `TP x MNS` 空间全部真实测量，oracle 是 `TP2/MNS32 = 3.2833 req/s/GPU`。
+- **Broader empirical reference**：dash0 两个 sequential run 中观察到的最好值 `3.35 req/s/GPU`。它包含 surface 外的 MBBT/chunk/GMU action，但只是 best observed，不是全局 oracle。
+
+不能把 empirical best 写成 global oracle，也不能让每个方法使用不同的 oracle 定义。
+
+## 2. 现有方案的 cost-to-oracle 审计
+
+可复算输入和完整结果在：
+
+- `runs/tuning-cost/manifest.json`
+- `runs/tuning-cost/analyze.py`
+- `runs/tuning-cost/metrics.json`
+
+### 2.1 严格同任务对照：纯 LLM vs 当前 guided harness
+
+两组均为 dash0、Qwen3-30B-A3B、community-vLLM 0.20.0、8xH20 可见、`chat_w20260311_1000`、input 0--8k、output 128、replay scale 0.1、TTFT 2/4/6s、TPOT 50ms、pass rate 0.95。除 tuner method 和服务端口外，固定 task spec 相同。
+
+Reference 是两组中 best observed `3.35 req/s/GPU`：
+
+| Method | 到 <=5% regret | 到 <=2% regret | 到 <=1% regret | 完整 run 成本 | 最终 best |
+|---|---:|---:|---:|---:|---:|
+| Pure LLM, no harness | 0.2847 H20h，trial 2，regret 2.736% | 1.1458，trial 6，regret 1.493% | 1.3719，trial 7，regret 0% | 2.2825 | 3.35 |
+| Guided harness v2 | 0.2681 H20h，trial 2，regret 2.736% | 0.4458，trial 3，regret 1.990% | 未达到 | 0.6231 | 3.30，regret 1.493% |
+
+直接结论：
+
+- 5% endpoint：guided 比 pure LLM 少 **5.85%**，不是 material contribution。
+- 2% endpoint：guided 比 pure LLM 少 **61.09%**，有明显 headroom signal，但只有一个 task，不能外推。
+- Pure LLM 在 trial 7 已找到 best observed，之后又花了 `2.2825 - 1.3719 = 0.9106 H20h` 而没有改进，说明 trustworthy stopping 本身就是成本来源。
+- Pure LLM 的 trial 3 使用当前 binary 不支持的 `--expert-parallel-size` 并在 launch 前失败。当前 harness 的 legality/version contract 有实际价值，但它仍不是性能 action-ranking 贡献。
+
+### 2.2 Simulator：零边际 GPU cost 不等于零 tuning cost
+
+Frontier fidelity suite 在 CPU 上执行 184 个 simulation，耗时 **2.055 CPU-hours**，simulation 本身为 0 marginal H20-hours。其对应的 exact dash1 12-cell real surface annotation lower bound 为 **3.5953 H20-hours**。
+
+Decision-bearing `frozen-calibrated/throughput-proxy`：
+
+| Policy | Real cells evaluated | Real-final H20h lower bound | Selected real regret |
+|---|---:|---:|---:|
+| Simulator-only top-1 | 0 | 0 | **30.46%**，选 TP1/MNS64 |
+| Throughput top-1 + real final | 1 | 0.1353 | **30.46%** |
+| Throughput top-2 + real final | 2 | 0.2672 | **30.46%** |
+| Throughput nominal top-3 + real final | tie-expanded 4 | 0.7828 | 0%，找到 TP2/MNS32 |
+
+Post-hoc `SLO-gated` reading 把 `{TP2/MNS32, TP2/MNS64}` 放在 top tie bucket；测两个 cell 需 **0.5156 H20h** 并能找到 oracle。但它不是 preregistered decision-bearing policy，而且 anchor verdict 中有 21 个 false-feasible、7 个 false-infeasible，只能作为诊断上界，不能反写成 prospective simulator 结果。
+
+Pure LLM/harness 数据来自 dash0，simulator exact surface 来自 dash1。模型、engine、trace、GPU type 匹配，但 host 和 campaign 不同。因此两块内部可以直接比较，跨块只能做 development-level 指示；paper 结论必须在同 host、同 task execution protocol 下重跑。
+
+### 2.3 我们要达到的成本目标
+
+在当前 reconstructed lower-bound 口径下，一个有意义的单任务 development bar 是：
+
+| Endpoint | 当前最强同任务 baseline | 20% reduction bar | 兼顾 post-hoc sim+real 的 30% bar | 暂定目标 |
+|---|---:|---:|---:|---:|
+| <=5% empirical regret | guided 0.2681 | 0.2144 | 0.3609 | **<=0.2144 H20h** |
+| <=2% empirical regret | guided 0.4458 | 0.3567 | 0.3609 | **<=0.3567 H20h** |
+
+这两个数字不是 paper result，只用于检查 proposed method 是否有足够 headroom：
+
+- 5% endpoint 已经由 baseline + TP2 两个完整 trial 达到。任何必须先跑 source 再跑 target 的 telemetry tuner 都不能靠减少 trial count 获得 20% 优势；它必须能够 one-shot warm-start、跳过 baseline，或安全地缩短其中一次测量。
+- 2% endpoint 有更合理的结构性空间：从一个 source 直接选择 joint `TP2 + MBBT/chunk` target，可能跳过当前中间 trial；如果仍按当前三次完整 trial 顺序执行，就不会达到 bar。
+
+Paper-facing gate 不使用这些跨 campaign 绝对数，而使用 prospective same-host all-in cost：在每个 held-out task 上 regret <=5%，相对最强 safe outcome-only/current harness 至少省 20%，相对 frozen simulator+real 至少省 30%，并报告 task-level paired confidence interval。
+
+## 3. 四个最核心的 tuning challenge
+
+### Challenge 1：响应面是联合、条件化且 regime-dependent 的
+
+#### 问题本质
+
+一般情况下：
+
+```text
+f(c) != base + sum_k effect_k(c_k)
+```
+
+一个 knob 的 effect 是当前完整 context 的函数：
+
+```text
+Delta_x(c, workload, engine state)
+```
+
+它可能随 topology、另一个 runtime knob、load、SLO 或 engine version 改变大小甚至改变符号。因此不能先分别求每个 knob 的最优值再 merge，也不能固定一个低质量 context 去判断另一个 knob。
+
+#### 已有真实证据
+
+在 C1 12-cell real surface：
+
+- `MNS 8 -> 32` 在 TP1/TP2/TP4 下分别提升约 **8.7% / 44.3% / 90.3%**。
+- 从同一 `TP1/MNS8` 起点，先 tune MNS 再 TP 会停在 `TP4/MNS16 = 2.4417`；该点沿任一单维都没有 strictly improving move，但 joint/global surface oracle `TP2/MNS32 = 3.2833` 高 **34.5% relative to the local point**，即 local point 对 oracle 有 **25.6% regret**。
+- C3 中 `MBT 256 -> 384` 的 effect 根据 topology/MNS 从 0 到约 -9.2%；`MNS 64 -> 128` 从 0 到约 +10.1%。
+- Action-aware Regime A 中 MBBT 几乎从不作为 exclusive cap，但 MBBT action 仍把 source goodput 提高 48.0%--77.1%。它通过 chunk size、prefill packing 和 scarce MNS slot residency 的联合变化获得收益。
+
+这直接否定两类通用策略：OAT/coordinate greedy，以及 `which cap is full -> tune that knob`。
+
+#### Tuner 必须具备的能力
+
+- Action 的基本单位是完整 `config delta`，允许 sparse joint action，而不是孤立 knob/value。
+- 对 topology/runtime family 使用 crossed anchors 或信息增益设计，主动测 interaction；不是默认所有 interaction 都强。
+- 能从数据判断 task 是 topology-dominant、runtime-interaction-dominant 还是 flat/noisy，并据此分配实验，而不是把固定 search order 写进规则。
+
+### Challenge 2：当前状态是 observational signal，tuning 需要 counterfactual identification
+
+#### 问题本质
+
+一次 telemetry trace 只能告诉我们：
+
+```text
+P(engine trajectory | current config, workload)
+```
+
+Tuning 真正需要的是：
+
+```text
+P(Delta SLO-goodput, failure, cost
+  | source trajectory, proposed full-config action)
+```
+
+Queue、KV、padding、split prefill 等状态既可能是原因，也可能是 workload/config 的结果。看见某种状态，不等于知道哪个 action 能修复它。一个 action 也可能同时改变多条机制；例如 MBBT 同时改变总 token budget、per-request chunk 和 multi-request packing，现有 telemetry 的解释是 mechanism-consistent，不是已完成的 causal decomposition。
+
+#### 已有真实证据
+
+- 5/10 秒 telemetry 确实太短；300 秒 phase-aware experiment 中，MNS action 的 queue/padding 机制直到 replay 75%--100% 才稳定出现。
+- 但 external TTFT outcome 在 25% 已完美区分该 action 是否修复 SLO。Telemetry 解释了 why，却没有比 outcome 更早或更可靠地指导 tuning。
+- 3.125 req/s/GPU 的 source 无法在 timeout 内 drain；另一组 source 已达 offered ceiling 的 99.1%--100%，数学上不可能通过 10% improvement gate。没有 exposure/headroom 和 censoring control，模型学到的不是 action response。
+- Same-config repeats 与 matched intervention 的波动不可忽略；只比较两个未经配对的 run 会混入 arrival/order/warm-state noise。
+
+#### Tuner 必须具备的能力
+
+- 训练样本必须是 exact-workload paired intervention：`(source trajectory, action) -> target delta`，保留失败和 censoring。
+- 使用 phase-binned continuous trajectory，而不是人工 bottleneck label 或 threshold rule。
+- 输出 response distribution 和 uncertainty；证据不足时 abstain，而不是强行给 diagnosis。
+- Telemetry 的价值必须通过同 cutoff、同 model capacity 的 outcome-only ablation 证明。若不能降低 end-to-end H20-hours，instrumentation 只保留为 debugging/解释工具。
+
+### Challenge 3：这是异构成本下的 sequential experimental design，不是静态 ranking
+
+#### 问题本质
+
+每个 trial 的成本不同：TP4 是 TP1 的四倍 GPU multiplier，startup/warm-up 可能主导短 probe，失败也有成本；同时 tuner 不知道 oracle，只能在 exploitation、information gain 和 cost 之间权衡。选对 top-1 的 accuracy 不能代表 tuning 效果。
+
+必须回答三个连续问题：
+
+1. 下一次测哪个联合 action？
+2. 测多久，何时 continuation/confirmation？
+3. 什么证据允许停止，并声称 best 已在 `epsilon` 内？
+
+#### 已有真实证据
+
+- Pure LLM 达到 best observed 后仍浪费 0.9106 reconstructed H20h。
+- Simulator top-1 虽然 0 marginal GPUh，却因 rank error 损失 30.46%；real-final 的 k 增大又迅速增加 H20h。
+- 5% endpoint 上两个方法都只需两个 trial，selection-count headroom 很小；2% endpoint 才暴露 action quality 和 stopping 的巨大差异。
+- Prefix 不是天然便宜：如果 startup、warm-up 和稳定状态形成占主要成本，缩短 replay window 未必带来等比例 H20h reduction。
+
+#### Tuner 必须具备的能力
+
+- Acquisition 直接优化 expected regret reduction / predicted H20 cost，并把 failure probability 纳入约束。
+- 在 run 前做与 tuning policy 分离的 workload admissibility check：避免 outcome ceiling、无法 drain、无请求或 measurement cap。
+- 使用 uncertainty-aware continuation 和 stop；stop criterion 针对声明的 candidate set 中“仍存在 >epsilon improvement 的概率”，而不是连续几次没提升。
+- 主结果报告 H20-hours-to-5%/2%/1%、fixed-budget regret 和 cost-normalized regret AUC，不 metric shopping。
+
+### Challenge 4：任何 mechanism model 都有 fidelity 和 transfer boundary
+
+#### 问题本质
+
+Simulator、learned surrogate、LLM prior 都是近似。Workload、SLO、model、hardware、engine version 改变后，operator cost、scheduler state transition、合法 flag 和 response surface 都可能变化。模型在 calibration task 上解释得好，不表示能在 held-out task 上排序正确。
+
+#### 已有真实证据
+
+- Frontier throughput reading 在完全匹配的 12-cell task 上仍把 real oracle 排错，top-1 regret 30.46%。这说明预测绝对 throughput 还不够，局部 rank fidelity 才是 tuning 关键。
+- Post-hoc SLO reading 的 top bucket 正确，但有大量 anchor feasibility error，也没有 prospective policy status。
+- Pure LLM 提出了当前 community-vLLM binary 不支持的 flag；engine/API version knowledge 本身会漂移。
+- 已有 cross-version experiment 中 vLLM 0.20 的强配置在 0.24 上出现大幅退化，说明 response prior 不能无条件迁移。
+
+#### Tuner 必须具备的能力
+
+- Simulator 只能作为 prior mean 或 candidate prior；真实 outcome 是 authoritative update。
+- 学习 simulator residual：把 `sim prediction + source state + action` 映射到 real response，而不是用 telemetry 重新实现另一个无校准 simulator。
+- 对 task-level OOD 显式提高 uncertainty/abstain；train/test 按完整 task 分割，不能按 request、anchor 或同一 surface cell 随机分割。
+- 分开报告 cold-start profile/training cost 与 per-task marginal cost，并在 N=1/10/100 等 amortization horizon 下展示。
+
+## 4. 对应的系统设计
+
+### 4.1 Harness：从 rule-based tuner 收缩成 experimental control plane
+
+Harness 保留以下确定性职责：
+
+- engine-version-aware config schema、合法性和资源约束；
+- 完整 config/action canonicalization，禁止隐式 merge 和重复试验；
+- exact trace/request/arrival/length hash，配对、随机化和 counter-rotation；
+- engine trajectory、external outcome、failure/censoring 的统一时间轴；
+- all-in GPU cost ledger、oracle annotation 分账、budget enforcement；
+- data sanity、coverage、SLO/correctness 和 stop-proof audit。
+
+Harness **不**包含 `queue > N -> increase MNS`、`cap full -> tune knob` 或人工 diagnosis-to-action mapping。这里的规则是实验语义和安全 invariant，不是性能决策 heuristic。
+
+### 4.2 Action-conditioned response model
+
+每条学习记录为：
+
+```text
+x = {source full config,
+     workload/SLO context,
+     source external outcome,
+     phase-binned engine trajectory}
+a = normalized full-config delta
+y = {Delta SLO-goodput, target feasibility/failure, measured H20 cost}
+```
+
+学习：
+
+```text
+p_theta(y | x, a, optional simulator prediction)
+```
+
+第一版应使用适合小数据且有 uncertainty 的 action-conditioned Gaussian-process/bootstrapped surrogate；kernel/feature ablation包括：
+
+1. config + external outcome；
+2. 同样输入 + telemetry trajectory；
+3. simulator + config + outcome；
+4. 同样输入 + telemetry residual features。
+
+Telemetry 保留 continuous phase distributions：queue/running residency、MNS/token slack、prefill/decode composition、partial/split prefill、step duration、KV、graph/padding。模型学习它们与 action 的 interaction；不先压成 bottleneck label。
+
+### 4.3 Cost-aware policy
+
+在合法的 single/joint candidate set 上选择：
+
+```text
+a* = argmax_a
+       expected constrained improvement(a)
+       / expected all-in H20 cost(a)
+```
+
+探索项来自 posterior uncertainty/information gain；launch/SLO failure 有显式 penalty。Simulator 可提供 prior mean，但 simulator 与 real discrepancy 会被 posterior residual 更新。一次 target measurement 后更新 response model，并重新计算下一步 action 或停止概率。
+
+LLM 在这个 tuning core 中不是 telemetry classifier。它最多作为可移除的 candidate/prior source，提出 schema 内的 sparse joint actions 或检索 engine mechanism；每个 proposal 都由同一个 response model、cost acquisition 和 real validator 评分。只有 `with LLM` 相对 `same tuner without LLM` 在 held-out tasks 上继续降低 cost-to-oracle，才能讨论 LLM 必要性。
+
+### 4.4 Stop 条件
+
+对一个预先声明的有限 candidate set，满足以下条件才 stop：
+
+```text
+P(exists c: f(c) > best_observed / (1 - epsilon) | D_t) < alpha
+```
+
+并且 best config 通过独立 confirmation、SLO/correctness gate，remaining candidate 的 cost-aware value of information 低于阈值。停止原因、posterior coverage 和未测区域必须写入 audit。
+
+## 5. 下一阶段如何证明，而不是再次构造 heuristic
+
+### R0：已有数据 retrospective premise check
+
+- 用 C1/C3 response surfaces 检查 joint model 是否能避免 OAT trap。
+- 用 action-aware paired records 比较 outcome-only 与 +telemetry 的 action-delta calibration。
+- 用 SimFid surface 比较 direct model 与 simulator-residual model 的 rank/regret。
+- 所有 feature、kernel、candidate encoding 在 held-out task 结果之前冻结。
+
+R0 只能筛选 model family，不能作为 paper result，因为现有 tasks 已参与路线设计。
+
+### R1：prospective same-host cost-to-oracle pilot
+
+- dash0 8xH20，固定 engine build/model；serialized placement，禁止共置干扰。
+- 至少一个未参与 feature/threshold 选择的新 trace window；选择非 ceiling、可 drain 的 offered load。
+- 声明一个可穷举的小 surface，至少包含 topology/runtime crossed actions，而不是只有一个 MNS ladder。
+- Oracle annotation 与 tuner online actions 分开记账；method 只能看到当时可用的数据。
+- 运行 random/search、OAT、纯 LLM、当前 guided harness、frozen simulator+real、outcome-only response、+telemetry response、sim-residual +telemetry。
+- 比较完整 H20 cost-to-regret curve，而不是 action classification accuracy。
+
+Pilot opening gate：
+
+1. telemetry model 相对相同 response model 去掉 telemetry，确实改变至少一个正确的 prospective action ranking；
+2. 最终 regret <=5%，无 false-safe accept；
+3. all-in H20-hours 相对 strongest safe outcome-only 至少下降 20%；
+4. 如果使用 simulator，需相对 frozen simulator+real 至少下降 30%；
+5. instrumentation overhead <=1%，所有成本和失败均计入。
+
+若 1--5 任一失败，就不能把 telemetry/harness 写成 tuning contribution；保留其 debugging/measurement 价值即可。
+
+### R2：task-held-out replication
+
+至少 3 个 workload window x 2 个 SLO regime，按完整 task 做 leave-one-task-out 或固定 train/test split。报告每个 task 的 regret、安全和成本，以及 task-level paired bootstrap CI。只有 R2 通过，才能把单 task 的 61.09% lower-bound saving 升级为项目贡献。
+
+## 6. 当前能说与不能说的贡献
+
+当前能说：
+
+- 我们有真实反例证明 OAT 和 cap-to-knob mapping 不是通用 tuning strategy。
+- Harness 的 legality、exact replay、failure/cost accounting 有必要的实验基础设施价值。
+- 当前 guided sequence 在一个严格同任务比较中显著减少了达到 2% empirical regret 的 reconstructed engine cost。
+- Simulator 的边际计算便宜，但 rank error 会转化成显著 real regret 或更多 real-final 成本。
+
+当前不能说：
+
+- telemetry 已经对 end-to-end tuning 提供独立增益；现有 direct pilot 对此为 negative。
+- 当前 harness 的 heuristic action ranking 是系统贡献；5% endpoint 只省 5.85%。
+- LLM 是必要组件；尚无同 policy 的 with/without LLM held-out ablation。
+- simulator 总 tuning cost 是 0；profile GPU cost 未审计，real verification 不能忽略。
+- 3.35 是 global oracle，或 dash0 与 dash1 数字是完全 controlled comparison。
+
+## Data sanity
+
+- Dash0 sequential numeric scores：n=9，min/max `1.1042/3.35`，distinct=7；两组 config outcome 不全相同。
+- Exact surface scores：n=12，min/max `1.2833/3.2833`，distinct=8；12 cells 完整且与 simulator metrics 中的 real scores 一致。
+- Reconstructed trial/cell attempts 包括 4 个无 engine timestamp 的失败：n=32，min/max `0/0.49778 H20h`，distinct=26；所有可重建成本均非负。
+- Sequential regret observations：n=16，min/max `0/0.34328`，distinct=6，全部在 `[0,1]`。
+- Checked invariants：dash0 fixed task contexts 相同（除 method/port）；trial counts 与 manifest 相符；engine log timestamps monotonic；surface cell 唯一且 MBT=8192；simulator 无失败且 predictions 不全相同；scores/results 不全相同；cost 非负；regret bounded。
+- Measurement limitation：primary 12-cell campaign 的 4 个 TP4 pre-ready failure 没有 engine timestamp，随后由 companion campaign 完整重跑；其失败成本在 engine-lifetime reconstruction 中为 0。因此 `3.5953 H20h` 是 completed annotation lower bound，不能作为 all-in annotation cost。这个缺口已显式保留，没有在其上建立 total-cost claim。

docs/two-stop-summary.md

@@ -0,0 +1,63 @@
+# Two-stop work — summary (feat/two-stop)
+
+Aligns the tuning harness with paper.pdf by implementing and validating the two
+distinct stopping mechanisms the paper conflates, plus a length-aware SLO and two
+harness-robustness fixes. 117 unit tests pass.
+
+## The two stops (they are different things)
+
+- **Stop-A — evaluation sufficiency** (per replay / probe): how much trace to replay
+  before a measurement is trustworthy. Criterion: the replayed prefix's offered
+  L-C-A converges to the full set's. Saves per-evaluation GPU time.
+- **Stop-B — tuning convergence** (across iterations): whether any config-improvement
+  opportunity remains; stop if not. Saves iterations.
+
+## What was built
+
+| Area | Summary | Commit |
+| --- | --- | --- |
+| Unify L-C-A | The prompt's `workload_lca_profile` is now the canonical 10-dim `lca.py` vector, not an ad-hoc re-derivation | `6f8e3c9` |
+| Session-coherent load axis | `sampling_u` assigned per session (parent_chat_id chain) so thresholding keeps/drops whole multi-turn sessions, preserving intra-session KV reuse (C) across load levels | `0f15bbc` |
+| Stop-A | `lca.find_convergence_prefix` (deterministic offered-L-C-A convergence prefix + C-gate: never declare infeasible on a cold cache); `spec.AdaptiveStopSpec` (default off); `worker._adaptive_replay_set` truncates replay + writes a certificate | `51a9e4a` |
+| Stop-A boundary guard | Re-measure the full window when a truncated probe's pass-rate is within `boundary_delta` of the SLO target (fixes feasibility-knee false positives) | `dfc823f` |
+| Stop-B authority | `harness._stop_authority` (mirrors the deterministic validator); `study tune` honors an LLM `should_stop` only if the validator agrees, else a bounded veto | `a8f9034` |
+| Length-aware SLO | `linear_ms` rule: `threshold = intercept_ms + per_token_ms·L_in` (e.g. 4s + L_in/8k) | `ed2bbe0` |
+| Robustness: timeout | HTTP stream now wraps `OSError`/`TimeoutError` as `HttpClientError` — a request exceeding `request_timeout_s` is a failed request, not a crashed trial | `2fcaf80` |
+| Robustness: SIGTERM | `run_trial` installs a SIGTERM handler so killing `study tune` tears down the engine (and EngineCore workers) instead of orphaning it / leaking GPU memory | `b17b213` |
+| Tooling | `stop_a_calibration.py` (CPU convergence curve), `stop_a_validate.py` (offline truncation-fidelity) | `08e53fd`, `3af1d84` |
+
+A subagent code review found no blocking bugs (and independently validated session
+coherence against a real trace); three minor fixes applied in `43125f4`.
+
+## Validation results (real GPU runs, dash0 H20)
+
+- **Stop-A** (`stop-a-validation-20260615.md`): CPU calibration reproduces the paper's
+  C-slowest ordering; GPU fidelity check on Qwen3-30B-A3B saves ~52% replay at τ_c=0.90
+  with 3/4 probe verdicts preserved; the one mismatch is a feasibility-knee false
+  positive that the **boundary guard fixes** — with the guard, best threshold matches
+  full replay exactly while still saving ~38% replay.
+- **27B TP sweep** (`qwen27b-tp-sweep-20260616.md`): under the length-aware SLO, dense
+  Qwen3.5-27B per-GPU rises sharply with TP — TP1 0.065 → TP2 0.29 → TP4 ≥0.91 — the
+  **opposite** of the MoE 30B-A3B (TP1 best per-GPU). TP1 is TPOT-bound.
+- **Stop-B** (`stop-b-e2e-20260615.md`, `stop-b-e2e-27b-20260616.md`): the 30B run
+  shows the deterministic `validator` stop + a premature-LLM-stop **veto**; the 27B run
+  (real gpt-5.4 loop) shows a genuine **improving climb** TP1 0.123 → TP2 0.29 → TP4
+  1.00 req/s/GPU (8.1×), each a correctly-diagnosed gain, then correctly **rejecting** a
+  TP4 runtime tweak that measured worse (no regression). The SIGTERM fix was validated
+  in practice (clean teardown, no leak).
+
+## Open items (next round)
+
+- **Harness convergence ablation (NOT yet done on this branch).** The paper's harness
+  result — domain-knowledge knob-family rules steering the LLM and cutting iterations —
+  has only *qualitative* evidence here (the 27B climb shows correct steering) plus older
+  smoke-regime ablations (`qwen27b-chat-0-8k-harness-fig18.md`: iters-to-best 4→2). A
+  controlled `use_harness=true` vs `false` (naive tuner) comparison on the 27B is the
+  missing quantified result.
+- **Loop efficiency**: at scale=1.0 infeasible high-θ probes burn the
+  `early_stop_max_elapsed_s=900` cap (a TP4 trial took ~3h). Lower it to ~300s for
+  practical agentic loops.
+- **dash0 GPUs 0/1** still hold leaked memory (pre-fix orphans) — needs a root
+  `nvidia-smi --gpu-reset`.
+- Deferred: more robust C feature for the low-reuse regime; Stop-C cross-day retune
+  trigger (paper Q1); §7 baselines (SCOOT / naive / community).

infra/gpu_fleet/config/fleet.example.toml

@@ -51,6 +51,13 @@ enabled = true
 sync_remote_path = "~/aituner"
 fleet_root = "~/.aituner_gpu_fleet"
 
+[[hosts]]
+name = "dash4"
+ssh_alias = "dash4"
+enabled = true
+sync_remote_path = "~/workspace/aituner"
+fleet_root = "~/.aituner_gpu_fleet"
+
 [[hosts]]
 name = "dash5"
 ssh_alias = "dash5"

infra/gpu_fleet/config/ssh_aliases.example.txt

@@ -4,5 +4,5 @@ dash0
 dash1
 dash2
 dash3
+dash4
 dash5
-

patches/vllm-0.24.0-opprof/0001-Add-lightweight-per-step-OpProf-telemetry.patch

@@ -0,0 +1,487 @@
+From f6f1cacbce0e39992d04843f652c1adda373ae43 Mon Sep 17 00:00:00 2001
+From: Gahow Wang <gahow.wang@gmail.com>
+Date: Sat, 11 Jul 2026 17:29:02 +0800
+Subject: [PATCH 1/5] Add lightweight per-step OpProf telemetry
+
+Assisted-by: OpenAI Codex
+---
+ vllm/envs.py                       |   4 +
+ vllm/v1/core/sched/scheduler.py    |  28 +++
+ vllm/v1/opprof.py                  | 337 +++++++++++++++++++++++++++++
+ vllm/v1/worker/gpu_model_runner.py |   6 +-
+ 4 files changed, 374 insertions(+), 1 deletion(-)
+ create mode 100644 vllm/v1/opprof.py
+
+diff --git a/vllm/envs.py b/vllm/envs.py
+index 27a85bb..b3093e9 100755
+--- a/vllm/envs.py
++++ b/vllm/envs.py
+@@ -45,6 +45,7 @@ if TYPE_CHECKING:
+     VLLM_LOGGING_COLOR: str = "auto"
+     NO_COLOR: bool = False
+     VLLM_LOG_STATS_INTERVAL: float = 10.0
++    VLLM_OPPROF_DIR: str = ""
+     VLLM_TRACE_FUNCTION: int = 0
+     VLLM_USE_FLASHINFER_SAMPLER: bool = True
+     VLLM_PP_LAYER_PARTITION: str | None = None
+@@ -786,6 +787,9 @@ environment_variables: dict[str, Callable[[], Any]] = {
+         if (val := float(os.getenv("VLLM_LOG_STATS_INTERVAL", "10."))) > 0.0
+         else 10.0
+     ),
++    # Directory for per-step OpProf JSONL telemetry.
++    # Empty disables OpProf.
++    "VLLM_OPPROF_DIR": lambda: os.getenv("VLLM_OPPROF_DIR", ""),
+     # Trace function calls
+     # If set to 1, vllm will trace function calls
+     # Useful for debugging
+diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py
+index 90d93a1..303c562 100644
+--- a/vllm/v1/core/sched/scheduler.py
++++ b/vllm/v1/core/sched/scheduler.py
+@@ -7,6 +7,7 @@ from collections.abc import Iterable
+ from dataclasses import replace
+ from typing import Any
+ 
++import vllm.envs as envs
+ from vllm.compilation.cuda_graph import CUDAGraphStat
+ from vllm.config import VllmConfig
+ from vllm.distributed.ec_transfer.ec_connector.base import (
+@@ -55,6 +56,7 @@ from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutp
+ from vllm.v1.kv_cache_interface import KVCacheConfig
+ from vllm.v1.metrics.perf import ModelMetrics, PerfStats
+ from vllm.v1.metrics.stats import PrefixCacheStats, SchedulerStats
++from vllm.v1.opprof import OpProfRecorder
+ from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
+ from vllm.v1.request import Request, RequestStatus, StreamingUpdate
+ from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
+@@ -271,6 +273,12 @@ class Scheduler(SchedulerInterface):
+         if self.connector is not None:
+             self.connector.bind_gpu_block_pool(self.kv_cache_manager.block_pool)
+ 
++        self.opprof = OpProfRecorder.create(
++            output_dir=envs.VLLM_OPPROF_DIR,
++            dp_rank=self.parallel_config.data_parallel_index,
++            log_stats=self.log_stats,
++        )
++
+         self.use_pp = self.parallel_config.pipeline_parallel_size > 1
+         self.use_v2_model_runner = vllm_config.use_v2_model_runner
+         # Scheduler iteration counter. Drives the V2+PP+async decode-throttle
+@@ -386,6 +394,9 @@ class Scheduler(SchedulerInterface):
+         return num_new_tokens
+ 
+     def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput:
++        opprof_start = (
++            self.opprof.capture_start(self) if self.opprof is not None else None
++        )
+         self.current_step += 1
+         # NOTE(woosuk) on the scheduling algorithm:
+         # There's no "decoding phase" nor "prefill phase" in the scheduler.
+@@ -1090,6 +1101,14 @@ class Scheduler(SchedulerInterface):
+             )
+             scheduler_output.ec_connector_metadata = ec_meta
+ 
++        if self.opprof is not None:
++            assert opprof_start is not None
++            self.opprof.begin(
++                scheduler=self,
++                output=scheduler_output,
++                start=opprof_start,
++            )
++
+         # Advance the fence only for non-empty steps (those that actually
+         # write KV and have their output processed later in update_from_output).
+         if self.defer_block_free and total_num_scheduled_tokens > 0:
+@@ -1800,6 +1819,12 @@ class Scheduler(SchedulerInterface):
+                 engine_core_outputs[0] = eco = EngineCoreOutputs()
+             eco.scheduler_stats = stats
+ 
++        if self.opprof is not None:
++            self.opprof.finalize(
++                output=scheduler_output,
++                cudagraph_stat=cudagraph_stats,
++            )
++
+         return engine_core_outputs
+ 
+     @staticmethod
+@@ -2292,6 +2317,9 @@ class Scheduler(SchedulerInterface):
+         if self.ec_connector is not None:
+             self.ec_connector.shutdown()
+ 
++        if self.opprof is not None:
++            self.opprof.close()
++
+         logger.debug_once("[shutdown] Scheduler: complete")
+ 
+     ########################################################################
+diff --git a/vllm/v1/opprof.py b/vllm/v1/opprof.py
+new file mode 100644
+index 0000000..f0330d0
+--- /dev/null
++++ b/vllm/v1/opprof.py
+@@ -0,0 +1,337 @@
++# SPDX-License-Identifier: Apache-2.0
++# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
++import atexit
++import logging
++import os
++import queue
++import threading
++import time
++from bisect import bisect_left
++from pathlib import Path
++from typing import Any
++
++import msgspec
++
++logger = logging.getLogger(__name__)
++
++SCHEMA_VERSION = 1
++CONTEXT_LENGTH_EDGES = tuple(1 << exponent for exponent in range(7, 18))
++CHUNK_SIZE_EDGES = tuple(1 << exponent for exponent in range(4, 12))
++DEFAULT_QUEUE_CAPACITY = 8192
++_CLOSE_TIMEOUT_SECONDS = 1.0
++_STOP = object()
++_PREFIX_FIELDS = (  # noqa: SIM905
++    "requests queries hits preempted_requests preempted_queries preempted_hits"
++).split()
++
++
++ScheduleStart = tuple[int, int, dict[str, dict[str, int] | None]]
++
++
++def classify_chunk(was_chunk: bool, end: int, target: int) -> str:
++    assert 0 <= end <= target
++    if was_chunk:
++        return "middle" if end < target else "final"
++    return "first" if end < target else "unsplit"
++
++
++def _prefix_values(stats: Any | None) -> dict[str, int] | None:
++    if stats is None:
++        return None
++    return {name: int(getattr(stats, name, 0)) for name in _PREFIX_FIELDS}
++
++
++def _prefix_snapshot(scheduler: Any) -> dict[str, dict[str, int] | None]:
++    return {
++        "local": _prefix_values(scheduler.kv_cache_manager.prefix_cache_stats),
++        "external": _prefix_values(scheduler.connector_prefix_cache_stats),
++    }
++
++
++def _prefix_delta(
++    before: dict[str, int] | None, after: dict[str, int] | None
++) -> dict[str, int] | None:
++    if before is None and after is None:
++        return None
++    before = before or dict.fromkeys(_PREFIX_FIELDS, 0)
++    after = after or dict.fromkeys(_PREFIX_FIELDS, 0)
++    delta = {name: after[name] - before[name] for name in _PREFIX_FIELDS}
++    assert all(value >= 0 for value in delta.values())
++    return delta
++
++
++class JSONLWriter:
++    def __init__(
++        self,
++        path: Path,
++        capacity: int = DEFAULT_QUEUE_CAPACITY,
++        start: bool = True,
++    ) -> None:
++        self._queue: queue.Queue[Any] = queue.Queue(capacity)
++        self._encoder = msgspec.json.Encoder()
++        self._file = path.open("xb", buffering=1 << 20)
++        self._thread = threading.Thread(target=self._run, daemon=True)
++        self._failure_lock = threading.Lock()
++        self._started = self._closed = False
++        self.failed = False
++        self.failure: Exception | None = None
++        self.encoded_records = self.written_records = 0
++        self.dropped_records = self._unreported_drops = 0
++        if start:
++            self.start()
++
++    def start(self) -> None:
++        if not self._started:
++            self._started = True
++            self._thread.start()
++
++    def _record_failure(self, error: Exception) -> None:
++        with self._failure_lock:
++            if self.failed:
++                return
++            self.failed = True
++            self.failure = error
++            logger.error("OpProf writer failed: %s", error)
++
++    def _writer_unavailable(self) -> bool:
++        if self.failed:
++            return True
++        if self._started and not self._thread.is_alive():
++            self._record_failure(RuntimeError("writer thread stopped unexpectedly"))
++            return True
++        return False
++
++    def _drop(self, pending: int) -> bool:
++        self.dropped_records += 1
++        self._unreported_drops = pending + 1
++        return False
++
++    def submit(self, record: dict[str, Any]) -> bool:
++        if self._closed:
++            raise RuntimeError("OpProf writer is closed")
++        pending = self._unreported_drops
++        record["dropped_records_before"] = pending
++        if self._writer_unavailable():
++            return self._drop(pending)
++        payload = self._encoder.encode(record) + b"\n"
++        self.encoded_records += 1
++        if self._writer_unavailable():
++            return self._drop(pending)
++        try:
++            self._queue.put_nowait(payload)
++        except queue.Full:
++            return self._drop(pending)
++        self._unreported_drops = 0
++        return True
++
++    def _run(self) -> None:
++        buffered = 0
++        last_flush = time.monotonic()
++        try:
++            while True:
++                try:
++                    item = self._queue.get(timeout=1.0)
++                except queue.Empty:
++                    self._file.flush()
++                    buffered = 0
++                    last_flush = time.monotonic()
++                    continue
++                try:
++                    if item is _STOP:
++                        break
++                    self._file.write(item)
++                    buffered += len(item)
++                    self.written_records += 1
++                    now = time.monotonic()
++                    if buffered >= 1 << 20 or now - last_flush >= 1.0:
++                        self._file.flush()
++                        buffered = 0
++                        last_flush = now
++                finally:
++                    self._queue.task_done()
++        except Exception as error:
++            self._record_failure(error)
++        finally:
++            footer = dict(
++                schema=SCHEMA_VERSION,
++                record_type="footer",
++                encoded_records=self.encoded_records,
++                written_records=self.written_records,
++                dropped_records=self.dropped_records,
++            )
++            try:
++                self._file.write(self._encoder.encode(footer) + b"\n")
++                self._file.flush()
++            except Exception as error:
++                self._record_failure(error)
++            finally:
++                try:
++                    self._file.close()
++                except Exception as error:
++                    self._record_failure(error)
++
++    def close(self) -> None:
++        if self._closed:
++            return
++        self._closed = True
++        self.start()
++        if not self._thread.is_alive():
++            return
++        try:
++            self._queue.put(_STOP, timeout=_CLOSE_TIMEOUT_SECONDS)
++        except queue.Full:
++            if not self._thread.is_alive():
++                return
++            try:
++                self._queue.put_nowait(_STOP)
++            except queue.Full:
++                self._record_failure(
++                    TimeoutError("timed out enqueueing writer stop sentinel")
++                )
++                return
++        self._thread.join(timeout=_CLOSE_TIMEOUT_SECONDS)
++        if self._thread.is_alive():
++            self._record_failure(TimeoutError("timed out joining writer thread"))
++
++
++class OpProfRecorder:
++    def __init__(self, engine_id: str, writer: JSONLWriter) -> None:
++        self.engine_id = engine_id
++        self.writer = writer
++        self._next_step = 0
++        self._pending: dict[int, dict[str, Any]] = {}
++        atexit.register(self.close)
++
++    @classmethod
++    def create(
++        cls, output_dir: str, dp_rank: int, log_stats: bool
++    ) -> "OpProfRecorder | None":
++        if not output_dir:
++            return None
++        if not log_stats:
++            raise ValueError("VLLM_OPPROF_DIR requires log stats to be enabled")
++        directory = Path(output_dir).expanduser()
++        if not directory.is_absolute():
++            raise ValueError("VLLM_OPPROF_DIR must be an absolute path")
++        directory.mkdir(parents=True, exist_ok=True)
++        engine_id = f"dp{dp_rank}-pid{os.getpid()}"
++        name = f"opprof-v{SCHEMA_VERSION}-{engine_id}-{time.time_ns()}.jsonl"
++        return cls(engine_id, JSONLWriter(directory / name))
++
++    @staticmethod
++    def capture_start(scheduler: Any) -> ScheduleStart:
++        return time.time_ns(), time.monotonic_ns(), _prefix_snapshot(scheduler)
++
++    def begin(self, scheduler: Any, output: Any, start: ScheduleStart) -> None:
++        key = id(output)
++        assert key not in self._pending, "duplicate OpProf begin"
++        new_ids = {request.req_id for request in output.scheduled_new_reqs}
++        prefill_requests = prefill_tokens = decode_requests = decode_tokens = 0
++        context_length_hist = [0] * (len(CONTEXT_LENGTH_EDGES) + 1)
++        chunk_size_hist = [0] * (len(CHUNK_SIZE_EDGES) + 1)
++        chunks: dict[str, Any] = dict.fromkeys(
++            ("first", "middle", "final", "unsplit"), 0
++        )
++        for req_id, num_tokens in output.num_scheduled_tokens.items():
++            request = scheduler.requests[req_id]
++            end = request.num_computed_tokens + num_tokens
++            assert end >= 0
++            context_length_hist[bisect_left(CONTEXT_LENGTH_EDGES, end)] += 1
++            is_prefill = (
++                req_id in new_ids
++                or output.scheduled_cached_reqs.is_context_phase(req_id)
++            )
++            if is_prefill:
++                prefill_requests += 1
++                prefill_tokens += num_tokens
++                assert num_tokens >= 0
++                chunk_size_hist[bisect_left(CHUNK_SIZE_EDGES, num_tokens)] += 1
++                target = request.num_tokens + request.num_output_placeholders
++                chunks[classify_chunk(request.is_prefill_chunk, end, target)] += 1
++            else:
++                decode_requests += 1
++                decode_tokens += num_tokens
++        prefix_after = _prefix_snapshot(scheduler)
++        block_pool = scheduler.kv_cache_manager.block_pool
++        total_blocks = block_pool.num_gpu_blocks - 1
++        free_blocks = block_pool.get_num_free_blocks()
++        assert 0 <= free_blocks <= total_blocks
++        chunks["tokens"] = prefill_tokens
++        chunks["chunk_size_hist"] = chunk_size_hist
++        values = dict(
++            schema=SCHEMA_VERSION,
++            engine_id=self.engine_id,
++            step_index=self._next_step,
++            submit_wall_ns=start[0],
++            submit_mono_ns=start[1],
++            model_executed=output.total_num_scheduled_tokens > 0,
++            scheduled_requests=len(output.num_scheduled_tokens),
++            decode_batch_size=decode_requests,
++            prefill_requests=prefill_requests,
++            prefill_tokens=prefill_tokens,
++            decode_tokens=decode_tokens,
++            chunked_prefill=chunks,
++            context_length_hist=context_length_hist,
++            preemptions=len(output.preempted_req_ids or ()),
++            queues=dict(
++                running=len(scheduler.running),
++                waiting=len(scheduler.waiting),
++                deferred=len(scheduler.skipped_waiting),
++            ),
++            kv=dict(
++                total_blocks=total_blocks,
++                free_blocks=free_blocks,
++                used_blocks=total_blocks - free_blocks,
++                usage=scheduler.kv_cache_manager.usage,
++            ),
++            prefix=dict(
++                local=_prefix_delta(start[2]["local"], prefix_after["local"]),
++                external=_prefix_delta(start[2]["external"], prefix_after["external"]),
++            ),
++        )
++        assert prefill_tokens + decode_tokens == output.total_num_scheduled_tokens
++        self._pending[key] = values
++        self._next_step += 1
++
++    def finalize(self, output: Any, cudagraph_stat: Any | None) -> bool:
++        try:
++            values = self._pending.pop(id(output))
++        except KeyError:
++            raise AssertionError("missing or already finalized OpProf step") from None
++        if cudagraph_stat is None:
++            assert not values["model_executed"]
++            cudagraph = dict(
++                hit=False,
++                runtime_mode="NONE",
++                unpadded_tokens=0,
++                bucket_tokens=0,
++                padding_tokens=0,
++            )
++        else:
++            mode = str(cudagraph_stat.runtime_mode).rsplit(".", 1)[-1]
++            cudagraph = dict(
++                hit=mode != "NONE",
++                runtime_mode=mode,
++                unpadded_tokens=cudagraph_stat.num_unpadded_tokens,
++                bucket_tokens=cudagraph_stat.num_padded_tokens,
++                padding_tokens=cudagraph_stat.num_paddings,
++            )
++        record = dict(
++            values,
++            complete_mono_ns=time.monotonic_ns(),
++            cudagraph=cudagraph,
++            moe_expert_load=None,
++            dropped_records_before=0,
++        )
++        return self.writer.submit(record)
++
++    def close(self) -> None:
++        self.writer.close()
++
++    @property
++    def failed(self) -> bool:
++        return self.writer.failed
++
++    @property
++    def failure(self) -> Exception | None:
++        return self.writer.failure
+diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py
+index 74938a8..c11d773 100644
+--- a/vllm/v1/worker/gpu_model_runner.py
++++ b/vllm/v1/worker/gpu_model_runner.py
+@@ -437,6 +437,7 @@ class GPUModelRunner(
+         self.scheduler_config = vllm_config.scheduler_config
+         self.speculative_config = vllm_config.speculative_config
+         self.observability_config = vllm_config.observability_config
++        self.opprof_enabled = bool(envs.VLLM_OPPROF_DIR)
+ 
+         model_config = self.model_config
+         cache_config = self.cache_config
+@@ -3917,7 +3918,10 @@ class GPUModelRunner(
+                 assert batch_descriptor.num_tokens == num_tokens_padded
+ 
+         cudagraph_stats = None
+-        if self.vllm_config.observability_config.cudagraph_metrics:
++        if (
++            self.vllm_config.observability_config.cudagraph_metrics
++            or self.opprof_enabled
++        ):
+             cudagraph_stats = CUDAGraphStat(
+                 num_unpadded_tokens=num_tokens,
+                 num_padded_tokens=batch_descriptor.num_tokens,
+-- 
+2.43.0
+

patches/vllm-0.24.0-opprof/0002-Add-standalone-OpProf-telemetry-tests.patch

@@ -0,0 +1,417 @@
+From 4f4ee674f217698436b00c3ab6357f59a792477a Mon Sep 17 00:00:00 2001
+From: Gahow Wang <gahow.wang@gmail.com>
+Date: Sat, 11 Jul 2026 17:29:02 +0800
+Subject: [PATCH 2/5] Add standalone OpProf telemetry tests
+
+Assisted-by: OpenAI Codex
+---
+ tests/v1/core/test_opprof.py | 397 +++++++++++++++++++++++++++++++++++
+ 1 file changed, 397 insertions(+)
+ create mode 100644 tests/v1/core/test_opprof.py
+
+diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
+new file mode 100644
+index 0000000..9bfbfcc
+--- /dev/null
++++ b/tests/v1/core/test_opprof.py
+@@ -0,0 +1,397 @@
++# SPDX-License-Identifier: Apache-2.0
++# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
++"""Standalone tests: this file intentionally does not import the vllm package."""
++
++import errno
++import importlib.util
++import logging
++import sys
++import threading
++from pathlib import Path
++from types import SimpleNamespace
++
++import msgspec
++import pytest
++
++_ROOT = Path(__file__).parents[3]
++_SPEC = importlib.util.spec_from_file_location(
++    "opprof_standalone", _ROOT / "vllm" / "v1" / "opprof.py"
++)
++assert _SPEC is not None and _SPEC.loader is not None
++opprof = importlib.util.module_from_spec(_SPEC)
++sys.modules[_SPEC.name] = opprof
++_SPEC.loader.exec_module(opprof)
++
++
++class CachedRequests:
++    def __init__(self, context_ids=()):
++        self.context_ids = set(context_ids)
++
++    def is_context_phase(self, req_id):
++        return req_id in self.context_ids
++
++
++def prefix_stats(**overrides):
++    values = dict.fromkeys(opprof._PREFIX_FIELDS, 0)
++    values.update(overrides)
++    return SimpleNamespace(**values)
++
++
++def request(computed, total, was_chunk=False, placeholders=0):
++    return SimpleNamespace(
++        num_computed_tokens=computed,
++        num_tokens=total,
++        num_output_placeholders=placeholders,
++        is_prefill_chunk=was_chunk,
++    )
++
++
++def scheduler(requests, local=None, external=None):
++    block_pool = SimpleNamespace(
++        num_gpu_blocks=101,
++        get_num_free_blocks=lambda: 40,
++    )
++    kv_manager = SimpleNamespace(
++        prefix_cache_stats=local or prefix_stats(),
++        block_pool=block_pool,
++        usage=0.6,
++    )
++    return SimpleNamespace(
++        requests=requests,
++        kv_cache_manager=kv_manager,
++        connector_prefix_cache_stats=external,
++        running=list(range(len(requests))),
++        waiting=[0, 1],
++        skipped_waiting=[0],
++    )
++
++
++def schedule_output(tokens, context_ids=(), new_ids=(), preempted=()):
++    return SimpleNamespace(
++        scheduled_new_reqs=[SimpleNamespace(req_id=req_id) for req_id in new_ids],
++        scheduled_cached_reqs=CachedRequests(context_ids),
++        num_scheduled_tokens=tokens,
++        total_num_scheduled_tokens=sum(tokens.values()),
++        preempted_req_ids=set(preempted),
++    )
++
++
++def graph(mode="FULL", unpadded=1, padded=1):
++    return SimpleNamespace(
++        runtime_mode=mode,
++        num_unpadded_tokens=unpadded,
++        num_padded_tokens=padded,
++        num_paddings=padded - unpadded,
++    )
++
++
++def recorder(tmp_path, *, capacity=8192, start=True):
++    path = tmp_path / "opprof.jsonl"
++    writer = opprof.JSONLWriter(path, capacity=capacity, start=start)
++    return opprof.OpProfRecorder("dp0-pid1", writer), path
++
++
++def emit(rec, sched, output, cg=None):
++    start = rec.capture_start(sched)
++    rec.begin(sched, output, start)
++    return rec.finalize(output, cg or graph())
++
++
++def read_jsonl(path):
++    return [msgspec.json.decode(line) for line in path.read_bytes().splitlines()]
++
++
++def test_import_light_and_approved_constants():
++    assert "torch" not in sys.modules
++    assert "vllm" not in sys.modules
++    assert opprof.DEFAULT_QUEUE_CAPACITY == 8192
++    assert tuple(1 << i for i in range(7, 18)) == opprof.CONTEXT_LENGTH_EDGES
++    assert tuple(1 << i for i in range(4, 12)) == opprof.CHUNK_SIZE_EDGES
++
++
++def test_schema_and_invariants(tmp_path):
++    sched = scheduler(
++        {
++            "first": request(0, 100),
++            "final": request(64, 100, was_chunk=True),
++            "decode": request(1024, 1025),
++        }
++    )
++    output = schedule_output(
++        {"first": 64, "final": 36, "decode": 1},
++        context_ids={"final"},
++        new_ids={"first"},
++        preempted={"old"},
++    )
++    rec, path = recorder(tmp_path)
++    start = rec.capture_start(sched)
++    sched.kv_cache_manager.prefix_cache_stats = prefix_stats(
++        requests=1, queries=100, hits=64
++    )
++    rec.begin(sched, output, start)
++    assert rec.finalize(output, graph("FULL", 101, 128))
++    rec.close()
++
++    record, footer = read_jsonl(path)
++    assert record["schema"] == 1
++    assert record["scheduled_requests"] == 3
++    assert record["prefill_requests"] == 2
++    assert record["decode_batch_size"] == 1
++    assert record["prefill_tokens"] + record["decode_tokens"] == 101
++    assert sum(record["context_length_hist"]) == 3
++    assert len(record["context_length_hist"]) == 12
++    assert sum(record["chunked_prefill"]["chunk_size_hist"]) == 2
++    assert len(record["chunked_prefill"]["chunk_size_hist"]) == 9
++    assert record["chunked_prefill"]["first"] == 1
++    assert record["chunked_prefill"]["final"] == 1
++    assert record["preemptions"] == 1
++    assert record["kv"] == {
++        "total_blocks": 100,
++        "free_blocks": 40,
++        "used_blocks": 60,
++        "usage": 0.6,
++    }
++    assert record["prefix"]["local"]["hits"] == 64
++    assert record["moe_expert_load"] is None
++    assert record["complete_mono_ns"] >= record["submit_mono_ns"]
++    assert footer["record_type"] == "footer"
++    assert footer["written_records"] == 1
++
++
++def test_capture_record_matches_pre_refactor_golden(tmp_path, monkeypatch):
++    sched = scheduler(
++        {
++            "edge": request(0, 128),
++            "after": request(65, 129, was_chunk=True),
++            "decode": request(256, 257),
++        }
++    )
++    output = schedule_output(
++        {"edge": 128, "after": 64, "decode": 1},
++        context_ids={"after"},
++        new_ids={"edge"},
++        preempted={"old"},
++    )
++    rec, path = recorder(tmp_path)
++    zero_prefix = dict.fromkeys(opprof._PREFIX_FIELDS, 0)
++    start = (100, 200, {"local": zero_prefix, "external": None})
++    monkeypatch.setattr(opprof.time, "monotonic_ns", lambda: 300)
++
++    rec.begin(sched, output, start)
++    assert rec.finalize(output, graph("FULL", 193, 256))
++    rec.close()
++
++    record = read_jsonl(path)[0]
++    assert record == {
++        "schema": 1,
++        "engine_id": "dp0-pid1",
++        "step_index": 0,
++        "submit_wall_ns": 100,
++        "submit_mono_ns": 200,
++        "model_executed": True,
++        "scheduled_requests": 3,
++        "decode_batch_size": 1,
++        "prefill_requests": 2,
++        "prefill_tokens": 192,
++        "decode_tokens": 1,
++        "chunked_prefill": {
++            "first": 0,
++            "middle": 0,
++            "final": 1,
++            "unsplit": 1,
++            "tokens": 192,
++            "chunk_size_hist": [0, 0, 1, 1, 0, 0, 0, 0, 0],
++        },
++        "context_length_hist": [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
++        "preemptions": 1,
++        "queues": {"running": 3, "waiting": 2, "deferred": 1},
++        "kv": {
++            "total_blocks": 100,
++            "free_blocks": 40,
++            "used_blocks": 60,
++            "usage": 0.6,
++        },
++        "prefix": {"local": zero_prefix, "external": None},
++        "complete_mono_ns": 300,
++        "cudagraph": {
++            "hit": True,
++            "runtime_mode": "FULL",
++            "unpadded_tokens": 193,
++            "bucket_tokens": 256,
++            "padding_tokens": 63,
++        },
++        "moe_expert_load": None,
++        "dropped_records_before": 0,
++    }
++
++
++@pytest.mark.parametrize(
++    ("was_chunk", "end", "target", "expected"),
++    [
++        (False, 64, 100, "first"),
++        (True, 80, 100, "middle"),
++        (True, 100, 100, "final"),
++        (False, 100, 100, "unsplit"),
++    ],
++)
++def test_chunk_classification(was_chunk, end, target, expected):
++    assert opprof.classify_chunk(was_chunk, end, target) == expected
++
++
++def test_async_pairing_out_of_order_and_double_finalize(tmp_path):
++    sched = scheduler({"a": request(10, 11), "b": request(200, 201)})
++    first = schedule_output({"a": 1})
++    second = schedule_output({"b": 1})
++    rec, path = recorder(tmp_path)
++    rec.begin(sched, first, rec.capture_start(sched))
++    rec.begin(sched, second, rec.capture_start(sched))
++    assert rec.finalize(second, graph())
++    assert rec.finalize(first, graph("NONE"))
++    with pytest.raises(AssertionError, match="already finalized"):
++        rec.finalize(second, graph())
++    assert not rec._pending
++    rec.close()
++    records = read_jsonl(path)[:-1]
++    assert [record["step_index"] for record in records] == [1, 0]
++    assert records[0]["context_length_hist"][1] == 1
++    assert records[1]["context_length_hist"][0] == 1
++
++
++def test_disabled_noop_and_log_stats_fail_fast(tmp_path):
++    assert opprof.OpProfRecorder.create("", dp_rank=0, log_stats=False) is None
++    with pytest.raises(ValueError, match="requires log stats"):
++        opprof.OpProfRecorder.create(str(tmp_path), dp_rank=0, log_stats=False)
++    assert not list(tmp_path.iterdir())
++
++
++def test_bounded_queue_drop_accounting(tmp_path):
++    sched = scheduler({str(i): request(i, i + 1) for i in range(3)})
++    rec, path = recorder(tmp_path, capacity=1, start=False)
++    assert emit(rec, sched, schedule_output({"0": 1}))
++    assert not emit(rec, sched, schedule_output({"1": 1}))
++    rec.writer.start()
++    rec.writer._queue.join()
++    assert emit(rec, sched, schedule_output({"2": 1}))
++    rec.close()
++
++    first, after_drop, footer = read_jsonl(path)
++    assert first["step_index"] == 0
++    assert after_drop["step_index"] == 2
++    assert after_drop["dropped_records_before"] == 1
++    assert footer["encoded_records"] == 3
++    assert footer["written_records"] == 2
++    assert footer["dropped_records"] == 1
++
++
++def test_writer_enospc_is_exposed_and_shutdown_is_bounded(
++    tmp_path, monkeypatch, caplog
++):
++    sched = scheduler(
++        {
++            "first": request(0, 1),
++            "after_failure": request(1, 2),
++        }
++    )
++    rec, _ = recorder(tmp_path, capacity=1, start=False)
++    assert emit(rec, sched, schedule_output({"first": 1}))
++
++    real_file = rec.writer._file
++
++    def fail_enospc(*_args, **_kwargs):
++        raise OSError(errno.ENOSPC, "No space left on device")
++
++    failing_file = SimpleNamespace(
++        write=fail_enospc,
++        flush=fail_enospc,
++        close=real_file.close,
++    )
++    monkeypatch.setattr(rec.writer, "_file", failing_file)
++    caplog.set_level(logging.ERROR, logger=opprof.__name__)
++
++    rec.writer.start()
++    rec.writer._thread.join(timeout=1.0)
++    assert not rec.writer._thread.is_alive()
++
++    producer_result = emit(
++        rec, sched, schedule_output({"after_failure": 1})
++    )
++
++    closer = threading.Thread(target=rec.close, daemon=True)
++    closer.start()
++    closer.join(timeout=1.0)
++    assert not closer.is_alive(), "OpProf close blocked after writer failure"
++    assert not producer_result
++    assert rec.writer.dropped_records == 1
++    assert rec.failed
++    assert isinstance(rec.failure, OSError)
++    assert rec.failure.errno == errno.ENOSPC
++    errors = [
++        record
++        for record in caplog.records
++        if "OpProf writer failed" in record.getMessage()
++    ]
++    assert len(errors) == 1
++
++
++def test_shutdown_flush_is_idempotent(tmp_path):
++    sched = scheduler({"decode": request(8, 9)})
++    rec, path = recorder(tmp_path)
++    assert emit(rec, sched, schedule_output({"decode": 1}))
++    rec.close()
++    rec.close()
++    record, footer = read_jsonl(path)
++    assert record["step_index"] == 0
++    assert footer["written_records"] == 1
++    assert path.stat().st_size > 0
++
++
++def test_piecewise_cudagraph_record_preserved(tmp_path):
++    sched = scheduler({"decode": request(4096, 4097)})
++    output = schedule_output({"decode": 1})
++    rec, path = recorder(tmp_path)
++    assert emit(rec, sched, output, graph("PIECEWISE", 513, 520))
++    rec.close()
++    record = read_jsonl(path)[0]
++    assert record["cudagraph"] == {
++        "hit": True,
++        "runtime_mode": "PIECEWISE",
++        "unpadded_tokens": 513,
++        "bucket_tokens": 520,
++        "padding_tokens": 7,
++    }
++
++
++def test_zero_scheduled_tokens_finalize_without_cudagraph(tmp_path):
++    sched = scheduler({})
++    output = schedule_output({})
++    rec, path = recorder(tmp_path)
++    rec.begin(sched, output, rec.capture_start(sched))
++    assert rec._pending
++
++    assert rec.finalize(output, cudagraph_stat=None)
++    assert not rec._pending
++    rec.close()
++
++    record = read_jsonl(path)[0]
++    assert record["model_executed"] is False
++    assert record["scheduled_requests"] == 0
++    assert record["prefill_requests"] == 0
++    assert record["prefill_tokens"] == 0
++    assert record["decode_batch_size"] == 0
++    assert record["decode_tokens"] == 0
++    assert record["context_length_hist"] == [0] * 12
++    assert record["chunked_prefill"] == {
++        "first": 0,
++        "middle": 0,
++        "final": 0,
++        "unsplit": 0,
++        "tokens": 0,
++        "chunk_size_hist": [0] * 9,
++    }
++    assert record["cudagraph"] == {
++        "hit": False,
++        "runtime_mode": "NONE",
++        "unpadded_tokens": 0,
++        "bucket_tokens": 0,
++        "padding_tokens": 0,
++    }
+-- 
+2.43.0
+

patches/vllm-0.24.0-opprof/0003-Log-the-OpProf-output-path-at-startup.patch

@@ -0,0 +1,52 @@
+From 668cfb7e27e488454dbf09a4927b8a60d6d49b40 Mon Sep 17 00:00:00 2001
+From: Gahow Wang <gahow.wang@gmail.com>
+Date: Sat, 11 Jul 2026 17:32:27 +0800
+Subject: [PATCH 3/5] Log the OpProf output path at startup
+
+Assisted-by: OpenAI Codex
+---
+ tests/v1/core/test_opprof.py    | 1 +
+ vllm/v1/core/sched/scheduler.py | 2 ++
+ vllm/v1/opprof.py               | 1 +
+ 3 files changed, 4 insertions(+)
+
+diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
+index 9bfbfcc..79c1fae 100644
+--- a/tests/v1/core/test_opprof.py
++++ b/tests/v1/core/test_opprof.py
+@@ -336,6 +336,7 @@ def test_writer_enospc_is_exposed_and_shutdown_is_bounded(
+ def test_shutdown_flush_is_idempotent(tmp_path):
+     sched = scheduler({"decode": request(8, 9)})
+     rec, path = recorder(tmp_path)
++    assert rec.writer.path == path
+     assert emit(rec, sched, schedule_output({"decode": 1}))
+     rec.close()
+     rec.close()
+diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py
+index 303c562..769a02a 100644
+--- a/vllm/v1/core/sched/scheduler.py
++++ b/vllm/v1/core/sched/scheduler.py
+@@ -278,6 +278,8 @@ class Scheduler(SchedulerInterface):
+             dp_rank=self.parallel_config.data_parallel_index,
+             log_stats=self.log_stats,
+         )
++        if self.opprof is not None:
++            logger.info("OpProf telemetry enabled: %s", self.opprof.writer.path)
+ 
+         self.use_pp = self.parallel_config.pipeline_parallel_size > 1
+         self.use_v2_model_runner = vllm_config.use_v2_model_runner
+diff --git a/vllm/v1/opprof.py b/vllm/v1/opprof.py
+index f0330d0..75f63de 100644
+--- a/vllm/v1/opprof.py
++++ b/vllm/v1/opprof.py
+@@ -68,6 +68,7 @@ class JSONLWriter:
+         start: bool = True,
+     ) -> None:
+         self._queue: queue.Queue[Any] = queue.Queue(capacity)
++        self.path = path
+         self._encoder = msgspec.json.Encoder()
+         self._file = path.open("xb", buffering=1 << 20)
+         self._thread = threading.Thread(target=self._run, daemon=True)
+-- 
+2.43.0
+

patches/vllm-0.24.0-opprof/0004-Exclude-OpProf-output-path-from-compile-cache-key.patch

@@ -0,0 +1,64 @@
+From 335da4abe60e0177872e0b7751e86eeec6756a2b Mon Sep 17 00:00:00 2001
+From: Gahow Wang <gahow.wang@gmail.com>
+Date: Sat, 11 Jul 2026 22:32:30 +0800
+Subject: [PATCH 4/5] Exclude OpProf output path from compile cache key
+
+---
+ tests/v1/core/test_opprof.py | 21 ++++++++++++++++++++-
+ vllm/envs.py                 |  1 +
+ 2 files changed, 21 insertions(+), 1 deletion(-)
+
+diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
+index 79c1fae..a820e7e 100644
+--- a/tests/v1/core/test_opprof.py
++++ b/tests/v1/core/test_opprof.py
+@@ -8,7 +8,7 @@ import logging
+ import sys
+ import threading
+ from pathlib import Path
+-from types import SimpleNamespace
++from types import ModuleType, SimpleNamespace
+ 
+ import msgspec
+ import pytest
+@@ -23,6 +23,25 @@ sys.modules[_SPEC.name] = opprof
+ _SPEC.loader.exec_module(opprof)
+ 
+ 
++def test_output_dir_is_not_a_compile_factor(monkeypatch: pytest.MonkeyPatch):
++    spec = importlib.util.spec_from_file_location(
++        "envs_standalone", _ROOT / "vllm" / "envs.py"
++    )
++    assert spec is not None and spec.loader is not None
++    envs = importlib.util.module_from_spec(spec)
++    monkeypatch.setitem(sys.modules, spec.name, envs)
++    spec.loader.exec_module(envs)
++
++    monkeypatch.setitem(sys.modules, "vllm", ModuleType("vllm"))
++    monkeypatch.setitem(sys.modules, "vllm.config", ModuleType("vllm.config"))
++    config_utils = ModuleType("vllm.config.utils")
++    config_utils.__dict__["normalize_value"] = lambda value: value
++    monkeypatch.setitem(sys.modules, "vllm.config.utils", config_utils)
++    monkeypatch.setenv("VLLM_OPPROF_DIR", "/tmp/opprof")
++
++    assert "VLLM_OPPROF_DIR" not in envs.compile_factors()
++
++
+ class CachedRequests:
+     def __init__(self, context_ids=()):
+         self.context_ids = set(context_ids)
+diff --git a/vllm/envs.py b/vllm/envs.py
+index b3093e9..5634708 100755
+--- a/vllm/envs.py
++++ b/vllm/envs.py
+@@ -2044,6 +2044,7 @@ def compile_factors() -> dict[str, object]:
+         "VLLM_LOGGING_CONFIG_PATH",
+         "VLLM_LOGGING_COLOR",
+         "VLLM_LOG_STATS_INTERVAL",
++        "VLLM_OPPROF_DIR",
+         "VLLM_DEBUG_LOG_API_SERVER_RESPONSE",
+         "VLLM_TUNED_CONFIG_FOLDER",
+         "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR",
+-- 
+2.43.0
+

patches/vllm-0.24.0-opprof/0005-Keep-compile-factor-regression-import-light.patch

@@ -0,0 +1,24 @@
+From bbfa7176a6a3686a88ee66696f1ad8d754559d96 Mon Sep 17 00:00:00 2001
+From: Gahow Wang <gahow.wang@gmail.com>
+Date: Sat, 11 Jul 2026 22:38:08 +0800
+Subject: [PATCH 5/5] Keep compile-factor regression import-light
+
+---
+ tests/v1/core/test_opprof.py | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
+index a820e7e..0b8a8a1 100644
+--- a/tests/v1/core/test_opprof.py
++++ b/tests/v1/core/test_opprof.py
+@@ -37,6 +37,7 @@ def test_output_dir_is_not_a_compile_factor(monkeypatch: pytest.MonkeyPatch):
+     config_utils = ModuleType("vllm.config.utils")
+     config_utils.__dict__["normalize_value"] = lambda value: value
+     monkeypatch.setitem(sys.modules, "vllm.config.utils", config_utils)
++    monkeypatch.setitem(sys.modules, "torch", ModuleType("torch"))
+     monkeypatch.setenv("VLLM_OPPROF_DIR", "/tmp/opprof")
+ 
+     assert "VLLM_OPPROF_DIR" not in envs.compile_factors()
+-- 
+2.43.0
+

patches/vllm-0.24.0-opprof/0006-Checkpoint-OpProf-accounting-across-hard-kills.patch

@@ -0,0 +1,306 @@
+From f8b68f2452c424d22de4a69527a427207dcfbca5 Mon Sep 17 00:00:00 2001
+From: Gahow Wang <gahow.wang@gmail.com>
+Date: Sun, 12 Jul 2026 12:56:39 +0800
+Subject: [PATCH] Checkpoint OpProf accounting across hard kills
+
+---
+ tests/v1/core/test_opprof.py | 118 +++++++++++++++++++++++++++++++++++
+ vllm/v1/opprof.py            |  84 ++++++++++++++++++++++---
+ 2 files changed, 195 insertions(+), 7 deletions(-)
+
+diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
+index 0b8a8a1..007c5bb 100644
+--- a/tests/v1/core/test_opprof.py
++++ b/tests/v1/core/test_opprof.py
+@@ -5,6 +5,8 @@
+ import errno
+ import importlib.util
+ import logging
++import os
++import subprocess
+ import sys
+ import threading
+ from pathlib import Path
+@@ -121,6 +123,12 @@ def read_jsonl(path):
+     return [msgspec.json.decode(line) for line in path.read_bytes().splitlines()]
+ 
+ 
++def read_sidecar(path):
++    return msgspec.json.decode(
++        path.with_name(f"{path.name}.footer.json").read_bytes()
++    )
++
++
+ def test_import_light_and_approved_constants():
+     assert "torch" not in sys.modules
+     assert "vllm" not in sys.modules
+@@ -366,6 +374,116 @@ def test_shutdown_flush_is_idempotent(tmp_path):
+     assert path.stat().st_size > 0
+ 
+ 
++def test_sidecar_updates_are_atomic(tmp_path, monkeypatch):
++    writer = opprof.JSONLWriter(tmp_path / "atomic.jsonl", start=False)
++    writer._write_sidecar(
++        encoded_records=1,
++        written_records=1,
++        dropped_records=0,
++        last_step_index=0,
++        final=False,
++    )
++    original = writer.sidecar_path.read_bytes()
++    real_replace = os.replace
++    replacements = []
++
++    def inspect_replace(source, destination):
++        assert Path(destination) == writer.sidecar_path
++        assert writer.sidecar_path.read_bytes() == original
++        candidate = msgspec.json.decode(Path(source).read_bytes())
++        assert candidate["written_records"] == 2
++        replacements.append(candidate)
++        real_replace(source, destination)
++
++    monkeypatch.setattr(opprof.os, "replace", inspect_replace)
++    writer._write_sidecar(
++        encoded_records=3,
++        written_records=2,
++        dropped_records=1,
++        last_step_index=2,
++        final=False,
++    )
++    monkeypatch.setattr(opprof.os, "replace", real_replace)
++
++    assert len(replacements) == 1
++    assert read_sidecar(writer.path)["encoded_records"] == 3
++    assert not list(tmp_path.glob(".*.tmp-*"))
++    writer.close()
++
++
++def test_hard_kill_sidecar_balances_last_flush(tmp_path):
++    path = tmp_path / "hard-kill.jsonl"
++    child = """
++import importlib.util
++import sys
++import time
++from pathlib import Path
++
++source, output = sys.argv[1:]
++spec = importlib.util.spec_from_file_location("opprof_hard_kill", source)
++module = importlib.util.module_from_spec(spec)
++sys.modules[spec.name] = module
++spec.loader.exec_module(module)
++writer = module.JSONLWriter(Path(output))
++for step in range(3):
++    assert writer.submit({"schema": 1, "step_index": step})
++writer._queue.join()
++while not writer.sidecar_path.exists():
++    time.sleep(0.01)
++print("READY", flush=True)
++time.sleep(60)
++"""
++    process = subprocess.Popen(
++        [
++            sys.executable,
++            "-c",
++            child,
++            str(_ROOT / "vllm/v1/opprof.py"),
++            str(path),
++        ],
++        stdout=subprocess.PIPE,
++        text=True,
++    )
++    try:
++        assert process.stdout is not None
++        assert process.stdout.readline().strip() == "READY"
++        process.kill()
++        assert process.wait(timeout=5) < 0
++    finally:
++        if process.poll() is None:
++            process.kill()
++            process.wait(timeout=5)
++
++    records = read_jsonl(path)
++    sidecar = read_sidecar(path)
++    assert len(records) == 3
++    assert all(record.get("record_type") != "footer" for record in records)
++    assert sidecar["final"] is False
++    assert sidecar["written_records"] == len(records)
++    assert sidecar["encoded_records"] == (
++        sidecar["written_records"] + sidecar["dropped_records"]
++    )
++    assert sidecar["last_step_index"] == records[-1]["step_index"] == 2
++
++
++def test_clean_footer_and_final_sidecar_agree(tmp_path):
++    sched = scheduler({"decode": request(8, 9)})
++    rec, path = recorder(tmp_path)
++    assert emit(rec, sched, schedule_output({"decode": 1}))
++    rec.close()
++
++    record, footer = read_jsonl(path)
++    sidecar = read_sidecar(path)
++    assert sidecar["record_type"] == "footer_checkpoint"
++    assert sidecar["stream"] == path.name
++    assert sidecar["final"] is True
++    assert sidecar["last_step_index"] == record["step_index"] == 0
++    assert sidecar["checkpoint_wall_ns"] > 0
++    assert sidecar["flush_interval_seconds"] == opprof.FLUSH_INTERVAL_SECONDS
++    for counter in ("encoded_records", "written_records", "dropped_records"):
++        assert sidecar[counter] == footer[counter]
++
++
+ def test_piecewise_cudagraph_record_preserved(tmp_path):
+     sched = scheduler({"decode": request(4096, 4097)})
+     output = schedule_output({"decode": 1})
+diff --git a/vllm/v1/opprof.py b/vllm/v1/opprof.py
+index 75f63de..28d9635 100644
+--- a/vllm/v1/opprof.py
++++ b/vllm/v1/opprof.py
+@@ -7,6 +7,7 @@ import queue
+ import threading
+ import time
+ from bisect import bisect_left
++from contextlib import suppress
+ from pathlib import Path
+ from typing import Any
+ 
+@@ -18,6 +19,7 @@ SCHEMA_VERSION = 1
+ CONTEXT_LENGTH_EDGES = tuple(1 << exponent for exponent in range(7, 18))
+ CHUNK_SIZE_EDGES = tuple(1 << exponent for exponent in range(4, 12))
+ DEFAULT_QUEUE_CAPACITY = 8192
++FLUSH_INTERVAL_SECONDS = 1.0
+ _CLOSE_TIMEOUT_SECONDS = 1.0
+ _STOP = object()
+ _PREFIX_FIELDS = (  # noqa: SIM905
+@@ -69,6 +71,7 @@ class JSONLWriter:
+     ) -> None:
+         self._queue: queue.Queue[Any] = queue.Queue(capacity)
+         self.path = path
++        self.sidecar_path = path.with_name(f"{path.name}.footer.json")
+         self._encoder = msgspec.json.Encoder()
+         self._file = path.open("xb", buffering=1 << 20)
+         self._thread = threading.Thread(target=self._run, daemon=True)
+@@ -78,6 +81,9 @@ class JSONLWriter:
+         self.failure: Exception | None = None
+         self.encoded_records = self.written_records = 0
+         self.dropped_records = self._unreported_drops = 0
++        self._checkpoint_encoded_records = 0
++        self._checkpoint_dropped_records = 0
++        self._last_written_step_index: int | None = None
+         if start:
+             self.start()
+ 
+@@ -119,33 +125,90 @@ class JSONLWriter:
+         if self._writer_unavailable():
+             return self._drop(pending)
+         try:
+-            self._queue.put_nowait(payload)
++            self._queue.put_nowait(
++                (
++                    payload,
++                    self.encoded_records,
++                    self.dropped_records,
++                    int(record["step_index"]),
++                )
++            )
+         except queue.Full:
+             return self._drop(pending)
+         self._unreported_drops = 0
+         return True
+ 
++    def _write_sidecar(
++        self,
++        *,
++        encoded_records: int,
++        written_records: int,
++        dropped_records: int,
++        last_step_index: int | None,
++        final: bool,
++    ) -> None:
++        sidecar = dict(
++            schema=SCHEMA_VERSION,
++            record_type="footer_checkpoint",
++            stream=self.path.name,
++            encoded_records=encoded_records,
++            written_records=written_records,
++            dropped_records=dropped_records,
++            last_step_index=last_step_index,
++            checkpoint_wall_ns=time.time_ns(),
++            flush_interval_seconds=FLUSH_INTERVAL_SECONDS,
++            final=final,
++        )
++        temporary_path = self.sidecar_path.with_name(
++            f".{self.sidecar_path.name}.tmp-{os.getpid()}-{threading.get_ident()}"
++        )
++        try:
++            with temporary_path.open("wb") as temporary_file:
++                temporary_file.write(self._encoder.encode(sidecar) + b"\n")
++                temporary_file.flush()
++            os.replace(temporary_path, self.sidecar_path)
++        finally:
++            with suppress(FileNotFoundError):
++                temporary_path.unlink()
++
++    def _flush_checkpoint(self) -> None:
++        self._file.flush()
++        self._write_sidecar(
++            encoded_records=self._checkpoint_encoded_records,
++            written_records=self.written_records,
++            dropped_records=self._checkpoint_dropped_records,
++            last_step_index=self._last_written_step_index,
++            final=False,
++        )
++
+     def _run(self) -> None:
+         buffered = 0
+         last_flush = time.monotonic()
+         try:
+             while True:
+                 try:
+-                    item = self._queue.get(timeout=1.0)
++                    item = self._queue.get(timeout=FLUSH_INTERVAL_SECONDS)
+                 except queue.Empty:
+-                    self._file.flush()
++                    self._flush_checkpoint()
+                     buffered = 0
+                     last_flush = time.monotonic()
+                     continue
+                 try:
+                     if item is _STOP:
+                         break
+-                    self._file.write(item)
+-                    buffered += len(item)
++                    payload, encoded, dropped, step_index = item
++                    self._file.write(payload)
++                    buffered += len(payload)
+                     self.written_records += 1
++                    self._checkpoint_encoded_records = encoded
++                    self._checkpoint_dropped_records = dropped
++                    self._last_written_step_index = step_index
+                     now = time.monotonic()
+-                    if buffered >= 1 << 20 or now - last_flush >= 1.0:
+-                        self._file.flush()
++                    if (
++                        buffered >= 1 << 20
++                        or now - last_flush >= FLUSH_INTERVAL_SECONDS
++                    ):
++                        self._flush_checkpoint()
+                         buffered = 0
+                         last_flush = now
+                 finally:
+@@ -163,6 +226,13 @@ class JSONLWriter:
+             try:
+                 self._file.write(self._encoder.encode(footer) + b"\n")
+                 self._file.flush()
++                self._write_sidecar(
++                    encoded_records=footer["encoded_records"],
++                    written_records=footer["written_records"],
++                    dropped_records=footer["dropped_records"],
++                    last_step_index=self._last_written_step_index,
++                    final=True,
++                )
+             except Exception as error:
+                 self._record_failure(error)
+             finally:
+-- 
+2.43.0
+

patches/vllm-0.24.0-opprof/0007-Recreate-scheduled-torch-profiler-between-windows.patch

@@ -0,0 +1,27 @@
+From 23450fb21ac255b0cf710f4ee965ee694921975d Mon Sep 17 00:00:00 2001
+From: Gahow Wang <gahow.wang@gmail.com>
+Date: Sun, 12 Jul 2026 13:12:52 +0800
+Subject: [PATCH] Recreate scheduled torch profiler between windows
+
+---
+ vllm/v1/worker/gpu_worker.py | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py
+index 5e266a3..0058f96 100644
+--- a/vllm/v1/worker/gpu_worker.py
++++ b/vllm/v1/worker/gpu_worker.py
+@@ -978,6 +978,10 @@ class Worker(WorkerBase):
+                 logger.warning("Profiler was not started, nothing to stop.")
+                 return
+             self.profiler.stop()
++            # A scheduled torch.profiler.profile does not reset its schedule
++            # after stop(). Recreate it for the next /start_profile window.
++            if isinstance(self.profiler, TorchProfilerWrapper):
++                self.profiler = None
+ 
+     def execute_dummy_batch(self) -> None:
+         num_tokens = getattr(self.model_runner, "uniform_decode_query_len", 1)
+-- 
+2.43.0
+

patches/vllm-0.24.0-opprof/README.md

@@ -0,0 +1,119 @@
+# vLLM 0.24.0 OpProf patch series
+
+## Goal
+
+Apply the accepted OpProf Layer-1 instrumentation to exactly vLLM `v0.24.0`
+at base commit `ee0da84ab9e04ac7610e28580af62c365e898389`. The series adds one
+scheduler-owned composition record per step without installing new runtime
+dependencies or changing GPU kernels.
+
+## Contents
+
+- `0001-Add-lightweight-per-step-OpProf-telemetry.patch`: adds the environment
+  switch, import-light JSONL recorder/writer, scheduler hooks, and reuse of the
+  existing CUDA-graph stat. Writer failures are exposed without blocking
+  producers or shutdown, and request histograms are accumulated in-place.
+- `0002-Add-standalone-OpProf-telemetry-tests.patch`: adds CPU-only tests that
+  load the recorder directly without importing or installing vLLM or torch,
+  including ENOSPC, golden-record, and zero-token regressions.
+- `0003-Log-the-OpProf-output-path-at-startup.patch`: logs the resolved JSONL
+  output path and covers it in the standalone shutdown test.
+- `0004-Exclude-OpProf-output-path-from-compile-cache-key.patch`: prevents the
+  per-run telemetry destination from invalidating vLLM's torch.compile/AOT
+  cache and adds an import-light regression test.
+- `0005-Keep-compile-factor-regression-import-light.patch`: isolates the new
+  regression from torch in full vLLM test environments.
+- `0006-Checkpoint-OpProf-accounting-across-hard-kills.patch`: atomically
+  checkpoints balanced writer counters beside each JSONL stream once per
+  flush interval, with clean-close and hard-kill regressions.
+- `0007-Recreate-scheduled-torch-profiler-between-windows.patch`: discards a
+  stopped scheduled torch-profiler wrapper so each subsequent official profile
+  endpoint call receives a fresh 2+8 schedule and emits its own trace.
+- `apply.sh`: verifies the exact base, refuses dirty/wrong revisions, applies
+  all numbered patches with `git am`, and exits successfully only when the
+  exact series is already applied directly on the required base.
+- `pytest-evidence.txt`: exact isolated test command, dependency versions, and
+  all-pass summary.
+
+The source branch tip used to generate the patches is
+`23450fb21ac255b0cf710f4ee965ee694921975d` (`opprof`).
+
+## Apply
+
+Prerequisite: a clean checkout whose `HEAD` is the exact base commit.
+
+```bash
+./patches/vllm-0.24.0-opprof/apply.sh /path/to/vllm-v0.24.0
+```
+
+Running the command again is a no-op only when the five matching patch commits
+are rooted directly at the required base. A partially applied series, dirty
+tree, unrelated commit, or any other `HEAD` is rejected instead of being
+guessed around.
+
+## Enable and output
+
+Set an absolute output directory before starting vLLM:
+
+```bash
+export VLLM_OPPROF_DIR=/absolute/path/to/run/opprof
+```
+
+Unset or empty disables the feature before recorder construction. Combining it
+with `--disable-log-stats` fails fast, as approved.
+
+Each EngineCore/DP scheduler writes one file named approximately
+`opprof-v1-dp0-pid1234-<start_ns>.jsonl`. Records contain schema/engine/step and
+timestamps; scheduled prefill/decode composition; first/middle/final/unsplit
+prefill chunks; 12-bin context and 9-bin chunk-size histograms; preemptions;
+running/waiting/deferred queues; KV blocks/usage; local/external prefix deltas;
+CUDA-graph hit/mode/bucket/padding; explicit null Layer-1 MoE load; and drop-gap
+accounting. A clean close writes a final writer-count footer in the stream.
+
+Every JSONL flush also atomically replaces
+`<stream>.footer.json` through a same-directory temporary file. The sidecar
+contains the encoded, written, and dropped counts through that durable flush,
+the last written step index, a wall-clock timestamp, the one-second flush
+interval, and whether it is final. Queue entries carry their submission
+ordinal and cumulative drops, so a periodic checkpoint always satisfies
+`encoded = written + dropped` without decoding records in the writer thread.
+On clean close the in-stream footer is authoritative and the final sidecar must
+agree with all three counters. If a hard kill prevents the in-stream footer,
+the latest sidecar is authoritative: the decoded data-line count must equal
+its `written_records`, its final data-line step must equal
+`last_step_index`, and its counters must balance. Data after that checkpoint
+may be lost, bounded by at most the configured one-second flush interval.
+
+The bounded queue holds 8192 encoded records. Producers never wait for disk;
+full queues or a failed writer drop the new record and report the gap on the
+next successful record. A writer I/O failure is exposed through recorder state,
+logged once, and cannot make shutdown wait indefinitely. The writer flushes at
+1 MiB, one second, or shutdown.
+
+## Test
+
+Only pytest and msgspec are required. `--confcutdir` prevents vLLM's global
+test configuration from importing its full dependency stack.
+
+```bash
+cd /path/to/vllm-v0.24.0
+uv run --no-project --with pytest --with msgspec \
+  pytest --confcutdir=tests/v1/core tests/v1/core/test_opprof.py -q
+```
+
+Expected summary:
+
+```text
+18 passed in 1.09s
+```
+
+## Caveats
+
+- Layer 1 intentionally records no expert-load arrays. Exact routed experts
+  remain a separate Layer-2 run.
+- `PIECEWISE` means graph-wrapped compiled regions, not full-step graph replay.
+- Phase 2 must measure the always-on overhead; acceptance requires the upper
+  bound of the 95% confidence interval to remain below 3% for every primary
+  serving metric.
+- Primary campaign topology is TP1 on community BF16 Qwen3-30B-A3B, with TP2
+  and TP4 counterpoints. Record the selected MoE backend log every run.

patches/vllm-0.24.0-opprof/apply.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+base_commit=ee0da84ab9e04ac7610e28580af62c365e898389
+script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+repo=${1:-.}
+patches=("$script_dir"/0*.patch)
+
+git -C "$repo" rev-parse --git-dir >/dev/null
+if [[ -n $(git -C "$repo" status --porcelain) ]]; then
+    echo "Refusing to apply to a dirty worktree." >&2
+    exit 1
+fi
+if ((${#patches[@]} == 0)) || [[ ! -f ${patches[0]} ]]; then
+    echo "No numbered patch files found in $script_dir" >&2
+    exit 1
+fi
+
+head_commit=$(git -C "$repo" rev-parse HEAD)
+mapfile -t recent_commits < <(
+    git -C "$repo" rev-list --max-count="${#patches[@]}" --reverse HEAD
+)
+patches_match=true
+((${#recent_commits[@]} == ${#patches[@]})) || patches_match=false
+for i in "${!patches[@]}"; do
+    $patches_match || break
+    patch_id=$(git patch-id --stable <"${patches[$i]}" | awk '{print $1}')
+    commit_id=$(
+        git -C "$repo" show --pretty=format: "${recent_commits[$i]}" |
+            git patch-id --stable | awk '{print $1}'
+    )
+    [[ $patch_id == "$commit_id" ]] || patches_match=false
+done
+first_parent=""
+if ((${#recent_commits[@]} > 0)); then
+    first_parent=$(
+        git -C "$repo" rev-parse --verify "${recent_commits[0]}^" 2>/dev/null || true
+    )
+fi
+if $patches_match && [[ $first_parent == "$base_commit" ]]; then
+    echo "OpProf patch series is already applied."
+    exit 0
+fi
+
+if [[ $head_commit == "$base_commit" ]]; then
+    git -C "$repo" am "${patches[@]}"
+    echo "Applied OpProf patch series to $repo"
+    exit 0
+fi
+if $patches_match; then
+    echo "Refusing to treat OpProf patches as already applied:" >&2
+    echo "first patch parent is $first_parent, expected $base_commit" >&2
+    exit 1
+fi
+
+echo "Refusing to apply: HEAD is $head_commit; expected $base_commit" >&2
+echo "or the exact OpProf patch series rooted at that base." >&2
+exit 1

patches/vllm-0.24.0-opprof/pytest-evidence.txt

@@ -0,0 +1,16 @@
+OpProf standalone pytest evidence
+Date: 2026-07-12
+Source branch: opprof
+Source tip: 23450fb21ac255b0cf710f4ee965ee694921975d
+Base: ee0da84ab9e04ac7610e28580af62c365e898389 (v0.24.0)
+Environment: Python 3.11.13, pytest 9.1.1, msgspec 0.21.1
+vLLM installed: no
+torch installed in isolated test environment: no
+GPU/remote access: no
+
+Command:
+uv run --no-project --with pytest --with msgspec pytest --confcutdir=tests/v1/core tests/v1/core/test_opprof.py -q
+
+Output:
+..................                                                       [100%]
+18 passed in 1.09s

runs/action-aware-v0/action_aware_client.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Add explicit MBBT/config provenance to the accepted Phase-6 replay client."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+PHASE6 = Path(__file__).resolve().parents[1] / "opprof-phase6"
+sys.path.insert(0, str(PHASE6))
+
+import opprof_phase6_client as base  # noqa: E402
+
+
+def parser() -> argparse.ArgumentParser:
+    result = argparse.ArgumentParser()
+    result.add_argument("command", choices=("warmup", "run-anchor"))
+    result.add_argument("--study", required=True)
+    result.add_argument("--cell", required=True)
+    result.add_argument("--anchor", type=float, required=True)
+    result.add_argument("--tp", type=int, required=True)
+    result.add_argument("--mns", type=int, required=True)
+    result.add_argument("--mbbt", type=int, required=True)
+    result.add_argument("--base-url", required=True)
+    result.add_argument("--result-dir", required=True)
+    result.add_argument("--disable-slo-early-stop", action="store_true")
+    return result
+
+
+def main() -> None:
+    args = parser().parse_args()
+    result = base.run_replay(args, warmup=args.command == "warmup")
+    result.update(
+        {
+            "schema": "action-aware-pilot-result-v0",
+            "config_id": args.cell,
+            "mbbt": args.mbbt,
+        }
+    )
+    base.atomic_json(Path(args.result_dir) / "result.json", result)
+    print(
+        json.dumps(
+            {
+                key: result[key]
+                for key in (
+                    "config_id",
+                    "mns",
+                    "mbbt",
+                    "kind",
+                    "pass_rate",
+                    "feasible",
+                )
+            },
+            sort_keys=True,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

runs/action-aware-v0/analyze_pilot.py

@@ -0,0 +1,697 @@
+#!/usr/bin/env python3
+"""Audit source-only constraint signals against crossed real interventions."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import statistics
+import sys
+from pathlib import Path
+from typing import Any, Iterable, Mapping
+
+
+HERE = Path(__file__).resolve().parent
+COMMON_STATE = HERE.parent / "telemetry-residual"
+sys.path.insert(0, str(COMMON_STATE))
+
+from common_state import summarize_engine  # noqa: E402
+
+
+SCHEMA = "action-aware-constraint-pilot-audit-v0"
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def atomic_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(
+        json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    os.replace(temporary, path)
+
+
+def numeric(values: Iterable[float]) -> dict[str, Any]:
+    finite = [float(value) for value in values]
+    if not finite:
+        raise ValueError("numeric summary requires values")
+    if any(not math.isfinite(value) for value in finite):
+        raise ValueError("numeric summary received non-finite values")
+    return {
+        "n": len(finite),
+        "min": min(finite),
+        "max": max(finite),
+        "distinct_n": len(set(finite)),
+    }
+
+
+def distribution(values: Iterable[float]) -> dict[str, Any]:
+    finite = [float(value) for value in values]
+    summary = numeric(finite)
+    return {
+        **summary,
+        "mean": statistics.fmean(finite),
+        "p50": quantile(finite, 0.50),
+        "p95": quantile(finite, 0.95),
+        "p99": quantile(finite, 0.99),
+    }
+
+
+def quantile(values: Iterable[float], probability: float) -> float:
+    ordered = sorted(float(value) for value in values)
+    if not ordered:
+        raise ValueError("quantile requires values")
+    position = probability * (len(ordered) - 1)
+    lower = math.floor(position)
+    upper = math.ceil(position)
+    if lower == upper:
+        return ordered[lower]
+    weight = position - lower
+    return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    records = []
+    with path.open(encoding="utf-8") as source:
+        for line_number, line in enumerate(source, 1):
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError as error:
+                raise ValueError(f"{path}:{line_number}: invalid JSON") from error
+    return records
+
+
+def binding_summary(
+    records: list[Mapping[str, Any]], *, mns: int, mbbt: int
+) -> dict[str, Any]:
+    if not records:
+        raise ValueError("binding summary requires scheduler records")
+    counts = {
+        "mns_exclusive": 0,
+        "mbbt_exclusive": 0,
+        "both": 0,
+        "waiting_unresolved": 0,
+        "waiting": 0,
+    }
+    running_utilization = []
+    token_utilization = []
+    kv_usage = []
+    preemptions = 0
+    for record in records:
+        waiting = int(record["queues"]["waiting"]) + int(
+            record["queues"]["deferred"]
+        )
+        running = int(record["queues"]["running"])
+        scheduled_tokens = int(record["prefill_tokens"]) + int(
+            record["decode_tokens"]
+        )
+        if running > mns:
+            raise ValueError("running requests exceed configured MNS")
+        if scheduled_tokens > mbbt:
+            raise ValueError("scheduled tokens exceed configured MBBT")
+        mns_hit = waiting > 0 and running == mns
+        mbbt_hit = waiting > 0 and scheduled_tokens == mbbt
+        if waiting > 0:
+            counts["waiting"] += 1
+            if mns_hit and mbbt_hit:
+                counts["both"] += 1
+            elif mns_hit:
+                counts["mns_exclusive"] += 1
+            elif mbbt_hit:
+                counts["mbbt_exclusive"] += 1
+            else:
+                counts["waiting_unresolved"] += 1
+        running_utilization.append(running / mns)
+        token_utilization.append(scheduled_tokens / mbbt)
+        kv_usage.append(float(record["kv"]["usage"]))
+        preemptions += int(record["preemptions"])
+    count = len(records)
+    return {
+        "records": count,
+        **{f"{name}_count": value for name, value in counts.items()},
+        **{f"{name}_fraction": value / count for name, value in counts.items()},
+        "running_utilization_mean": statistics.fmean(running_utilization),
+        "running_utilization_max": max(running_utilization),
+        "token_utilization_mean": statistics.fmean(token_utilization),
+        "token_utilization_max": max(token_utilization),
+        "kv_usage_mean": statistics.fmean(kv_usage),
+        "kv_usage_max": max(kv_usage),
+        "preemptions": preemptions,
+    }
+
+
+def telemetry_coverage(
+    records: list[Mapping[str, Any]], *, start_ns: int, end_ns: int
+) -> tuple[dict[str, float], bool]:
+    if not records:
+        raise ValueError("telemetry coverage requires records")
+    submit_gaps = [
+        (int(right["submit_mono_ns"]) - int(left["submit_mono_ns"])) / 1e9
+        for left, right in zip(records, records[1:], strict=False)
+    ]
+    uncovered_gaps = [
+        max(
+            0,
+            int(right["submit_mono_ns"]) - int(left["complete_mono_ns"]),
+        )
+        / 1e9
+        for left, right in zip(records, records[1:], strict=False)
+    ]
+    coverage = {
+        "start_gap_s": (int(records[0]["submit_mono_ns"]) - start_ns) / 1e9,
+        "end_gap_s": (end_ns - int(records[-1]["submit_mono_ns"])) / 1e9,
+        "max_internal_submit_gap_s": max(submit_gaps, default=0.0),
+        "max_uncovered_gap_s": max(uncovered_gaps, default=0.0),
+    }
+    covered = (
+        0.0 <= coverage["start_gap_s"] <= 1.0
+        and 0.0 <= coverage["end_gap_s"] <= 1.0
+        and 0.0 <= coverage["max_uncovered_gap_s"] <= 1.0
+    )
+    return coverage, covered
+
+
+def mechanism_summary(records: list[Mapping[str, Any]]) -> dict[str, Any]:
+    executed = [record for record in records if bool(record["model_executed"])]
+    if not executed:
+        raise ValueError("mechanism summary requires executed steps")
+    prefill = [record for record in executed if int(record["prefill_tokens"]) > 0]
+    decode_only = [
+        record for record in executed if int(record["prefill_tokens"]) == 0
+    ]
+    if not prefill or not decode_only:
+        raise ValueError("mechanism summary requires prefill and decode-only steps")
+
+    def durations_ms(selected: list[Mapping[str, Any]]) -> list[float]:
+        values = [
+            (int(record["complete_mono_ns"]) - int(record["submit_mono_ns"]))
+            / 1e6
+            for record in selected
+        ]
+        if any(value < 0.0 for value in values):
+            raise ValueError("engine step duration must be non-negative")
+        return values
+
+    chunk_keys = ("first", "middle", "final", "unsplit", "tokens")
+    chunks = {
+        key: sum(int(record["chunked_prefill"][key]) for record in executed)
+        for key in chunk_keys
+    }
+    prefill_tokens = [int(record["prefill_tokens"]) for record in prefill]
+    prefill_requests = sum(int(record["prefill_requests"]) for record in prefill)
+    prefix_queries = sum(
+        int(record["prefix"]["local"]["queries"]) for record in executed
+    )
+    prefix_hits = sum(
+        int(record["prefix"]["local"]["hits"]) for record in executed
+    )
+    invariants = {
+        "nonnegative_counts": all(
+            value >= 0
+            for value in (
+                *chunks.values(),
+                prefill_requests,
+                prefix_queries,
+                prefix_hits,
+            )
+        ),
+        "chunk_tokens_match_prefill_tokens": chunks["tokens"]
+        == sum(prefill_tokens),
+        "prefix_hits_bounded": 0 <= prefix_hits <= prefix_queries,
+    }
+    return {
+        "executed_steps": len(executed),
+        "step_duration_ms": distribution(durations_ms(executed)),
+        "prefill_steps": len(prefill),
+        "prefill_step_duration_ms": distribution(durations_ms(prefill)),
+        "decode_only_steps": len(decode_only),
+        "decode_only_step_duration_ms": distribution(durations_ms(decode_only)),
+        "prefill": {
+            "requests": prefill_requests,
+            "requests_per_step": prefill_requests / len(prefill),
+            "tokens": sum(prefill_tokens),
+            "tokens_per_step": distribution(prefill_tokens),
+            "chunks": chunks,
+        },
+        "prefix": {
+            "queries": prefix_queries,
+            "hits": prefix_hits,
+            "hit_rate": prefix_hits / prefix_queries if prefix_queries else 0.0,
+        },
+        "sanity": {"invariants": invariants},
+    }
+
+
+def request_summary(path: Path, expected_count: int) -> dict[str, Any]:
+    rows = load_jsonl(path)
+    if len(rows) != expected_count:
+        raise ValueError(f"request row count mismatch: {path}")
+    ttft = [float(row["ttft_ms"]) for row in rows if row["ttft_ms"] is not None]
+    tpot = [float(row["tpot_ms"]) for row in rows if row["tpot_ms"] is not None]
+    if not ttft or not tpot:
+        raise ValueError(f"missing request latency values: {path}")
+    return {
+        "ttft_ms": {f"p{int(p * 100)}": quantile(ttft, p) for p in (0.5, 0.95, 0.99)},
+        "tpot_ms": {f"p{int(p * 100)}": quantile(tpot, p) for p in (0.5, 0.95, 0.99)},
+    }
+
+
+def load_stream(session_root: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    streams = sorted((session_root / "opprof").glob("*.jsonl"))
+    sidecars = sorted((session_root / "opprof").glob("*.jsonl.footer.json"))
+    if len(streams) != 1 or len(sidecars) != 1:
+        raise ValueError(f"expected one OpProf stream and sidecar: {session_root}")
+    decoded = load_jsonl(streams[0])
+    records = [row for row in decoded if "step_index" in row]
+    footers = [row for row in decoded if row.get("record_type") == "footer"]
+    sidecar = json.loads(sidecars[0].read_text(encoding="utf-8"))
+    indexes = [int(row["step_index"]) for row in records]
+    invariants = {
+        "one_footer_last": len(footers) == 1 and decoded[-1] is footers[0],
+        "sidecar_final": sidecar.get("final") is True,
+        "zero_drops": sidecar.get("dropped_records") == 0,
+        "written_matches_records": sidecar.get("written_records") == len(records),
+        "contiguous_step_indexes": indexes == list(range(len(indexes))),
+        "monotonic_timestamps": all(
+            int(right["submit_mono_ns"]) >= int(left["submit_mono_ns"])
+            for left, right in zip(records, records[1:], strict=False)
+        ),
+    }
+    return records, {
+        "stream": str(streams[0]),
+        "stream_sha256": sha256_file(streams[0]),
+        "records": len(records),
+        "invariants": invariants,
+    }
+
+
+def analyze_run(
+    *,
+    run_root: Path,
+    config: Mapping[str, Any],
+    repetition: int,
+    expected: Mapping[str, Any],
+    stream_records: list[Mapping[str, Any]],
+    duration_s: float,
+    phase_fractions: list[float],
+) -> dict[str, Any]:
+    result_root = run_root / "sessions" / str(config["id"]) / f"rep{repetition}"
+    result_path = result_root / "result.json"
+    result = json.loads(result_path.read_text(encoding="utf-8"))
+    selection = result["selection"]
+    invariants = {
+        "result_schema": result.get("schema") == "action-aware-pilot-result-v0",
+        "config_id": result.get("config_id") == config["id"],
+        "tp": int(result.get("tp", -1)) == 4,
+        "mns": int(result.get("mns", -1)) == int(config["mns"]),
+        "mbbt": int(result.get("mbbt", -1)) == int(config["mbbt"]),
+        "uncensored": not bool(result.get("early_stopped", True)),
+        "slo_early_stop_disabled": result.get("slo_early_stop_disabled") is True,
+        "selection_count": int(selection["count"]) == int(expected["selected_count"]),
+        "request_accounting": int(result["observed_count"])
+        == int(expected["selected_count"]),
+        "request_hash": selection["request_id_order_sha256"]
+        == expected["request_id_order_sha256"],
+        "arrival_hash": selection["arrival_order_sha256"]
+        == expected["arrival_order_sha256"],
+        "length_hash": selection["raw_length_order_sha256"]
+        == expected["input_length_order_sha256"],
+    }
+    start_ns = int(result["interval"]["start_mono_ns"])
+    arrival_end_ns = start_ns + round(duration_s * 1e9)
+    full_records = [
+        record
+        for record in stream_records
+        if start_ns <= int(record["submit_mono_ns"]) <= arrival_end_ns
+    ]
+    if not full_records:
+        raise ValueError(f"no telemetry records in measured window: {result_path}")
+    coverage, invariants["telemetry_coverage"] = telemetry_coverage(
+        full_records, start_ns=start_ns, end_ns=arrival_end_ns
+    )
+    binding = binding_summary(
+        full_records, mns=int(config["mns"]), mbbt=int(config["mbbt"])
+    )
+    mechanism = mechanism_summary(full_records)
+    invariants["mechanism_summary"] = all(
+        mechanism["sanity"]["invariants"].values()
+    )
+    phases = {}
+    for fraction in phase_fractions:
+        phase_end = start_ns + round(duration_s * fraction * 1e9)
+        phase_records = [
+            record
+            for record in full_records
+            if int(record["submit_mono_ns"]) <= phase_end
+        ]
+        phases[f"{fraction:.2f}"] = binding_summary(
+            phase_records, mns=int(config["mns"]), mbbt=int(config["mbbt"])
+        )
+    state = summarize_engine(
+        full_records,
+        start_ns=start_ns,
+        end_ns=arrival_end_ns,
+        request_count=int(result["observed_count"]),
+    )
+    latency = request_summary(
+        result_root / "requests.jsonl", int(result["observed_count"])
+    )
+    return {
+        "config_id": config["id"],
+        "mns": int(config["mns"]),
+        "mbbt": int(config["mbbt"]),
+        "repetition": repetition,
+        "result_path": str(result_path),
+        "result_sha256": sha256_file(result_path),
+        "selection": {
+            "count": int(selection["count"]),
+            "request_id_order_sha256": selection["request_id_order_sha256"],
+            "arrival_order_sha256": selection["arrival_order_sha256"],
+            "raw_length_order_sha256": selection["raw_length_order_sha256"],
+        },
+        "outcome": {
+            "pass_rate": float(result["pass_rate"]),
+            "feasible": bool(result["feasible"]),
+            "slo_pass_count": int(result["slo_pass_count"]),
+            "slo_goodput_req_s": int(result["slo_pass_count"]) / duration_s,
+            "elapsed_s": float(result["interval"]["elapsed_s"]),
+            **latency,
+        },
+        "binding": binding,
+        "mechanism": mechanism,
+        "phases": phases,
+        "state": state,
+        "coverage": coverage,
+        "invariants": invariants,
+    }
+
+
+def median(values: Iterable[float]) -> float:
+    return float(statistics.median(float(value) for value in values))
+
+
+def evaluate_decisions(
+    runs: list[Mapping[str, Any]], manifest: Mapping[str, Any]
+) -> dict[str, Any]:
+    by_key = {
+        (str(run["config_id"]), int(run["repetition"])): run for run in runs
+    }
+    repetitions = sorted(int(key) for key in manifest["repetitions"])
+    regime_results = {}
+    all_predictions = []
+    crossed_pass = True
+    binding_pass = True
+    material_ambiguity = False
+    for regime_name, regime in manifest["regimes"].items():
+        rows = []
+        source_runs = []
+        for repetition in repetitions:
+            source = by_key[(str(regime["source"]), repetition)]
+            mns_target = by_key[(str(regime["actions"]["mns"]), repetition)]
+            mbbt_target = by_key[(str(regime["actions"]["mbbt"]), repetition)]
+            source_runs.append(source)
+            source_goodput = float(source["outcome"]["slo_goodput_req_s"])
+            mns_goodput = float(mns_target["outcome"]["slo_goodput_req_s"])
+            mbbt_goodput = float(mbbt_target["outcome"]["slo_goodput_req_s"])
+            observed = (
+                "mns"
+                if mns_goodput > mbbt_goodput
+                else "mbbt"
+                if mbbt_goodput > mns_goodput
+                else "tie"
+            )
+            mns_score = float(source["binding"]["mns_exclusive_fraction"])
+            mbbt_score = float(source["binding"]["mbbt_exclusive_fraction"])
+            predicted = (
+                "mns"
+                if mns_score > mbbt_score
+                else "mbbt"
+                if mbbt_score > mns_score
+                else "tie"
+            )
+            phase_predictions = {}
+            for phase, summary in source["phases"].items():
+                left = float(summary["mns_exclusive_fraction"])
+                right = float(summary["mbbt_exclusive_fraction"])
+                phase_predictions[phase] = (
+                    "mns" if left > right else "mbbt" if right > left else "tie"
+                )
+            margin = (
+                abs(mns_goodput - mbbt_goodput) / source_goodput
+                if source_goodput > 0
+                else None
+            )
+            row = {
+                "repetition": repetition,
+                "source_goodput_req_s": source_goodput,
+                "mns_target_goodput_req_s": mns_goodput,
+                "mbbt_target_goodput_req_s": mbbt_goodput,
+                "observed_winner": observed,
+                "predicted_winner": predicted,
+                "prediction_correct": predicted == observed,
+                "relative_winner_margin_over_source": margin,
+                "mns_exclusive_fraction": mns_score,
+                "mbbt_exclusive_fraction": mbbt_score,
+                "phase_predictions": phase_predictions,
+                "phase_stable": all(value == predicted for value in phase_predictions.values()),
+            }
+            rows.append(row)
+            all_predictions.append(row)
+
+        expected_winner = "mns" if regime_name == "A" else "mbbt"
+        minimum_margin = float(manifest["gates"]["minimum_relative_winner_margin"])
+        regime_crossed = all(
+            row["observed_winner"] == expected_winner
+            and row["relative_winner_margin_over_source"] is not None
+            and row["relative_winner_margin_over_source"] >= minimum_margin
+            for row in rows
+        )
+        crossed_pass &= regime_crossed
+        winning_key = f"{expected_winner}_exclusive_fraction"
+        losing_key = (
+            "mbbt_exclusive_fraction" if expected_winner == "mns" else "mns_exclusive_fraction"
+        )
+        winning_median = median(row[winning_key] for row in rows)
+        losing_median = median(row[losing_key] for row in rows)
+        ratio_pass = winning_median >= float(
+            manifest["gates"]["minimum_exclusive_ratio"]
+        ) * losing_median
+        regime_binding = (
+            all(row["prediction_correct"] and row["phase_stable"] for row in rows)
+            and winning_median
+            >= float(manifest["gates"]["minimum_exclusive_fraction"])
+            and ratio_pass
+        )
+        binding_pass &= regime_binding
+        ambiguity_median = median(
+            float(run["binding"]["both_fraction"])
+            + float(run["binding"]["waiting_unresolved_fraction"])
+            for run in source_runs
+        )
+        score_gap_median = median(
+            abs(
+                float(run["binding"]["mns_exclusive_fraction"])
+                - float(run["binding"]["mbbt_exclusive_fraction"])
+            )
+            for run in source_runs
+        )
+        kv_max_median = median(
+            float(run["binding"]["kv_usage_max"]) for run in source_runs
+        )
+        any_preemption = any(
+            int(run["binding"]["preemptions"]) > 0 for run in source_runs
+        )
+        regime_material = (
+            ambiguity_median >= score_gap_median
+            or kv_max_median >= float(manifest["gates"]["material_kv_usage"])
+            or any_preemption
+        )
+        material_ambiguity |= regime_material
+        regime_results[regime_name] = {
+            "source": regime["source"],
+            "actions": regime["actions"],
+            "expected_winner": expected_winner,
+            "crossed_response_pass": regime_crossed,
+            "binding_pass": regime_binding,
+            "winning_exclusive_median": winning_median,
+            "losing_exclusive_median": losing_median,
+            "exclusive_ratio_pass": ratio_pass,
+            "ambiguity_median": ambiguity_median,
+            "exclusive_gap_median": score_gap_median,
+            "kv_usage_max_median": kv_max_median,
+            "any_preemption": any_preemption,
+            "material_ambiguity": regime_material,
+            "repetitions": rows,
+        }
+
+    if not crossed_pass:
+        decision = "STOP_WORKLOAD_NOT_CROSSED"
+    elif not binding_pass:
+        decision = "STOP_BINDING_NOT_PREDICTIVE"
+    elif material_ambiguity:
+        decision = "OPEN_EXACT_ATTRIBUTION_ABLATION"
+    else:
+        decision = "STOP_NO_NEW_INSTRUMENTATION_NEEDED"
+    correct = sum(int(row["prediction_correct"]) for row in all_predictions)
+    return {
+        "decision": decision,
+        "crossed_response_pass": crossed_pass,
+        "binding_pass": binding_pass,
+        "material_ambiguity": material_ambiguity,
+        "regimes": regime_results,
+        "baselines": {
+            "always_mns_correct": sum(
+                int(row["observed_winner"] == "mns") for row in all_predictions
+            ),
+            "always_mbbt_correct": sum(
+                int(row["observed_winner"] == "mbbt") for row in all_predictions
+            ),
+            "binding_correct": correct,
+            "decision_count": len(all_predictions),
+        },
+    }
+
+
+def analyze(run_root: Path, manifest_path: Path) -> dict[str, Any]:
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    if manifest.get("schema") not in {
+        "action-aware-constraint-pilot-manifest-v0",
+        "action-aware-constraint-pilot-manifest-v1",
+    }:
+        raise ValueError("unexpected manifest schema")
+    duration_s = float(manifest["engine"]["duration_s"])
+    phase_fractions = [float(value) for value in manifest["gates"]["phase_fractions"]]
+    runs = []
+    stream_audits = []
+    for config in manifest["configs"]:
+        session_root = run_root / "sessions" / str(config["id"])
+        stream_records, stream_audit = load_stream(session_root)
+        stream_audit["config_id"] = config["id"]
+        stream_audits.append(stream_audit)
+        for repetition in sorted(int(key) for key in manifest["repetitions"]):
+            runs.append(
+                analyze_run(
+                    run_root=run_root,
+                    config=config,
+                    repetition=repetition,
+                    expected=manifest["repetitions"][str(repetition)]["selection"],
+                    stream_records=stream_records,
+                    duration_s=duration_s,
+                    phase_fractions=phase_fractions,
+                )
+            )
+    invariants = {
+        "fifteen_runs": len(runs) == 15,
+        "five_streams": len(stream_audits) == 5,
+        "all_run_invariants": all(
+            all(bool(value) for value in run["invariants"].values()) for run in runs
+        ),
+        "all_stream_invariants": all(
+            all(bool(value) for value in stream["invariants"].values())
+            for stream in stream_audits
+        ),
+        "nonnegative_counters": all(
+            all(
+                float(run["binding"][key]) >= 0
+                for key in (
+                    "mns_exclusive_count",
+                    "mbbt_exclusive_count",
+                    "both_count",
+                    "waiting_unresolved_count",
+                    "preemptions",
+                )
+            )
+            for run in runs
+        ),
+        "ratios_bounded": all(
+            all(
+                0.0 <= float(run["binding"][key]) <= 1.0
+                for key in (
+                    "mns_exclusive_fraction",
+                    "mbbt_exclusive_fraction",
+                    "both_fraction",
+                    "waiting_unresolved_fraction",
+                    "kv_usage_mean",
+                    "kv_usage_max",
+                )
+            )
+            for run in runs
+        ),
+        "per_config_results_not_all_identical": len(
+            {float(run["outcome"]["pass_rate"]) for run in runs}
+        )
+        > 1,
+    }
+    red_flags = [name for name, passed in invariants.items() if not passed]
+    decisions = (
+        evaluate_decisions(runs, manifest)
+        if not red_flags
+        else {
+            "decision": "STOP_DATA_INVALID",
+            "crossed_response_pass": False,
+            "binding_pass": False,
+            "material_ambiguity": False,
+            "regimes": {},
+            "baselines": {},
+        }
+    )
+    payload = {
+        "schema": SCHEMA,
+        "decision": decisions["decision"],
+        "manifest": str(manifest_path),
+        "manifest_sha256": sha256_file(manifest_path),
+        "run_root": str(run_root),
+        "runs": runs,
+        "streams": stream_audits,
+        "decision_audit": decisions,
+        "sanity": {
+            "runs": len(runs),
+            "pass_rate": numeric(run["outcome"]["pass_rate"] for run in runs),
+            "slo_goodput_req_s": numeric(
+                run["outcome"]["slo_goodput_req_s"] for run in runs
+            ),
+            "telemetry_records_per_run": numeric(
+                run["binding"]["records"] for run in runs
+            ),
+            "mns_values": numeric(run["mns"] for run in runs),
+            "mbbt_values": numeric(run["mbbt"] for run in runs),
+            "invariants": invariants,
+            "red_flags": red_flags,
+        },
+    }
+    return payload
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-root", type=Path, required=True)
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    payload = analyze(args.run_root, args.manifest)
+    atomic_json(args.output, payload)
+    print(
+        json.dumps(
+            {
+                "decision": payload["decision"],
+                "sanity": payload["sanity"],
+                "decision_audit": payload["decision_audit"],
+            },
+            indent=2,
+            sort_keys=True,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

runs/action-aware-v0/pilot-manifest-v1.json

@@ -0,0 +1,227 @@
+{
+  "budget": {
+    "expected_h20_hours": [
+      6.0,
+      7.2
+    ],
+    "expected_wall_minutes": [
+      90,
+      110
+    ],
+    "global_hard_cap_h20_hours": 8.0,
+    "hard_cap_h20_hours": 7.614013100465138,
+    "prior_attempt_artifact": "/home/admin/cpfs/wjh/action-aware-constraint-v0-20260714/operational-stop-v0.json",
+    "prior_attempt_h20_hours": 0.38598689953486126,
+    "safety_h20_hours": 0.25,
+    "session_estimate_h20_hours": 1.35
+  },
+  "burnin": {
+    "anchor": 0.18919793755240089,
+    "arrival_order_sha256": "6c0ac4cb9a30ef501eeeacc8e6cc631c345e976db5ccf530ea5a1ec706d62a24",
+    "input_length_order_sha256": "7939cc20e1a00d1031d27d71508789f38decbbbb6ea59a1df18b2ec342fd2ef8",
+    "offered_req_s": 8.5,
+    "offered_req_s_per_gpu": 2.125,
+    "request_id_order_sha256": "84f4809acbc8acd3b1d14dfa357134a1dc0b9287341624b33f598dafeef54dc7",
+    "selected_count": 510,
+    "study": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/studies/burnin-tp4.json",
+    "study_sha256": "5d6c2098042909a863efd3112818fbee9bafe96f22898ac98b66846dbe1fef0f"
+  },
+  "configs": [
+    {
+      "id": "b_base",
+      "mbbt": 2048,
+      "mns": 64,
+      "repetition_order": [
+        1,
+        2,
+        3
+      ]
+    },
+    {
+      "id": "a_base",
+      "mbbt": 8192,
+      "mns": 16,
+      "repetition_order": [
+        2,
+        3,
+        1
+      ]
+    },
+    {
+      "id": "shared",
+      "mbbt": 8192,
+      "mns": 64,
+      "repetition_order": [
+        3,
+        1,
+        2
+      ]
+    },
+    {
+      "id": "b_mns",
+      "mbbt": 2048,
+      "mns": 128,
+      "repetition_order": [
+        1,
+        3,
+        2
+      ]
+    },
+    {
+      "id": "a_mbbt",
+      "mbbt": 16384,
+      "mns": 16,
+      "repetition_order": [
+        2,
+        1,
+        3
+      ]
+    }
+  ],
+  "engine": {
+    "burnin_max_elapsed_s": 90.0,
+    "client_timeout_s": 450.0,
+    "disable_slo_early_stop": true,
+    "duration_s": 300.0,
+    "tp": 4
+  },
+  "gates": {
+    "material_kv_usage": 0.9,
+    "minimum_exclusive_fraction": 0.1,
+    "minimum_exclusive_ratio": 5.0,
+    "minimum_relative_winner_margin": 0.1,
+    "phase_fractions": [
+      0.25,
+      0.5,
+      0.75,
+      1.0
+    ]
+  },
+  "regimes": {
+    "A": {
+      "actions": {
+        "mbbt": "a_mbbt",
+        "mns": "shared"
+      },
+      "source": "a_base"
+    },
+    "B": {
+      "actions": {
+        "mbbt": "shared",
+        "mns": "b_mns"
+      },
+      "source": "b_base"
+    }
+  },
+  "repetitions": {
+    "1": {
+      "merged_trace": {
+        "bytes": 337429767,
+        "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep1.jsonl",
+        "request_id_scheme": "sha256(source_sha256:line_number:original_id)",
+        "rows": 9420,
+        "sha256": "68983266aa0e66aa589562f7c08edbd966f9ba4405e20c105adb43777d2dfbf5",
+        "source_sha256": [
+          "b242d1d9086df3accab57b4c92445d5edd581e12f47e12cea227aa63964c6930",
+          "d23b549f7b69af3647308677bbf76f818a3c226a1c98f9a9f93f09ceee46be87"
+        ],
+        "sources": [
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low1.jsonl",
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high1.jsonl"
+        ]
+      },
+      "selection": {
+        "anchor": 0.48686986110831465,
+        "arrival_order_sha256": "c2ad99986ce558da5901a9c5ec0a00bd69f198c981d8779235f2773a5c87f1c0",
+        "input_length_order_sha256": "9442bfebdc3fab5062dc1f4d688dc28c02afe3fd806c56dd8159f0ac7e6d0b94",
+        "offered_req_s": 8.5,
+        "offered_req_s_per_gpu": 2.125,
+        "request_id_order_sha256": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771",
+        "selected_count": 2550,
+        "target_count": 2550,
+        "target_req_s_per_gpu": 2.125
+      },
+      "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep1-tp4.json",
+      "study_sha256": "ecfff96e33d458eb1e3b9a6d24386f00cc6f1b19ff926e2ec6320b3f671a7ae3"
+    },
+    "2": {
+      "merged_trace": {
+        "bytes": 337509330,
+        "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep2.jsonl",
+        "request_id_scheme": "sha256(source_sha256:line_number:original_id)",
+        "rows": 9457,
+        "sha256": "f38e8938f6a481fc6725b71b21aa04ff7eaf79783cdfd6e41aa2f074156f00c2",
+        "source_sha256": [
+          "4cbb0baac082bd54af562ce2f39104c5c23b4671672da365a67b1e8c146adf9f",
+          "bb0bcd2564a88000f435f12feb21c7c902eafc9ea5fe916adfe9d1eae47f3f9a"
+        ],
+        "sources": [
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low2.jsonl",
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high2.jsonl"
+        ]
+      },
+      "selection": {
+        "anchor": 0.4825698948735577,
+        "arrival_order_sha256": "b9fc12cf3f86bc8a79bee65296e65aa2b8bf2aeca46b2887094c669adcbb9a00",
+        "input_length_order_sha256": "d8d4bd6fc8ba852a45605b673b6b3e4f33b58f459e69f2a032d226ee175b074e",
+        "offered_req_s": 8.5,
+        "offered_req_s_per_gpu": 2.125,
+        "request_id_order_sha256": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382",
+        "selected_count": 2550,
+        "target_count": 2550,
+        "target_req_s_per_gpu": 2.125
+      },
+      "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep2-tp4.json",
+      "study_sha256": "d92a576db031db24bb58f354ea725d7f7567cb76699d387117ac5a6c9317bbb9"
+    },
+    "3": {
+      "merged_trace": {
+        "bytes": 337450256,
+        "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep3.jsonl",
+        "request_id_scheme": "sha256(source_sha256:line_number:original_id)",
+        "rows": 9431,
+        "sha256": "3094084b0bb20cc02eecf465091a5c919b4e5b112f704cdc36a563d1efdcee46",
+        "source_sha256": [
+          "1f7ececb142f9a363d2d1ca25eb7b8488b2cc319a51b55faa384f2a3d51f2142",
+          "6f326234791e1cff4ff866bface0d097d0d6e3844eebb1c97653d8e9c35e9397"
+        ],
+        "sources": [
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low3.jsonl",
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high3.jsonl"
+        ]
+      },
+      "selection": {
+        "anchor": 0.48664343020532463,
+        "arrival_order_sha256": "efce7339e22d3618cb4d55e6b55bfddb2c563c18faba2a992d5829c13e3f55e9",
+        "input_length_order_sha256": "0792b05fff6729fbd92ab2bb4cb6d31bea7799e232ad42772936bc06efbafb54",
+        "offered_req_s": 8.5,
+        "offered_req_s_per_gpu": 2.125,
+        "request_id_order_sha256": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19",
+        "selected_count": 2550,
+        "target_count": 2550,
+        "target_req_s_per_gpu": 2.125
+      },
+      "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep3-tp4.json",
+      "study_sha256": "fb8ffe256dace32f4ca8a8d49b662d98c3b69b94ecc8fa826e43068b238884ab"
+    }
+  },
+  "sanity": {
+    "invariants": {
+      "all_repetition_orders_are_permutations": true,
+      "five_unique_configs": true,
+      "same_load_all_repetitions": true,
+      "shared_endpoint_reused_by_both_regimes": true,
+      "three_disjoint_repetitions": true
+    },
+    "red_flags": []
+  },
+  "schema": "action-aware-constraint-pilot-manifest-v1",
+  "source": {
+    "base_manifest": "/home/gahow/phd/aituner/runs/intervention-response-v2/pilot-manifest-v3.json",
+    "base_manifest_sha256": "273db1181dcc9d6b64439650d0642ebe553b12e6aa9adebfbe3758a7977e5611",
+    "source_trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260312_1000.jsonl",
+    "source_trace_sha256": "875ba869775deb78086477919f03b322da14e2673c7d070e26528c4190912757",
+    "window_id": "chat_w20260312_1000"
+  },
+  "status": "PASS"
+}

runs/action-aware-v0/pilot-manifest-v2.json

@@ -0,0 +1,227 @@
+{
+  "budget": {
+    "expected_h20_hours": [
+      6.0,
+      7.2
+    ],
+    "expected_wall_minutes": [
+      90,
+      110
+    ],
+    "global_hard_cap_h20_hours": 8.0,
+    "hard_cap_h20_hours": 7.295602157380846,
+    "prior_attempt_artifact": "/home/admin/cpfs/wjh/aituner/aituner-action-aware-20260714/runs/action-aware-v0/prior-attempts-v2.json",
+    "prior_attempt_h20_hours": 0.7043978426191542,
+    "safety_h20_hours": 0.25,
+    "session_estimate_h20_hours": 1.35
+  },
+  "burnin": {
+    "anchor": 0.18919793755240089,
+    "arrival_order_sha256": "6c0ac4cb9a30ef501eeeacc8e6cc631c345e976db5ccf530ea5a1ec706d62a24",
+    "input_length_order_sha256": "7939cc20e1a00d1031d27d71508789f38decbbbb6ea59a1df18b2ec342fd2ef8",
+    "offered_req_s": 8.5,
+    "offered_req_s_per_gpu": 2.125,
+    "request_id_order_sha256": "84f4809acbc8acd3b1d14dfa357134a1dc0b9287341624b33f598dafeef54dc7",
+    "selected_count": 510,
+    "study": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/studies/burnin-tp4.json",
+    "study_sha256": "5d6c2098042909a863efd3112818fbee9bafe96f22898ac98b66846dbe1fef0f"
+  },
+  "configs": [
+    {
+      "id": "b_base",
+      "mbbt": 2048,
+      "mns": 64,
+      "repetition_order": [
+        1,
+        2,
+        3
+      ]
+    },
+    {
+      "id": "a_base",
+      "mbbt": 8192,
+      "mns": 16,
+      "repetition_order": [
+        2,
+        3,
+        1
+      ]
+    },
+    {
+      "id": "shared",
+      "mbbt": 8192,
+      "mns": 64,
+      "repetition_order": [
+        3,
+        1,
+        2
+      ]
+    },
+    {
+      "id": "b_mns",
+      "mbbt": 2048,
+      "mns": 128,
+      "repetition_order": [
+        1,
+        3,
+        2
+      ]
+    },
+    {
+      "id": "a_mbbt",
+      "mbbt": 16384,
+      "mns": 16,
+      "repetition_order": [
+        2,
+        1,
+        3
+      ]
+    }
+  ],
+  "engine": {
+    "burnin_max_elapsed_s": 90.0,
+    "client_timeout_s": 450.0,
+    "disable_slo_early_stop": true,
+    "duration_s": 300.0,
+    "tp": 4
+  },
+  "gates": {
+    "material_kv_usage": 0.9,
+    "minimum_exclusive_fraction": 0.1,
+    "minimum_exclusive_ratio": 5.0,
+    "minimum_relative_winner_margin": 0.1,
+    "phase_fractions": [
+      0.25,
+      0.5,
+      0.75,
+      1.0
+    ]
+  },
+  "regimes": {
+    "A": {
+      "actions": {
+        "mbbt": "a_mbbt",
+        "mns": "shared"
+      },
+      "source": "a_base"
+    },
+    "B": {
+      "actions": {
+        "mbbt": "shared",
+        "mns": "b_mns"
+      },
+      "source": "b_base"
+    }
+  },
+  "repetitions": {
+    "1": {
+      "merged_trace": {
+        "bytes": 337429767,
+        "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep1.jsonl",
+        "request_id_scheme": "sha256(source_sha256:line_number:original_id)",
+        "rows": 9420,
+        "sha256": "68983266aa0e66aa589562f7c08edbd966f9ba4405e20c105adb43777d2dfbf5",
+        "source_sha256": [
+          "b242d1d9086df3accab57b4c92445d5edd581e12f47e12cea227aa63964c6930",
+          "d23b549f7b69af3647308677bbf76f818a3c226a1c98f9a9f93f09ceee46be87"
+        ],
+        "sources": [
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low1.jsonl",
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high1.jsonl"
+        ]
+      },
+      "selection": {
+        "anchor": 0.48686986110831465,
+        "arrival_order_sha256": "c2ad99986ce558da5901a9c5ec0a00bd69f198c981d8779235f2773a5c87f1c0",
+        "input_length_order_sha256": "9442bfebdc3fab5062dc1f4d688dc28c02afe3fd806c56dd8159f0ac7e6d0b94",
+        "offered_req_s": 8.5,
+        "offered_req_s_per_gpu": 2.125,
+        "request_id_order_sha256": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771",
+        "selected_count": 2550,
+        "target_count": 2550,
+        "target_req_s_per_gpu": 2.125
+      },
+      "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep1-tp4.json",
+      "study_sha256": "ecfff96e33d458eb1e3b9a6d24386f00cc6f1b19ff926e2ec6320b3f671a7ae3"
+    },
+    "2": {
+      "merged_trace": {
+        "bytes": 337509330,
+        "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep2.jsonl",
+        "request_id_scheme": "sha256(source_sha256:line_number:original_id)",
+        "rows": 9457,
+        "sha256": "f38e8938f6a481fc6725b71b21aa04ff7eaf79783cdfd6e41aa2f074156f00c2",
+        "source_sha256": [
+          "4cbb0baac082bd54af562ce2f39104c5c23b4671672da365a67b1e8c146adf9f",
+          "bb0bcd2564a88000f435f12feb21c7c902eafc9ea5fe916adfe9d1eae47f3f9a"
+        ],
+        "sources": [
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low2.jsonl",
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high2.jsonl"
+        ]
+      },
+      "selection": {
+        "anchor": 0.4825698948735577,
+        "arrival_order_sha256": "b9fc12cf3f86bc8a79bee65296e65aa2b8bf2aeca46b2887094c669adcbb9a00",
+        "input_length_order_sha256": "d8d4bd6fc8ba852a45605b673b6b3e4f33b58f459e69f2a032d226ee175b074e",
+        "offered_req_s": 8.5,
+        "offered_req_s_per_gpu": 2.125,
+        "request_id_order_sha256": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382",
+        "selected_count": 2550,
+        "target_count": 2550,
+        "target_req_s_per_gpu": 2.125
+      },
+      "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep2-tp4.json",
+      "study_sha256": "d92a576db031db24bb58f354ea725d7f7567cb76699d387117ac5a6c9317bbb9"
+    },
+    "3": {
+      "merged_trace": {
+        "bytes": 337450256,
+        "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep3.jsonl",
+        "request_id_scheme": "sha256(source_sha256:line_number:original_id)",
+        "rows": 9431,
+        "sha256": "3094084b0bb20cc02eecf465091a5c919b4e5b112f704cdc36a563d1efdcee46",
+        "source_sha256": [
+          "1f7ececb142f9a363d2d1ca25eb7b8488b2cc319a51b55faa384f2a3d51f2142",
+          "6f326234791e1cff4ff866bface0d097d0d6e3844eebb1c97653d8e9c35e9397"
+        ],
+        "sources": [
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low3.jsonl",
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high3.jsonl"
+        ]
+      },
+      "selection": {
+        "anchor": 0.48664343020532463,
+        "arrival_order_sha256": "efce7339e22d3618cb4d55e6b55bfddb2c563c18faba2a992d5829c13e3f55e9",
+        "input_length_order_sha256": "0792b05fff6729fbd92ab2bb4cb6d31bea7799e232ad42772936bc06efbafb54",
+        "offered_req_s": 8.5,
+        "offered_req_s_per_gpu": 2.125,
+        "request_id_order_sha256": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19",
+        "selected_count": 2550,
+        "target_count": 2550,
+        "target_req_s_per_gpu": 2.125
+      },
+      "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep3-tp4.json",
+      "study_sha256": "fb8ffe256dace32f4ca8a8d49b662d98c3b69b94ecc8fa826e43068b238884ab"
+    }
+  },
+  "sanity": {
+    "invariants": {
+      "all_repetition_orders_are_permutations": true,
+      "five_unique_configs": true,
+      "same_load_all_repetitions": true,
+      "shared_endpoint_reused_by_both_regimes": true,
+      "three_disjoint_repetitions": true
+    },
+    "red_flags": []
+  },
+  "schema": "action-aware-constraint-pilot-manifest-v1",
+  "source": {
+    "base_manifest": "/home/gahow/phd/aituner/runs/intervention-response-v2/pilot-manifest-v3.json",
+    "base_manifest_sha256": "273db1181dcc9d6b64439650d0642ebe553b12e6aa9adebfbe3758a7977e5611",
+    "source_trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260312_1000.jsonl",
+    "source_trace_sha256": "875ba869775deb78086477919f03b322da14e2673c7d070e26528c4190912757",
+    "window_id": "chat_w20260312_1000"
+  },
+  "status": "PASS"
+}

runs/action-aware-v0/pilot-manifest.json

@@ -0,0 +1,223 @@
+{
+  "budget": {
+    "expected_h20_hours": [
+      6.0,
+      7.2
+    ],
+    "expected_wall_minutes": [
+      90,
+      110
+    ],
+    "hard_cap_h20_hours": 8.0,
+    "safety_h20_hours": 0.25,
+    "session_estimate_h20_hours": 1.35
+  },
+  "burnin": {
+    "anchor": 0.18919793755240089,
+    "arrival_order_sha256": "6c0ac4cb9a30ef501eeeacc8e6cc631c345e976db5ccf530ea5a1ec706d62a24",
+    "input_length_order_sha256": "7939cc20e1a00d1031d27d71508789f38decbbbb6ea59a1df18b2ec342fd2ef8",
+    "offered_req_s": 8.5,
+    "offered_req_s_per_gpu": 2.125,
+    "request_id_order_sha256": "84f4809acbc8acd3b1d14dfa357134a1dc0b9287341624b33f598dafeef54dc7",
+    "selected_count": 510,
+    "study": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/studies/burnin-tp4.json",
+    "study_sha256": "5d6c2098042909a863efd3112818fbee9bafe96f22898ac98b66846dbe1fef0f"
+  },
+  "configs": [
+    {
+      "id": "b_base",
+      "mbbt": 256,
+      "mns": 64,
+      "repetition_order": [
+        1,
+        2,
+        3
+      ]
+    },
+    {
+      "id": "a_base",
+      "mbbt": 8192,
+      "mns": 16,
+      "repetition_order": [
+        2,
+        3,
+        1
+      ]
+    },
+    {
+      "id": "shared",
+      "mbbt": 8192,
+      "mns": 64,
+      "repetition_order": [
+        3,
+        1,
+        2
+      ]
+    },
+    {
+      "id": "b_mns",
+      "mbbt": 256,
+      "mns": 128,
+      "repetition_order": [
+        1,
+        3,
+        2
+      ]
+    },
+    {
+      "id": "a_mbbt",
+      "mbbt": 16384,
+      "mns": 16,
+      "repetition_order": [
+        2,
+        1,
+        3
+      ]
+    }
+  ],
+  "engine": {
+    "client_timeout_s": 450.0,
+    "disable_slo_early_stop": true,
+    "duration_s": 300.0,
+    "tp": 4
+  },
+  "gates": {
+    "material_kv_usage": 0.9,
+    "minimum_exclusive_fraction": 0.1,
+    "minimum_exclusive_ratio": 5.0,
+    "minimum_relative_winner_margin": 0.1,
+    "phase_fractions": [
+      0.25,
+      0.5,
+      0.75,
+      1.0
+    ]
+  },
+  "regimes": {
+    "A": {
+      "actions": {
+        "mbbt": "a_mbbt",
+        "mns": "shared"
+      },
+      "source": "a_base"
+    },
+    "B": {
+      "actions": {
+        "mbbt": "shared",
+        "mns": "b_mns"
+      },
+      "source": "b_base"
+    }
+  },
+  "repetitions": {
+    "1": {
+      "merged_trace": {
+        "bytes": 337429767,
+        "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep1.jsonl",
+        "request_id_scheme": "sha256(source_sha256:line_number:original_id)",
+        "rows": 9420,
+        "sha256": "68983266aa0e66aa589562f7c08edbd966f9ba4405e20c105adb43777d2dfbf5",
+        "source_sha256": [
+          "b242d1d9086df3accab57b4c92445d5edd581e12f47e12cea227aa63964c6930",
+          "d23b549f7b69af3647308677bbf76f818a3c226a1c98f9a9f93f09ceee46be87"
+        ],
+        "sources": [
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low1.jsonl",
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high1.jsonl"
+        ]
+      },
+      "selection": {
+        "anchor": 0.48686986110831465,
+        "arrival_order_sha256": "c2ad99986ce558da5901a9c5ec0a00bd69f198c981d8779235f2773a5c87f1c0",
+        "input_length_order_sha256": "9442bfebdc3fab5062dc1f4d688dc28c02afe3fd806c56dd8159f0ac7e6d0b94",
+        "offered_req_s": 8.5,
+        "offered_req_s_per_gpu": 2.125,
+        "request_id_order_sha256": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771",
+        "selected_count": 2550,
+        "target_count": 2550,
+        "target_req_s_per_gpu": 2.125
+      },
+      "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep1-tp4.json",
+      "study_sha256": "ecfff96e33d458eb1e3b9a6d24386f00cc6f1b19ff926e2ec6320b3f671a7ae3"
+    },
+    "2": {
+      "merged_trace": {
+        "bytes": 337509330,
+        "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep2.jsonl",
+        "request_id_scheme": "sha256(source_sha256:line_number:original_id)",
+        "rows": 9457,
+        "sha256": "f38e8938f6a481fc6725b71b21aa04ff7eaf79783cdfd6e41aa2f074156f00c2",
+        "source_sha256": [
+          "4cbb0baac082bd54af562ce2f39104c5c23b4671672da365a67b1e8c146adf9f",
+          "bb0bcd2564a88000f435f12feb21c7c902eafc9ea5fe916adfe9d1eae47f3f9a"
+        ],
+        "sources": [
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low2.jsonl",
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high2.jsonl"
+        ]
+      },
+      "selection": {
+        "anchor": 0.4825698948735577,
+        "arrival_order_sha256": "b9fc12cf3f86bc8a79bee65296e65aa2b8bf2aeca46b2887094c669adcbb9a00",
+        "input_length_order_sha256": "d8d4bd6fc8ba852a45605b673b6b3e4f33b58f459e69f2a032d226ee175b074e",
+        "offered_req_s": 8.5,
+        "offered_req_s_per_gpu": 2.125,
+        "request_id_order_sha256": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382",
+        "selected_count": 2550,
+        "target_count": 2550,
+        "target_req_s_per_gpu": 2.125
+      },
+      "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep2-tp4.json",
+      "study_sha256": "d92a576db031db24bb58f354ea725d7f7567cb76699d387117ac5a6c9317bbb9"
+    },
+    "3": {
+      "merged_trace": {
+        "bytes": 337450256,
+        "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep3.jsonl",
+        "request_id_scheme": "sha256(source_sha256:line_number:original_id)",
+        "rows": 9431,
+        "sha256": "3094084b0bb20cc02eecf465091a5c919b4e5b112f704cdc36a563d1efdcee46",
+        "source_sha256": [
+          "1f7ececb142f9a363d2d1ca25eb7b8488b2cc319a51b55faa384f2a3d51f2142",
+          "6f326234791e1cff4ff866bface0d097d0d6e3844eebb1c97653d8e9c35e9397"
+        ],
+        "sources": [
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low3.jsonl",
+          "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high3.jsonl"
+        ]
+      },
+      "selection": {
+        "anchor": 0.48664343020532463,
+        "arrival_order_sha256": "efce7339e22d3618cb4d55e6b55bfddb2c563c18faba2a992d5829c13e3f55e9",
+        "input_length_order_sha256": "0792b05fff6729fbd92ab2bb4cb6d31bea7799e232ad42772936bc06efbafb54",
+        "offered_req_s": 8.5,
+        "offered_req_s_per_gpu": 2.125,
+        "request_id_order_sha256": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19",
+        "selected_count": 2550,
+        "target_count": 2550,
+        "target_req_s_per_gpu": 2.125
+      },
+      "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep3-tp4.json",
+      "study_sha256": "fb8ffe256dace32f4ca8a8d49b662d98c3b69b94ecc8fa826e43068b238884ab"
+    }
+  },
+  "sanity": {
+    "invariants": {
+      "all_repetition_orders_are_permutations": true,
+      "five_unique_configs": true,
+      "same_load_all_repetitions": true,
+      "shared_endpoint_reused_by_both_regimes": true,
+      "three_disjoint_repetitions": true
+    },
+    "red_flags": []
+  },
+  "schema": "action-aware-constraint-pilot-manifest-v0",
+  "source": {
+    "base_manifest": "/home/gahow/phd/aituner/runs/intervention-response-v2/pilot-manifest-v3.json",
+    "base_manifest_sha256": "273db1181dcc9d6b64439650d0642ebe553b12e6aa9adebfbe3758a7977e5611",
+    "source_trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260312_1000.jsonl",
+    "source_trace_sha256": "875ba869775deb78086477919f03b322da14e2673c7d070e26528c4190912757",
+    "window_id": "chat_w20260312_1000"
+  },
+  "status": "PASS"
+}

runs/action-aware-v0/pilot_controller.py

@@ -0,0 +1,638 @@
+#!/usr/bin/env python3
+"""Serialized controller for the crossed-constraint action-aware pilot."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shlex
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any, Mapping
+
+
+HERE = Path(__file__).resolve().parent
+PHASE6 = HERE.parent / "opprof-phase6"
+sys.path.insert(0, str(PHASE6))
+
+import opprof_phase6_controller as base  # noqa: E402
+
+
+SCHEMA = "action-aware-constraint-pilot-state-v0"
+
+
+def atomic_json(path: Path, payload: Any) -> None:
+    base.atomic_json(path, payload)
+
+
+def wait_all_idle(timeout_s: float = 30.0) -> None:
+    deadline = time.monotonic() + timeout_s
+    last_error: Exception | None = None
+    while time.monotonic() < deadline:
+        try:
+            base.assert_all_idle()
+            return
+        except RuntimeError as error:
+            last_error = error
+            time.sleep(1.0)
+    raise last_error or RuntimeError("GPU idle timeout")
+
+
+def configure(args: argparse.Namespace, manifest: Mapping[str, Any]) -> None:
+    base.WORKDIR = args.run_root.parent
+    base.RUN_ROOT = args.run_root
+    base.STATE = args.run_root / "controller-state.json"
+    base.SOURCE = args.vllm_source
+    base.VENV = args.venv
+    base.AITUNER = args.aituner_root
+    base.MODEL = args.model
+    base.CLIENT = args.client
+    base.GPU_LIMIT = float(manifest["budget"]["hard_cap_h20_hours"])
+    base.MARKER = "action-aware-constraint-pilot-v0"
+
+
+def validate_inputs(args: argparse.Namespace, manifest: Mapping[str, Any]) -> None:
+    if manifest.get("schema") not in {
+        "action-aware-constraint-pilot-manifest-v0",
+        "action-aware-constraint-pilot-manifest-v1",
+    }:
+        raise RuntimeError("unexpected action-aware manifest schema")
+    if manifest.get("status") != "PASS":
+        raise RuntimeError("action-aware manifest did not pass preflight")
+    red_flags = manifest.get("sanity", {}).get("red_flags", [])
+    if red_flags:
+        raise RuntimeError(f"manifest red flags: {red_flags}")
+
+    required = {
+        "manifest": args.manifest,
+        "aituner_root": args.aituner_root,
+        "vllm_source": args.vllm_source,
+        "venv_python": args.venv / "bin/python",
+        "venv_vllm": args.venv / "bin/vllm",
+        "model": args.model,
+        "client": args.client,
+        "burnin_study": Path(manifest["burnin"]["study"]),
+    }
+    for repetition, item in manifest["repetitions"].items():
+        required[f"rep{repetition}_study"] = Path(item["study"])
+        required[f"rep{repetition}_trace"] = Path(item["merged_trace"]["path"])
+    missing = {name: str(path) for name, path in required.items() if not path.exists()}
+    if missing:
+        raise RuntimeError(f"action-aware input paths missing: {missing}")
+
+
+def config_map(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
+    return {str(item["id"]): dict(item) for item in manifest["configs"]}
+
+
+def server_command(
+    config: Mapping[str, Any], *, gpus: tuple[int, ...], port: int
+) -> list[str]:
+    return [
+        "taskset",
+        "-c",
+        base.cpu_mask(gpus),
+        str(base.VENV / "bin/vllm"),
+        "serve",
+        str(base.MODEL),
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(port),
+        "--served-model-name",
+        "qwen3-30b-a3b-community",
+        "--max-num-batched-tokens",
+        str(config["mbbt"]),
+        "--max-num-seqs",
+        str(config["mns"]),
+        "--tensor-parallel-size",
+        "4",
+        "--shutdown-timeout",
+        "120",
+    ]
+
+
+def client_command(
+    entry: Mapping[str, Any],
+    config: Mapping[str, Any],
+    *,
+    study: str,
+    anchor: float,
+    output: Path,
+    warmup: bool,
+) -> list[str]:
+    command = [
+        "taskset",
+        "-c",
+        base.cpu_mask(entry["gpus"]),
+        str(base.VENV / "bin/python"),
+        str(base.CLIENT),
+        "warmup" if warmup else "run-anchor",
+        "--study",
+        study,
+        "--cell",
+        str(config["id"]),
+        "--anchor",
+        str(anchor),
+        "--tp",
+        "4",
+        "--mns",
+        str(config["mns"]),
+        "--mbbt",
+        str(config["mbbt"]),
+        "--base-url",
+        f"http://127.0.0.1:{entry['port']}",
+        "--result-dir",
+        str(output),
+        "--disable-slo-early-stop",
+    ]
+    return command
+
+
+def remaining_projection(
+    manifest: Mapping[str, Any], *, completed_sessions: int
+) -> float:
+    remaining = len(manifest["configs"]) - completed_sessions
+    return (
+        remaining * float(manifest["budget"]["session_estimate_h20_hours"])
+        + float(manifest["budget"]["safety_h20_hours"])
+    )
+
+
+def dry_run_plan(
+    args: argparse.Namespace, manifest: Mapping[str, Any]
+) -> dict[str, Any]:
+    sessions = []
+    for index, config in enumerate(manifest["configs"]):
+        entry = {"gpus": (0, 1, 2, 3), "port": 9050 + index}
+        session_root = args.run_root / "sessions" / str(config["id"])
+        first_repetition = str(config["repetition_order"][0])
+        first = manifest["repetitions"][first_repetition]
+        commands = {
+            "server": server_command(config, gpus=entry["gpus"], port=entry["port"]),
+            "warmup": client_command(
+                entry,
+                config,
+                study=first["study"],
+                anchor=float(first["selection"]["anchor"]),
+                output=session_root / "warmup",
+                warmup=True,
+            ),
+            "burnin": client_command(
+                entry,
+                config,
+                study=manifest["burnin"]["study"],
+                anchor=float(manifest["burnin"]["anchor"]),
+                output=session_root / "burnin",
+                warmup=False,
+            ),
+        }
+        for repetition in config["repetition_order"]:
+            item = manifest["repetitions"][str(repetition)]
+            commands[f"rep{repetition}"] = client_command(
+                entry,
+                config,
+                study=item["study"],
+                anchor=float(item["selection"]["anchor"]),
+                output=session_root / f"rep{repetition}",
+                warmup=False,
+            )
+        sessions.append(
+            {
+                "config": config["id"],
+                "mns": config["mns"],
+                "mbbt": config["mbbt"],
+                "port": entry["port"],
+                "repetition_order": config["repetition_order"],
+                "commands": {
+                    role: shlex.join(command) for role, command in commands.items()
+                },
+            }
+        )
+    return {
+        "schema": "action-aware-constraint-pilot-dry-run-v0",
+        "status": "PASS",
+        "manifest": str(args.manifest),
+        "run_root": str(args.run_root),
+        "projected_h20_hours": remaining_projection(
+            manifest, completed_sessions=0
+        ),
+        "hard_cap_h20_hours": manifest["budget"]["hard_cap_h20_hours"],
+        "sessions": sessions,
+    }
+
+
+def load_state(path: Path, hard_cap: float) -> dict[str, Any]:
+    if path.exists():
+        return json.loads(path.read_text(encoding="utf-8"))
+    return {
+        "schema": SCHEMA,
+        "status": "initialized",
+        "hard_cap_h20_hours": hard_cap,
+        "gpu_hours_total": 0.0,
+        "completed_sessions": 0,
+        "sessions": {},
+        "failures": [],
+        "started_at": time.time(),
+    }
+
+
+def append_echo(run_root: Path, line: str) -> None:
+    run_root.mkdir(parents=True, exist_ok=True)
+    with (run_root / "launch-echo.log").open("a", encoding="utf-8") as target:
+        target.write(line + "\n")
+    print(line, flush=True)
+
+
+def start_server(
+    *,
+    args: argparse.Namespace,
+    config: Mapping[str, Any],
+    index: int,
+) -> dict[str, Any]:
+    gpus = (0, 1, 2, 3)
+    session_root = args.run_root / "sessions" / str(config["id"])
+    session_root.mkdir(parents=True, exist_ok=True)
+    port = 9050 + index
+    command = server_command(config, gpus=gpus, port=port)
+    with (session_root / "commands.log").open("a", encoding="utf-8") as log:
+        log.write(f"SERVER {shlex.join(command)}\n")
+    server_log = (session_root / "server.log").open("ab", buffering=0)
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "CUDA_VISIBLE_DEVICES": "0,1,2,3",
+            "VLLM_OPPROF_DIR": str(session_root / "opprof"),
+            "OPPROF_PHASE6_MARKER": base.MARKER,
+            "AITUNER_ROOT": str(base.AITUNER),
+            "HF_HUB_OFFLINE": "1",
+            "TRANSFORMERS_OFFLINE": "1",
+            "PYTHONUNBUFFERED": "1",
+        }
+    )
+    server = subprocess.Popen(
+        command,
+        cwd=base.SOURCE,
+        env=environment,
+        stdout=server_log,
+        stderr=subprocess.STDOUT,
+        start_new_session=True,
+    )
+    base.OWNED_PGIDS.add(server.pid)
+    return {
+        "cell": str(config["id"]),
+        "gpus": gpus,
+        "port": port,
+        "dir": session_root,
+        "server": server,
+        "server_handle": server_log,
+        "spawned_at": time.time(),
+        "results": [],
+    }
+
+
+def validate_result(
+    result: Mapping[str, Any],
+    *,
+    config: Mapping[str, Any],
+    selection: Mapping[str, Any],
+    role: str,
+    warmup: bool,
+) -> None:
+    if result.get("schema") != "action-aware-pilot-result-v0":
+        raise RuntimeError(f"unexpected result schema: {role}")
+    if result.get("config_id") != config["id"]:
+        raise RuntimeError(f"config id mismatch: {role}")
+    if int(result["tp"]) != 4:
+        raise RuntimeError(f"TP mismatch: {role}")
+    if int(result["mns"]) != int(config["mns"]):
+        raise RuntimeError(f"MNS mismatch: {role}")
+    if int(result["mbbt"]) != int(config["mbbt"]):
+        raise RuntimeError(f"MBBT mismatch: {role}")
+    if result.get("slo_early_stop_disabled") is not True:
+        raise RuntimeError(f"SLO early stop was not disabled: {role}")
+    if warmup:
+        if result["kind"] != "warmup" or int(result["selection"]["count"]) != 16:
+            raise RuntimeError(f"invalid warmup: {role}")
+        return
+    if bool(result["early_stopped"]):
+        raise RuntimeError(f"uncensored run early-stopped: {role}")
+    if int(result["selection"]["count"]) != int(selection["selected_count"]):
+        raise RuntimeError(f"selection count mismatch: {role}")
+    if int(result["observed_count"]) != int(selection["selected_count"]):
+        raise RuntimeError(f"request accounting mismatch: {role}")
+    for result_key, selection_key in (
+        ("request_id_order_sha256", "request_id_order_sha256"),
+        ("arrival_order_sha256", "arrival_order_sha256"),
+        ("raw_length_order_sha256", "input_length_order_sha256"),
+    ):
+        if result["selection"][result_key] != selection[selection_key]:
+            raise RuntimeError(f"selection hash mismatch {result_key}: {role}")
+
+
+def burnin_gate(
+    result: Mapping[str, Any],
+    *,
+    expected_count: int,
+    maximum_elapsed_s: float,
+) -> dict[str, Any]:
+    if result.get("kind") != "anchor":
+        raise RuntimeError("burnin gate received a non-anchor result")
+    if int(result["selection"]["count"]) != expected_count:
+        raise RuntimeError("burnin gate received the wrong request set")
+    elapsed_s = float(result["interval"]["elapsed_s"])
+    summary = {
+        "elapsed_s": elapsed_s,
+        "pass_rate": float(result["pass_rate"]),
+        "feasible": bool(result["feasible"]),
+    }
+    if elapsed_s > maximum_elapsed_s:
+        raise RuntimeError(
+            f"burnin throughput gate failed: {elapsed_s:.3f}s > "
+            f"{maximum_elapsed_s:.3f}s"
+        )
+    return summary
+
+
+def run_client(
+    *,
+    entry: dict[str, Any],
+    config: Mapping[str, Any],
+    role: str,
+    study: str,
+    selection: Mapping[str, Any],
+    output: Path,
+    state: Mapping[str, Any],
+    timeout_s: float,
+    warmup: bool = False,
+) -> dict[str, Any]:
+    command = client_command(
+        entry,
+        config,
+        study=study,
+        anchor=float(selection["anchor"]),
+        output=output,
+        warmup=warmup,
+    )
+    with (entry["dir"] / "commands.log").open("a", encoding="utf-8") as log:
+        log.write(f"CLIENT role={role} {shlex.join(command)}\n")
+    handle = (output.parent / f"{output.name}.log").open("ab", buffering=0)
+    environment = os.environ.copy()
+    environment.update({"AITUNER_ROOT": str(base.AITUNER), "PYTHONUNBUFFERED": "1"})
+    process = subprocess.Popen(
+        command,
+        cwd=base.WORKDIR,
+        env=environment,
+        stdout=handle,
+        stderr=subprocess.STDOUT,
+        start_new_session=True,
+    )
+    deadline = time.monotonic() + timeout_s
+    try:
+        while process.poll() is None:
+            if time.monotonic() > deadline:
+                raise TimeoutError(f"client timeout: {config['id']} {role}")
+            if entry["server"].poll() is not None:
+                raise RuntimeError(f"server exited during {config['id']} {role}")
+            base.assert_no_other_compute()
+            if state["gpu_hours_total"] + base.live_gpu_hours([entry]) >= base.GPU_LIMIT:
+                raise RuntimeError("action-aware pilot H20-hour hard cap reached")
+            time.sleep(1.0)
+    except Exception:
+        try:
+            os.killpg(process.pid, signal.SIGTERM)
+        except ProcessLookupError:
+            pass
+        try:
+            process.wait(timeout=10.0)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(process.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+            process.wait(timeout=10.0)
+        raise
+    finally:
+        handle.close()
+    if process.returncode:
+        raise RuntimeError(
+            f"client failed: config={config['id']} role={role} rc={process.returncode}"
+        )
+    result = json.loads((output / "result.json").read_text(encoding="utf-8"))
+    validate_result(
+        result,
+        config=config,
+        selection=selection,
+        role=role,
+        warmup=warmup,
+    )
+    entry["results"].append(
+        {"anchor": float(selection["anchor"]), "dir": str(output), "kind": result["kind"]}
+    )
+    return result
+
+
+def execute_session(
+    *,
+    args: argparse.Namespace,
+    manifest: Mapping[str, Any],
+    config: Mapping[str, Any],
+    index: int,
+    state: dict[str, Any],
+    state_path: Path,
+) -> None:
+    name = str(config["id"])
+    if state["sessions"].get(name, {}).get("status") == "complete":
+        return
+    projection = remaining_projection(
+        manifest, completed_sessions=int(state["completed_sessions"])
+    )
+    if float(state["gpu_hours_total"]) + projection > base.GPU_LIMIT:
+        raise RuntimeError(f"projected cost exceeds cap before {name}")
+    load_values = {
+        float(item["selection"]["offered_req_s_per_gpu"])
+        for item in manifest["repetitions"].values()
+    }
+    load_text = (
+        f"{next(iter(load_values)):.6g}"
+        if len(load_values) == 1
+        else ",".join(f"{value:.6g}" for value in sorted(load_values))
+    )
+    echo = (
+        f"ACTION_AWARE_SESSION_ECHO host=dash0 config={name} tp=4 "
+        f"mns={config['mns']} mbbt={config['mbbt']} gpus=0-3 "
+        f"workload={manifest['source']['window_id']} load_per_gpu={load_text} "
+        f"duration_s={manifest['engine']['duration_s']} "
+        f"repetitions={','.join(map(str, config['repetition_order']))} "
+        f"source={args.manifest} output={args.run_root / 'sessions' / name} "
+        f"spent_h20h={state['gpu_hours_total']:.6f} "
+        f"remaining_projection_h20h={projection:.3f} cap_h20h={base.GPU_LIMIT:.1f}"
+    )
+    append_echo(args.run_root, echo)
+    wait_all_idle()
+    session_state = {
+        "status": "starting",
+        "mns": int(config["mns"]),
+        "mbbt": int(config["mbbt"]),
+        "repetition_order": list(config["repetition_order"]),
+        "started_at": time.time(),
+        "runs": [],
+    }
+    state["status"] = "running"
+    state["sessions"][name] = session_state
+    atomic_json(state_path, state)
+    entry = start_server(args=args, config=config, index=index)
+    failure: Exception | None = None
+    try:
+        base.wait_ready(entry)
+        first = manifest["repetitions"][str(config["repetition_order"][0])]
+        session_state["status"] = "warmup"
+        atomic_json(state_path, state)
+        run_client(
+            entry=entry,
+            config=config,
+            role="warmup",
+            study=first["study"],
+            selection=first["selection"],
+            output=entry["dir"] / "warmup",
+            state=state,
+            timeout_s=180.0,
+            warmup=True,
+        )
+        session_state["status"] = "burnin"
+        atomic_json(state_path, state)
+        burnin = manifest["burnin"]
+        burnin_result = run_client(
+            entry=entry,
+            config=config,
+            role="burnin",
+            study=burnin["study"],
+            selection=burnin,
+            output=entry["dir"] / "burnin",
+            state=state,
+            timeout_s=float(manifest["engine"]["client_timeout_s"]),
+        )
+        session_state["burnin"] = burnin_gate(
+            burnin_result,
+            expected_count=int(burnin["selected_count"]),
+            maximum_elapsed_s=float(manifest["engine"]["burnin_max_elapsed_s"]),
+        )
+        atomic_json(state_path, state)
+        session_state["status"] = "measured"
+        atomic_json(state_path, state)
+        for repetition in config["repetition_order"]:
+            item = manifest["repetitions"][str(repetition)]
+            role = f"rep{repetition}"
+            result = run_client(
+                entry=entry,
+                config=config,
+                role=role,
+                study=item["study"],
+                selection=item["selection"],
+                output=entry["dir"] / role,
+                state=state,
+                timeout_s=float(manifest["engine"]["client_timeout_s"]),
+            )
+            session_state["runs"].append(
+                {
+                    "repetition": int(repetition),
+                    "pass_rate": result["pass_rate"],
+                    "feasible": result["feasible"],
+                    "slo_pass_count": result["slo_pass_count"],
+                    "elapsed_s": result["interval"]["elapsed_s"],
+                }
+            )
+            atomic_json(state_path, state)
+        session_state["status"] = "stopping"
+        atomic_json(state_path, state)
+    except Exception as error:  # noqa: BLE001
+        failure = error
+    finally:
+        try:
+            base.stop_entry(entry)
+        except Exception as error:  # noqa: BLE001
+            failure = failure or error
+        time.sleep(2.0)
+        try:
+            wait_all_idle()
+        except Exception as error:  # noqa: BLE001
+            failure = failure or error
+
+    session_hours = base.live_gpu_hours([entry])
+    state["gpu_hours_total"] += session_hours
+    session_state["gpu_hours"] = session_hours
+    if failure is not None:
+        session_state["status"] = "failed"
+        session_state["failure"] = repr(failure)
+        state["status"] = "failed"
+        state["failures"].append({"session": name, "failure": repr(failure)})
+        atomic_json(state_path, state)
+        raise failure
+    validation = base.validate_cell(entry)
+    session_state["validation"] = validation
+    session_state["status"] = "complete"
+    session_state["completed_at"] = time.time()
+    state["completed_sessions"] += 1
+    atomic_json(state_path, state)
+
+
+def parser() -> argparse.ArgumentParser:
+    result = argparse.ArgumentParser()
+    result.add_argument("--manifest", type=Path, required=True)
+    result.add_argument("--run-root", type=Path, required=True)
+    result.add_argument("--aituner-root", type=Path, required=True)
+    result.add_argument("--vllm-source", type=Path, required=True)
+    result.add_argument("--venv", type=Path, required=True)
+    result.add_argument("--model", type=Path, required=True)
+    result.add_argument("--client", type=Path, required=True)
+    result.add_argument("--dry-run", action="store_true")
+    return result
+
+
+def main() -> None:
+    args = parser().parse_args()
+    manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
+    validate_inputs(args, manifest)
+    configure(args, manifest)
+    if args.dry_run:
+        print(json.dumps(dry_run_plan(args, manifest), indent=2, sort_keys=True))
+        return
+    args.run_root.mkdir(parents=True, exist_ok=True)
+    copied_manifest = args.run_root / "pilot-manifest.json"
+    if not copied_manifest.exists():
+        atomic_json(copied_manifest, manifest)
+    state_path = args.run_root / "controller-state.json"
+    state = load_state(state_path, base.GPU_LIMIT)
+    state["status"] = "running"
+    atomic_json(state_path, state)
+    for index, config in enumerate(manifest["configs"]):
+        execute_session(
+            args=args,
+            manifest=manifest,
+            config=config,
+            index=index,
+            state=state,
+            state_path=state_path,
+        )
+    state["status"] = "complete"
+    state["completed_at"] = time.time()
+    atomic_json(state_path, state)
+    wait_all_idle()
+    print(
+        json.dumps(
+            {
+                "status": state["status"],
+                "completed_sessions": state["completed_sessions"],
+                "gpu_hours_total": state["gpu_hours_total"],
+            },
+            sort_keys=True,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

runs/action-aware-v0/prepare_pilot.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Freeze the crossed-constraint action-aware development pilot."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+
+SCHEMA_V0 = "action-aware-constraint-pilot-manifest-v0"
+SCHEMA_V1 = "action-aware-constraint-pilot-manifest-v1"
+
+
+def configs(token_source_mbbt: int) -> tuple[dict[str, Any], ...]:
+    return (
+        {
+            "id": "b_base",
+            "mns": 64,
+            "mbbt": token_source_mbbt,
+            "repetition_order": [1, 2, 3],
+        },
+        {"id": "a_base", "mns": 16, "mbbt": 8192, "repetition_order": [2, 3, 1]},
+        {"id": "shared", "mns": 64, "mbbt": 8192, "repetition_order": [3, 1, 2]},
+        {
+            "id": "b_mns",
+            "mns": 128,
+            "mbbt": token_source_mbbt,
+            "repetition_order": [1, 3, 2],
+        },
+        {"id": "a_mbbt", "mns": 16, "mbbt": 16384, "repetition_order": [2, 1, 3]},
+    )
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def atomic_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(
+        json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    os.replace(temporary, path)
+
+
+def build(
+    base_path: Path,
+    *,
+    token_source_mbbt: int = 256,
+    prior_attempt_h20_hours: float = 0.0,
+    prior_attempt_artifact: str | None = None,
+) -> dict[str, Any]:
+    if token_source_mbbt <= 0:
+        raise ValueError("token source MBBT must be positive")
+    if prior_attempt_h20_hours < 0.0 or prior_attempt_h20_hours >= 8.0:
+        raise ValueError("prior attempt cost must be in [0, 8)")
+    base = json.loads(base_path.read_text(encoding="utf-8"))
+    if base.get("schema") != "intervention-response-phase-aware-pilot-manifest-v3":
+        raise ValueError("unexpected base manifest schema")
+    if base.get("status") != "PASS":
+        raise ValueError("base manifest did not pass its preflight")
+    if sorted(int(key) for key in base["repetitions"]) != [1, 2, 3]:
+        raise ValueError("base manifest must contain exactly three repetitions")
+
+    repetitions = {}
+    selection_hashes = []
+    for repetition in (1, 2, 3):
+        source = base["repetitions"][str(repetition)]
+        selection = dict(source["selections"]["mid"])
+        selection_hashes.append(selection["request_id_order_sha256"])
+        repetitions[str(repetition)] = {
+            "study": source["study"],
+            "study_sha256": source["study_sha256"],
+            "selection": selection,
+            "merged_trace": source["merged_trace"],
+        }
+
+    frozen_configs = configs(token_source_mbbt)
+    config_ids = [str(config["id"]) for config in frozen_configs]
+    schema = (
+        SCHEMA_V0
+        if token_source_mbbt == 256 and prior_attempt_h20_hours == 0.0
+        else SCHEMA_V1
+    )
+    payload = {
+        "schema": schema,
+        "status": "PASS",
+        "source": {
+            "base_manifest": str(base_path.resolve()),
+            "base_manifest_sha256": sha256_file(base_path),
+            "window_id": base["source"]["window_id"],
+            "source_trace": base["source"]["source_trace"],
+            "source_trace_sha256": base["source"]["source_trace_sha256"],
+        },
+        "engine": {
+            "tp": 4,
+            "duration_s": 300.0,
+            "disable_slo_early_stop": True,
+            "client_timeout_s": 450.0,
+            "burnin_max_elapsed_s": 90.0,
+        },
+        "burnin": base["burnin"],
+        "repetitions": repetitions,
+        "configs": [dict(config) for config in frozen_configs],
+        "regimes": {
+            "A": {
+                "source": "a_base",
+                "actions": {"mns": "shared", "mbbt": "a_mbbt"},
+            },
+            "B": {
+                "source": "b_base",
+                "actions": {"mns": "b_mns", "mbbt": "shared"},
+            },
+        },
+        "budget": {
+            "global_hard_cap_h20_hours": 8.0,
+            "hard_cap_h20_hours": 8.0 - prior_attempt_h20_hours,
+            "prior_attempt_h20_hours": prior_attempt_h20_hours,
+            "prior_attempt_artifact": prior_attempt_artifact,
+            "session_estimate_h20_hours": 1.35,
+            "safety_h20_hours": 0.25,
+            "expected_h20_hours": [6.0, 7.2],
+            "expected_wall_minutes": [90, 110],
+        },
+        "gates": {
+            "minimum_relative_winner_margin": 0.10,
+            "minimum_exclusive_fraction": 0.10,
+            "minimum_exclusive_ratio": 5.0,
+            "phase_fractions": [0.25, 0.50, 0.75, 1.0],
+            "material_kv_usage": 0.90,
+        },
+        "sanity": {
+            "invariants": {
+                "five_unique_configs": len(config_ids) == len(set(config_ids)) == 5,
+                "three_disjoint_repetitions": len(set(selection_hashes)) == 3,
+                "same_load_all_repetitions": len(
+                    {
+                        float(item["selection"]["offered_req_s_per_gpu"])
+                        for item in repetitions.values()
+                    }
+                )
+                == 1,
+                "all_repetition_orders_are_permutations": all(
+                    sorted(config["repetition_order"]) == [1, 2, 3]
+                    for config in frozen_configs
+                ),
+            }
+        },
+    }
+    payload["sanity"]["invariants"]["shared_endpoint_reused_by_both_regimes"] = (
+        payload["regimes"]["A"]["actions"]["mns"]
+        == payload["regimes"]["B"]["actions"]["mbbt"]
+        == "shared"
+    )
+    payload["sanity"]["red_flags"] = [
+        name
+        for name, passed in payload["sanity"]["invariants"].items()
+        if not passed
+    ]
+    if payload["sanity"]["red_flags"]:
+        payload["status"] = "FAIL"
+    return payload
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-manifest", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--token-source-mbbt", type=int, default=256)
+    parser.add_argument("--prior-attempt-h20-hours", type=float, default=0.0)
+    parser.add_argument("--prior-attempt-artifact")
+    args = parser.parse_args()
+    payload = build(
+        args.base_manifest,
+        token_source_mbbt=args.token_source_mbbt,
+        prior_attempt_h20_hours=args.prior_attempt_h20_hours,
+        prior_attempt_artifact=args.prior_attempt_artifact,
+    )
+    atomic_json(args.output, payload)
+    print(json.dumps(payload["sanity"], sort_keys=True))
+    if payload["status"] != "PASS":
+        raise SystemExit("manifest preflight failed")
+
+
+if __name__ == "__main__":
+    main()

runs/action-aware-v0/prior-attempts-v2.json

@@ -0,0 +1,24 @@
+{
+  "global_hard_cap_h20_hours": 8.0,
+  "invariants": {
+    "all_gpus_idle_after_each_stop": true,
+    "no_completed_measured_runs": true,
+    "no_prior_runtime_data_reused": true
+  },
+  "prior_attempt_h20_hours": 0.7043978426191542,
+  "schema": "action-aware-prior-attempts-v2",
+  "stops": [
+    {
+      "artifact": "/home/admin/cpfs/wjh/action-aware-constraint-v0-20260714/operational-stop-v0.json",
+      "h20_hours": 0.38598689953486126,
+      "reason": "MBBT256 burn-in remained throughput-backlogged",
+      "stage": "burnin"
+    },
+    {
+      "artifact": "/home/admin/cpfs/wjh/action-aware-constraint-v1-20260714/operational-stop-v1.json",
+      "h20_hours": 0.31841094308429296,
+      "reason": "controller passed the warmup result to the burn-in gate",
+      "stage": "first measured run in flight; zero measured results completed"
+    }
+  ]
+}

runs/action-aware-v0/test_pilot.py

@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import copy
+import importlib.util
+from pathlib import Path
+from types import SimpleNamespace
+
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[1]
+
+
+def load(name: str, filename: str):
+    spec = importlib.util.spec_from_file_location(name, HERE / filename)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def record(*, waiting: int, running: int, tokens: int) -> dict:
+    return {
+        "queues": {"waiting": waiting, "deferred": 0, "running": running},
+        "prefill_tokens": tokens,
+        "decode_tokens": 0,
+        "kv": {"usage": 0.5},
+        "preemptions": 0,
+    }
+
+
+def fake_run(
+    config: str,
+    repetition: int,
+    *,
+    goodput: float,
+    mns_score: float = 0.0,
+    mbbt_score: float = 0.0,
+    ambiguous: float = 0.0,
+) -> dict:
+    binding = {
+        "mns_exclusive_fraction": mns_score,
+        "mbbt_exclusive_fraction": mbbt_score,
+        "both_fraction": ambiguous,
+        "waiting_unresolved_fraction": 0.0,
+        "kv_usage_max": 0.5,
+        "preemptions": 0,
+    }
+    phases = {
+        phase: {
+            "mns_exclusive_fraction": mns_score,
+            "mbbt_exclusive_fraction": mbbt_score,
+        }
+        for phase in ("0.25", "0.50", "0.75", "1.00")
+    }
+    return {
+        "config_id": config,
+        "repetition": repetition,
+        "outcome": {"slo_goodput_req_s": goodput},
+        "binding": binding,
+        "phases": phases,
+    }
+
+
+def main() -> None:
+    analysis = load("action_aware_analysis", "analyze_pilot.py")
+    summary = analysis.binding_summary(
+        [
+            record(waiting=1, running=16, tokens=8),
+            record(waiting=1, running=8, tokens=32),
+            record(waiting=1, running=16, tokens=32),
+            record(waiting=1, running=8, tokens=8),
+            record(waiting=0, running=8, tokens=8),
+        ],
+        mns=16,
+        mbbt=32,
+    )
+    assert summary["mns_exclusive_count"] == 1
+    assert summary["mbbt_exclusive_count"] == 1
+    assert summary["both_count"] == 1
+    assert summary["waiting_unresolved_count"] == 1
+    assert summary["waiting_count"] == 4
+
+    # A per-step stream may have a submit gap above one second when the
+    # preceding model execution itself spans that interval.  Such a gap is
+    # covered telemetry, not a dropped-record interval.
+    asynchronous = [
+        {"submit_mono_ns": 0, "complete_mono_ns": 1_200_000_000},
+        {"submit_mono_ns": 1_100_000_000, "complete_mono_ns": 1_300_000_000},
+    ]
+    coverage, covered = analysis.telemetry_coverage(
+        asynchronous, start_ns=0, end_ns=1_100_000_000
+    )
+    assert coverage["max_internal_submit_gap_s"] == 1.1
+    assert coverage["max_uncovered_gap_s"] == 0.0
+    assert covered
+    missing = copy.deepcopy(asynchronous)
+    missing[0]["complete_mono_ns"] = 0
+    assert not analysis.telemetry_coverage(
+        missing, start_ns=0, end_ns=1_100_000_000
+    )[1]
+
+    mechanism = analysis.mechanism_summary(
+        [
+            {
+                "model_executed": True,
+                "submit_mono_ns": 0,
+                "complete_mono_ns": 2_000_000,
+                "prefill_tokens": 8,
+                "prefill_requests": 2,
+                "chunked_prefill": {
+                    "first": 1,
+                    "middle": 0,
+                    "final": 0,
+                    "unsplit": 1,
+                    "tokens": 8,
+                },
+                "prefix": {"local": {"queries": 10, "hits": 2}},
+            },
+            {
+                "model_executed": True,
+                "submit_mono_ns": 2_000_000,
+                "complete_mono_ns": 3_000_000,
+                "prefill_tokens": 0,
+                "prefill_requests": 0,
+                "chunked_prefill": {
+                    "first": 0,
+                    "middle": 0,
+                    "final": 0,
+                    "unsplit": 0,
+                    "tokens": 0,
+                },
+                "prefix": {"local": {"queries": 0, "hits": 0}},
+            },
+        ]
+    )
+    assert mechanism["prefill"]["requests_per_step"] == 2.0
+    assert mechanism["prefill"]["chunks"]["first"] == 1
+    assert mechanism["prefix"]["hit_rate"] == 0.2
+    assert all(mechanism["sanity"]["invariants"].values())
+
+    manifest = {
+        "repetitions": {str(index): {} for index in (1, 2, 3)},
+        "regimes": {
+            "A": {
+                "source": "a_base",
+                "actions": {"mns": "shared", "mbbt": "a_mbbt"},
+            },
+            "B": {
+                "source": "b_base",
+                "actions": {"mns": "b_mns", "mbbt": "shared"},
+            },
+        },
+        "gates": {
+            "minimum_relative_winner_margin": 0.10,
+            "minimum_exclusive_fraction": 0.10,
+            "minimum_exclusive_ratio": 5.0,
+            "material_kv_usage": 0.90,
+        },
+    }
+    runs = []
+    for repetition in (1, 2, 3):
+        runs.extend(
+            [
+                fake_run(
+                    "a_base",
+                    repetition,
+                    goodput=1.0,
+                    mns_score=0.8,
+                    mbbt_score=0.01,
+                ),
+                fake_run(
+                    "b_base",
+                    repetition,
+                    goodput=1.0,
+                    mns_score=0.01,
+                    mbbt_score=0.7,
+                ),
+                fake_run("shared", repetition, goodput=3.0),
+                fake_run("a_mbbt", repetition, goodput=1.5),
+                fake_run("b_mns", repetition, goodput=1.2),
+            ]
+        )
+    result = analysis.evaluate_decisions(runs, manifest)
+    assert result["decision"] == "STOP_NO_NEW_INSTRUMENTATION_NEEDED"
+    assert result["baselines"] == {
+        "always_mns_correct": 3,
+        "always_mbbt_correct": 3,
+        "binding_correct": 6,
+        "decision_count": 6,
+    }
+
+    ambiguous = copy.deepcopy(runs)
+    for run in ambiguous:
+        if run["config_id"] == "b_base":
+            run["binding"]["both_fraction"] = 0.8
+    assert (
+        analysis.evaluate_decisions(ambiguous, manifest)["decision"]
+        == "OPEN_EXACT_ATTRIBUTION_ABLATION"
+    )
+
+    wrong = copy.deepcopy(runs)
+    for run in wrong:
+        if run["config_id"] == "b_base":
+            run["binding"]["mns_exclusive_fraction"] = 0.8
+            run["binding"]["mbbt_exclusive_fraction"] = 0.01
+            for phase in run["phases"].values():
+                phase["mns_exclusive_fraction"] = 0.8
+                phase["mbbt_exclusive_fraction"] = 0.01
+    assert (
+        analysis.evaluate_decisions(wrong, manifest)["decision"]
+        == "STOP_BINDING_NOT_PREDICTIVE"
+    )
+
+    prepare = load("action_aware_prepare", "prepare_pilot.py")
+    frozen = prepare.build(
+        ROOT / "runs/intervention-response-v2/pilot-manifest-v3.json"
+    )
+    assert frozen["status"] == "PASS"
+    assert frozen["sanity"]["red_flags"] == []
+    assert [config["id"] for config in frozen["configs"]] == [
+        "b_base",
+        "a_base",
+        "shared",
+        "b_mns",
+        "a_mbbt",
+    ]
+
+    controller = load("action_aware_controller", "pilot_controller.py")
+    args = SimpleNamespace(
+        manifest=Path("/tmp/manifest.json"),
+        run_root=Path("/tmp/action-aware"),
+        aituner_root=Path("/tmp/aituner"),
+        vllm_source=Path("/tmp/vllm"),
+        venv=Path("/tmp/venv"),
+        model=Path("/tmp/model"),
+        client=Path("/tmp/client.py"),
+    )
+    controller.configure(args, frozen)
+    plan = controller.dry_run_plan(args, frozen)
+    assert plan["status"] == "PASS"
+    assert len(plan["sessions"]) == 5
+    assert plan["projected_h20_hours"] == 7.0
+    assert "--max-num-batched-tokens 256" in plan["sessions"][0]["commands"]["server"]
+    revised = prepare.build(
+        ROOT / "runs/intervention-response-v2/pilot-manifest-v3.json",
+        token_source_mbbt=2048,
+        prior_attempt_h20_hours=0.38598689953486126,
+        prior_attempt_artifact="/tmp/operational-stop-v0.json",
+    )
+    assert revised["schema"] == "action-aware-constraint-pilot-manifest-v1"
+    assert revised["configs"][0]["mbbt"] == 2048
+    assert revised["configs"][3]["mbbt"] == 2048
+    assert revised["budget"]["hard_cap_h20_hours"] < 8.0
+    controller.configure(args, revised)
+    revised_plan = controller.dry_run_plan(args, revised)
+    assert revised_plan["projected_h20_hours"] < revised_plan["hard_cap_h20_hours"]
+    assert (
+        "--max-num-batched-tokens 2048"
+        in revised_plan["sessions"][0]["commands"]["server"]
+    )
+    accepted_burnin = {
+        "kind": "anchor",
+        "selection": {"count": 510},
+        "interval": {"elapsed_s": 61.25},
+        "pass_rate": 0.5,
+        "feasible": False,
+    }
+    assert controller.burnin_gate(
+        accepted_burnin, expected_count=510, maximum_elapsed_s=90.0
+    )["elapsed_s"] == 61.25
+    warmup = copy.deepcopy(accepted_burnin)
+    warmup["kind"] = "warmup"
+    try:
+        controller.burnin_gate(warmup, expected_count=510, maximum_elapsed_s=90.0)
+    except RuntimeError as error:
+        assert "non-anchor" in str(error)
+    else:
+        raise AssertionError("warmup incorrectly passed the burnin gate")
+    slow = copy.deepcopy(accepted_burnin)
+    slow["interval"]["elapsed_s"] = 91.0
+    try:
+        controller.burnin_gate(slow, expected_count=510, maximum_elapsed_s=90.0)
+    except RuntimeError as error:
+        assert "throughput gate failed" in str(error)
+    else:
+        raise AssertionError("slow burnin incorrectly passed the throughput gate")
+    print("action-aware constraint pilot: PASS")
+
+
+if __name__ == "__main__":
+    main()

runs/active-intervention-v0/analyze_prospective.py

@@ -0,0 +1,325 @@
+#!/usr/bin/env python3
+"""Audit held-out action/measurement choices against the exact 2x2 surface."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import statistics
+from pathlib import Path
+from typing import Any, Mapping
+
+
+SCHEMA = "active-intervention-prospective-audit-v0"
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def atomic_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(
+        json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    os.replace(temporary, path)
+
+
+def numeric(values: list[float]) -> dict[str, Any]:
+    finite = [float(value) for value in values]
+    if not finite or any(not math.isfinite(value) for value in finite):
+        raise ValueError("numeric summary requires finite values")
+    return {
+        "n": len(finite),
+        "min": min(finite),
+        "max": max(finite),
+        "distinct_n": len(set(finite)),
+    }
+
+
+def load_surface(
+    manifest: Mapping[str, Any], run_root: Path
+) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    rows = []
+    aggregate = {}
+    duration_s = float(manifest["engine"]["duration_s"])
+    tp = int(manifest["engine"]["tp"])
+    for config in manifest["configs"]:
+        config_id = str(config["id"])
+        values = []
+        for repetition in sorted(int(key) for key in manifest["repetitions"]):
+            expected = manifest["repetitions"][str(repetition)]["selection"]
+            result_path = (
+                run_root / "sessions" / config_id / f"rep{repetition}" / "result.json"
+            )
+            result = json.loads(result_path.read_text(encoding="utf-8"))
+            if result["selection"]["request_id_order_sha256"] != expected[
+                "request_id_order_sha256"
+            ]:
+                raise ValueError(f"request hash mismatch: {config_id} rep{repetition}")
+            offered_total = float(expected["offered_req_s_per_gpu"]) * tp
+            normalized = float(result["slo_pass_count"]) / duration_s / offered_total
+            values.append(normalized)
+            rows.append(
+                {
+                    "config_id": config_id,
+                    "mns": int(config["mns"]),
+                    "mbbt": int(config["mbbt"]),
+                    "repetition": repetition,
+                    "normalized_slo_goodput": normalized,
+                    "slo_goodput_req_s": float(result["slo_pass_count"]) / duration_s,
+                    "pass_rate": float(result["pass_rate"]),
+                    "elapsed_s": float(result["interval"]["elapsed_s"]),
+                    "result": str(result_path),
+                    "result_sha256": sha256_file(result_path),
+                }
+            )
+        aggregate[config_id] = {
+            "normalized_slo_goodput_values": values,
+            "median_normalized_slo_goodput": float(statistics.median(values)),
+            "sanity": numeric(values),
+        }
+    return aggregate, rows
+
+
+def source_cost_estimate(
+    *,
+    source_session: Mapping[str, Any],
+    source_rows: list[Mapping[str, Any]],
+    cutoff_s: float,
+    tp: int,
+) -> dict[str, float]:
+    actual_h20_hours = float(source_session["gpu_hours"])
+    measured_replay_h20_hours = (
+        tp * sum(float(row["elapsed_s"]) for row in source_rows) / 3600.0
+    )
+    fixed_h20_hours = max(0.0, actual_h20_hours - measured_replay_h20_hours)
+    prefix_replay_h20_hours = tp * len(source_rows) * cutoff_s / 3600.0
+    return {
+        "actual_full_session_h20_hours": actual_h20_hours,
+        "fixed_startup_warmup_burnin_cleanup_h20_hours": fixed_h20_hours,
+        "prefix_replay_h20_hours_lower_bound": prefix_replay_h20_hours,
+        "counterfactual_all_in_h20_hours_lower_bound": fixed_h20_hours
+        + prefix_replay_h20_hours,
+    }
+
+
+def replay_policy(
+    *,
+    mode: str,
+    manifest: Mapping[str, Any],
+    decision: Mapping[str, Any],
+    surface: Mapping[str, Any],
+    session_costs: Mapping[str, float],
+    source_cost: Mapping[str, float],
+) -> dict[str, Any]:
+    acceptable_regret = float(manifest["gates"]["acceptable_regret"])
+    source_id = str(manifest["source_config_id"])
+    oracle = max(
+        float(item["median_normalized_slo_goodput"]) for item in surface.values()
+    )
+    cumulative = float(source_cost["counterfactual_all_in_h20_hours_lower_bound"])
+    source_score = float(surface[source_id]["median_normalized_slo_goodput"])
+    source_regret = 1.0 - source_score / oracle if oracle > 0 else 0.0
+    points = [
+        {
+            "action_id": "noop",
+            "config_id": source_id,
+            "score": source_score,
+            "regret": source_regret,
+            "cumulative_h20_hours_lower_bound": cumulative,
+        }
+    ]
+    hit = points[0] if source_regret <= acceptable_regret + 1e-12 else None
+    seen = {source_id}
+    for action_id in decision["decisions"][mode]["intervention_order"]:
+        config_id = str(manifest["actions"][action_id])
+        if config_id in seen:
+            continue
+        seen.add(config_id)
+        cumulative += float(session_costs[config_id])
+        score = float(surface[config_id]["median_normalized_slo_goodput"])
+        regret = 1.0 - score / oracle if oracle > 0 else 0.0
+        point = {
+            "action_id": action_id,
+            "config_id": config_id,
+            "score": score,
+            "regret": regret,
+            "cumulative_h20_hours_lower_bound": cumulative,
+        }
+        points.append(point)
+        if hit is None and regret <= acceptable_regret + 1e-12:
+            hit = point
+    return {
+        "mode": mode,
+        "measurement_cutoff_s": float(
+            decision["decisions"][mode]["selected_cutoff_s"]
+        ),
+        "selected_action": decision["decisions"][mode]["selected_action"],
+        "decision_kind": decision["decisions"][mode]["decision_kind"],
+        "intervention_order": decision["decisions"][mode]["intervention_order"],
+        "source_cost": dict(source_cost),
+        "oracle_normalized_slo_goodput": oracle,
+        "cost_to_acceptable": hit,
+        "reached_acceptable": hit is not None,
+        "points": points,
+    }
+
+
+def build_audit(
+    *, manifest_path: Path, decision_path: Path, run_root: Path
+) -> dict[str, Any]:
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    decision = json.loads(decision_path.read_text(encoding="utf-8"))
+    state_path = run_root / "controller-state.json"
+    state = json.loads(state_path.read_text(encoding="utf-8"))
+    if manifest.get("schema") != "active-intervention-prospective-manifest-v0":
+        raise ValueError("unexpected prospective manifest schema")
+    if decision.get("schema") != "active-intervention-prospective-decision-v0":
+        raise ValueError("unexpected prospective decision schema")
+    if decision["manifest_sha256"] != sha256_file(manifest_path):
+        raise ValueError("decision does not match prospective manifest")
+    surface, rows = load_surface(manifest, run_root)
+    source_id = str(manifest["source_config_id"])
+    sessions = state["sessions"]
+    session_costs = {
+        config_id: float(sessions[config_id]["gpu_hours"])
+        for config_id in surface
+    }
+    source_rows = [row for row in rows if row["config_id"] == source_id]
+    policies = {}
+    for mode in ("outcome_only", "telemetry"):
+        cost = source_cost_estimate(
+            source_session=sessions[source_id],
+            source_rows=source_rows,
+            cutoff_s=float(decision["decisions"][mode]["selected_cutoff_s"]),
+            tp=int(manifest["engine"]["tp"]),
+        )
+        policies[mode] = replay_policy(
+            mode=mode,
+            manifest=manifest,
+            decision=decision,
+            surface=surface,
+            session_costs=session_costs,
+            source_cost=cost,
+        )
+    outcome_hit = policies["outcome_only"]["cost_to_acceptable"]
+    telemetry_hit = policies["telemetry"]["cost_to_acceptable"]
+    if outcome_hit is None or telemetry_hit is None:
+        reduction = None
+    else:
+        outcome_cost = float(outcome_hit["cumulative_h20_hours_lower_bound"])
+        telemetry_cost = float(telemetry_hit["cumulative_h20_hours_lower_bound"])
+        reduction = 1.0 - telemetry_cost / outcome_cost if outcome_cost > 0 else 0.0
+    confirmation_trigger = bool(
+        reduction is not None
+        and reduction
+        >= float(manifest["gates"]["confirmation_trigger_gpu_cost_reduction"])
+        and policies["telemetry"]["reached_acceptable"]
+    )
+    contribution_gate = bool(
+        reduction is not None
+        and reduction >= float(manifest["gates"]["contribution_gpu_cost_reduction"])
+        and policies["telemetry"]["reached_acceptable"]
+    )
+    status = (
+        "TRIGGER_ACTUAL_EARLY_STOP_CONFIRMATION"
+        if confirmation_trigger
+        else "STOP_NO_PROSPECTIVE_GPU_COST_SIGNAL"
+    )
+    normalized_values = [float(row["normalized_slo_goodput"]) for row in rows]
+    costs = list(session_costs.values())
+    invariants = {
+        "controller_complete": state.get("status") == "complete",
+        "four_sessions_complete": len(sessions) == 4
+        and all(item.get("status") == "complete" for item in sessions.values()),
+        "twelve_surface_outcomes": len(rows) == 12,
+        "nonnegative_goodput": all(value >= 0.0 for value in normalized_values),
+        "normalized_goodput_bounded": all(value <= 1.0 + 1e-12 for value in normalized_values),
+        "surface_not_all_identical": len(set(normalized_values)) > 1,
+        "nonnegative_session_costs": all(value >= 0.0 for value in costs),
+        "policy_replay_reaches_oracle_surface": all(
+            policy["reached_acceptable"] for policy in policies.values()
+        ),
+    }
+    red_flags = [name for name, passed in invariants.items() if not passed]
+    if red_flags:
+        status = "STOP_SANITY"
+    return {
+        "schema": SCHEMA,
+        "status": status,
+        "claim_boundary": (
+            "Prospective exact-surface replay. Prefix source costs reconstruct the "
+            "measured fixed overhead plus selected replay seconds; actual early-stop "
+            "confirmation is required before claiming GPU-cost reduction."
+        ),
+        "manifest": str(manifest_path),
+        "manifest_sha256": sha256_file(manifest_path),
+        "decision": str(decision_path),
+        "decision_sha256": sha256_file(decision_path),
+        "controller_state": str(state_path),
+        "controller_state_sha256": sha256_file(state_path),
+        "surface": surface,
+        "rows": rows,
+        "session_costs_h20_hours": session_costs,
+        "annotation_campaign_h20_hours": float(state["gpu_hours_total"]),
+        "policies": policies,
+        "comparison": {
+            "telemetry_gpu_cost_reduction_fraction": reduction,
+            "confirmation_trigger": confirmation_trigger,
+            "contribution_gate": contribution_gate,
+            "confirmation_trigger_threshold": manifest["gates"][
+                "confirmation_trigger_gpu_cost_reduction"
+            ],
+            "contribution_threshold": manifest["gates"][
+                "contribution_gpu_cost_reduction"
+            ],
+            "action_changed": policies["outcome_only"]["selected_action"]
+            != policies["telemetry"]["selected_action"],
+            "measurement_changed": policies["outcome_only"]["measurement_cutoff_s"]
+            != policies["telemetry"]["measurement_cutoff_s"],
+        },
+        "sanity": {
+            "invariants": invariants,
+            "red_flags": red_flags,
+            "normalized_slo_goodput": numeric(normalized_values),
+            "session_h20_hours": numeric(costs),
+        },
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--decision", type=Path, required=True)
+    parser.add_argument("--run-root", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    audit = build_audit(
+        manifest_path=args.manifest,
+        decision_path=args.decision,
+        run_root=args.run_root,
+    )
+    atomic_json(args.output, audit)
+    print(
+        json.dumps(
+            {
+                "status": audit["status"],
+                "comparison": audit["comparison"],
+                "sanity": audit["sanity"],
+            },
+            sort_keys=True,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

runs/active-intervention-v0/extract_training.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python3
+"""Extract paired source/action examples from the accepted action-aware run."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import sys
+from pathlib import Path
+from statistics import fmean
+from typing import Any, Mapping
+
+
+PHASES = ("0.25", "0.50", "0.75", "1.00")
+HERE = Path(__file__).resolve().parent
+COMMON_STATE = HERE.parent / "telemetry-residual"
+sys.path.insert(0, str(COMMON_STATE))
+
+from common_state import summarize_engine  # noqa: E402
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def atomic_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(
+        json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    os.replace(temporary, path)
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    records = []
+    with path.open(encoding="utf-8") as source:
+        for line_number, line in enumerate(source, 1):
+            if not line.strip():
+                continue
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError as error:
+                raise ValueError(f"{path}:{line_number}: invalid JSON") from error
+    if not records:
+        raise ValueError(f"{path}: no request records")
+    return records
+
+
+def prefix_outcome(
+    requests: list[Mapping[str, Any]], *, cutoff_s: float, offered_total: float
+) -> dict[str, float]:
+    admitted = [request for request in requests if float(request["arrival_s"]) <= cutoff_s]
+    completed = [
+        request
+        for request in requests
+        if request.get("completed_elapsed_s") is not None
+        and float(request["completed_elapsed_s"]) <= cutoff_s
+    ]
+    if not admitted:
+        raise ValueError("prefix has no admitted requests")
+    admitted_ids = {str(request["request_id"]) for request in admitted}
+    if any(str(request["request_id"]) not in admitted_ids for request in completed):
+        raise ValueError("prefix completion precedes admission")
+    passed = sum(bool(request["slo_pass"]) for request in completed)
+    ttft = [float(request["ttft_ms"]) for request in completed]
+    tpot = [float(request["tpot_ms"]) for request in completed]
+    total = len(requests)
+    return {
+        "normalized_slo_goodput": passed / cutoff_s / offered_total,
+        "admitted_fraction": len(admitted) / total,
+        "completed_over_admitted": len(completed) / len(admitted),
+        "completed_pass_rate": passed / max(1, len(completed)),
+        "completed_fail_fraction_of_total": (len(completed) - passed) / total,
+        "outstanding_over_admitted": (len(admitted) - len(completed)) / len(admitted),
+        "ttft_max_over_slo_max": max(ttft, default=0.0) / 6000.0,
+        "ttft_mean_over_slo_max": fmean(ttft) / 6000.0 if ttft else 0.0,
+        "tpot_max_over_slo": max(tpot, default=0.0) / 50.0,
+        "tpot_mean_over_slo": fmean(tpot) / 50.0 if tpot else 0.0,
+        "admitted_input_tokens_mean_over_limit": fmean(
+            float(request["raw_input_tokens"]) for request in admitted
+        )
+        / 8192.0,
+    }
+
+
+def telemetry_record(state: Mapping[str, Any]) -> dict[str, float]:
+    common = state["common"]
+    engine = state["engine_only"]
+    executed_steps = int(state["sanity"]["executed_steps"])
+    if executed_steps <= 0:
+        raise ValueError("telemetry phase contains no executed engine steps")
+    return {
+        "scheduler_steps_per_s": float(common["scheduler_steps_per_s"]),
+        "batch_size_mean": float(common["batch_size"]["mean"]),
+        "batch_size_cv": float(common["batch_size"]["cv"]),
+        "batch_tokens_mean": float(common["batch_tokens"]["mean"]),
+        "batch_tokens_cv": float(common["batch_tokens"]["cv"]),
+        "decode_batch_size_mean": float(common["decode_batch_size"]["mean"]),
+        "decode_batch_size_cv": float(common["decode_batch_size"]["cv"]),
+        "prefill_token_fraction": float(common["prefill_token_fraction"]),
+        "queue_waiting_mean": float(common["queue_waiting_mean"]),
+        "queue_running_mean": float(common["queue_running_mean"]),
+        "preemptions_per_step": float(common["preemptions"]) / executed_steps,
+        "kv_usage_mean": float(engine["kv_usage_mean"]),
+        "kv_usage_max": float(engine["kv_usage_max"]),
+        "kv_usage_end_minus_start": float(engine["kv_usage_end_minus_start"]),
+        "graph_none_share": float(engine["graph_none_share"]),
+        "graph_full_share": float(engine["graph_full_share"]),
+        "graph_padding_fraction": float(engine["graph_padding_fraction"]),
+    }
+
+
+def load_stream(path: Path, *, expected_sha256: str) -> list[dict[str, Any]]:
+    if sha256_file(path) != expected_sha256:
+        raise ValueError(f"engine stream hash mismatch: {path}")
+    decoded = load_jsonl(path)
+    records = [row for row in decoded if "step_index" in row]
+    if not records:
+        raise ValueError(f"engine stream has no Layer-1 records: {path}")
+    return records
+
+
+def build_dataset(
+    *, audit_path: Path, manifest_path: Path, run_root: Path
+) -> dict[str, Any]:
+    audit = json.loads(audit_path.read_text(encoding="utf-8"))
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    if audit.get("schema") != "action-aware-constraint-pilot-audit-v0":
+        raise ValueError("unexpected action-aware audit schema")
+    if audit["sanity"]["red_flags"]:
+        raise ValueError(f"action-aware audit red flags: {audit['sanity']['red_flags']}")
+    configs = {str(item["id"]): item for item in manifest["configs"]}
+    runs = {
+        (str(run["config_id"]), int(run["repetition"])): run
+        for run in audit["runs"]
+    }
+    source_ids = {str(regime["source"]) for regime in manifest["regimes"].values()}
+    stream_entries = {
+        str(item["config_id"]): item
+        for item in audit["streams"]
+        if str(item["config_id"]) in source_ids
+    }
+    if set(stream_entries) != source_ids:
+        raise ValueError("audit is missing a source config engine stream")
+    streams = {
+        config_id: load_stream(
+            Path(item["stream"]), expected_sha256=str(item["stream_sha256"])
+        )
+        for config_id, item in stream_entries.items()
+    }
+    examples = []
+    request_hashes = []
+    for regime_name, regime in sorted(manifest["regimes"].items()):
+        source_id = str(regime["source"])
+        for repetition in sorted(int(value) for value in manifest["repetitions"]):
+            source_run = runs[(source_id, repetition)]
+            source_config = configs[source_id]
+            request_path = run_root / "sessions" / source_id / f"rep{repetition}" / "requests.jsonl"
+            requests = load_jsonl(request_path)
+            request_hashes.append(sha256_file(request_path))
+            offered_rate_per_gpu = float(
+                manifest["repetitions"][str(repetition)]["selection"][
+                    "offered_req_s_per_gpu"
+                ]
+            )
+            offered_total = offered_rate_per_gpu * int(manifest["engine"]["tp"])
+            source_goodput = float(source_run["outcome"]["slo_goodput_req_s"])
+            source_normalized = min(1.0, source_goodput / offered_total)
+            decision_id = f"{regime_name}-rep{repetition}"
+            for phase in PHASES:
+                cutoff_s = float(manifest["engine"]["duration_s"]) * float(phase)
+                outcome = prefix_outcome(
+                    requests, cutoff_s=cutoff_s, offered_total=offered_total
+                )
+                admitted_count = sum(
+                    float(request["arrival_s"]) <= cutoff_s for request in requests
+                )
+                start_ns = int(source_run["state"]["interval"]["start_ns"])
+                phase_state = summarize_engine(
+                    streams[source_id],
+                    start_ns=start_ns,
+                    end_ns=start_ns + round(cutoff_s * 1e9),
+                    request_count=admitted_count,
+                )
+                if not all(phase_state["sanity"]["invariants"].values()):
+                    raise ValueError(
+                        f"engine state invariant failed: {decision_id} phase {phase}"
+                    )
+                telemetry = telemetry_record(phase_state)
+                actions = {"noop": source_id, **regime["actions"]}
+                for action_name, target_id in sorted(actions.items()):
+                    target_run = runs[(str(target_id), repetition)]
+                    target_config = configs[str(target_id)]
+                    target_goodput = float(target_run["outcome"]["slo_goodput_req_s"])
+                    normalized = target_goodput / offered_total
+                    if not 0.0 <= normalized <= 1.0 + 1e-12:
+                        raise ValueError("target normalized goodput is outside [0, 1]")
+                    examples.append(
+                        {
+                            "phase": phase,
+                            "cutoff_s": cutoff_s,
+                            "decision_id": decision_id,
+                            "regime": regime_name,
+                            "repetition": repetition,
+                            "source": {
+                                "config_id": source_id,
+                                "mns": int(source_config["mns"]),
+                                "mbbt": int(source_config["mbbt"]),
+                                "offered_rate_per_gpu": offered_rate_per_gpu,
+                                "outcome": outcome,
+                                "telemetry": telemetry,
+                            },
+                            "action": {
+                                "id": action_name,
+                                "target_config_id": str(target_id),
+                                "target_mns": int(target_config["mns"]),
+                                "target_mbbt": int(target_config["mbbt"]),
+                            },
+                            "target_slo_goodput_req_s": target_goodput,
+                            "target_normalized_goodput": min(1.0, normalized),
+                            "source_normalized_goodput": source_normalized,
+                            "target_delta_normalized_goodput": min(1.0, normalized)
+                            - source_normalized,
+                        }
+                    )
+    invariants = {
+        "expected_examples": len(examples) == len(PHASES) * 2 * 3 * 3,
+        "four_phases": sorted({example["phase"] for example in examples})
+        == sorted(PHASES),
+        "six_decisions": len({example["decision_id"] for example in examples}) == 6,
+        "three_actions_per_decision_phase": all(
+            sum(
+                item["decision_id"] == decision
+                and item["phase"] == phase
+                for item in examples
+            )
+            == 3
+            for decision in {item["decision_id"] for item in examples}
+            for phase in PHASES
+        ),
+        "targets_not_all_identical": len(
+            {example["target_normalized_goodput"] for example in examples}
+        )
+        > 1,
+        "bounded_prefix_ratios": all(
+            0.0 <= float(value) <= 1.0
+            for example in examples
+            for key, value in example["source"]["outcome"].items()
+            if key
+            in {
+                "admitted_fraction",
+                "completed_over_admitted",
+                "completed_pass_rate",
+                "completed_fail_fraction_of_total",
+                "outstanding_over_admitted",
+            }
+        ),
+        "direct_telemetry_without_binding_labels": all(
+            not any(token in key for token in ("exclusive", "unresolved", "both"))
+            for example in examples
+            for key in example["source"]["telemetry"]
+        ),
+        "treatment_effects_bounded": all(
+            -1.0 <= float(example["target_delta_normalized_goodput"]) <= 1.0
+            for example in examples
+        ),
+    }
+    red_flags = [name for name, passed in invariants.items() if not passed]
+    if red_flags:
+        raise RuntimeError(f"training dataset sanity failed: {red_flags}")
+    return {
+        "schema": "active-intervention-training-v0",
+        "status": "VALID",
+        "provenance": {
+            "audit": str(audit_path),
+            "audit_sha256": sha256_file(audit_path),
+            "manifest": str(manifest_path),
+            "manifest_sha256": sha256_file(manifest_path),
+            "run_root": str(run_root),
+            "source_request_sha256": sorted(set(request_hashes)),
+            "source_stream_sha256": sorted(
+                str(item["stream_sha256"]) for item in stream_entries.values()
+            ),
+        },
+        "examples": examples,
+        "sanity": {"invariants": invariants, "red_flags": red_flags},
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--audit", type=Path, required=True)
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--run-root", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    dataset = build_dataset(
+        audit_path=args.audit,
+        manifest_path=args.manifest,
+        run_root=args.run_root,
+    )
+    atomic_json(args.output, dataset)
+    print(
+        json.dumps(
+            {
+                "status": dataset["status"],
+                "examples": len(dataset["examples"]),
+                "sanity": dataset["sanity"],
+            },
+            sort_keys=True,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()