Document Stop-B end-to-end on dense 27B: the improving climb + no-regression

Real gpt-5.4 agentic loop raised per-GPU TP1 0.123 -> TP2 0.2925 -> TP4 1.0012 (8.1x), each a correctly-diagnosed real gain; then a TP4 runtime tweak measured 0.942 < 1.00 and was correctly rejected (no regression). With the 30B run (validator stop + LLM-stop veto), all Stop-B behaviors are now validated end-to-end. The SIGTERM-teardown fix was validated in practice (clean engine teardown, no GPU leak on stop). Efficiency finding: at scale=1.0, infeasible high-theta probes burn the 900s elapsed cap, so a practical loop needs a lower cap; this is why the run was stopped after iter-4 rather than driven to an explicit Stop-B firing. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Add 27B Stop-B agentic-loop config (harness-driven, GPUs 2-7)
2026-06-16 18:07:00 +08:00 · 2026-06-16 09:08:46 +08:00 · 2026-06-16 09:08:06 +08:00 · 2026-06-16 01:54:40 +08:00 · 2026-06-15 23:01:22 +08:00 · 2026-06-15 22:58:28 +08:00
20 changed files with 1527 additions and 12 deletions
--- a/configs/examples/dash0_qwen27b_stopB_loop.json
+++ b/configs/examples/dash0_qwen27b_stopB_loop.json
@@ -0,0 +1,177 @@
+{
+  "study_id": "dash0-qwen27b-stopB-loop-chat-0-8k",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "qwen3.5-27b-256k-0223-internal",
+    "served_model_name": "qwen35-27b-aituner"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "latest-release-on-dash0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18082,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 180,
+    "launch_args": [
+      "serve",
+      "/home/admin/resource/model/464482ce/qwen3.5-27b/256k-0223-internal"
+    ],
+    "base_envs": {
+      "VLLM_DISABLE_COMPILE_CACHE": "1",
+      "DS_LLM_IGNORE_WARMUP": "1",
+      "DS_LLM_IGNORE_CHECK_WARMUP": "1",
+      "VLLM_ENABLE_MODEL_RUNNER_WARMUP": "1",
+      "VLLM_GDN_USE_FUSED_QKVZBA_KERNEL": "0",
+      "PARAM_TOTAL_MAX": "262144",
+      "PARAM_IN_LENGTH_MAX": "262144",
+      "PARAM_MAX_LENGTH_MAX": "131072",
+      "DS_LLM_MAX_THINK_TOKENS": "81920",
+      "DS_LLM_GRACEFUL_SHUTDOWN_WAIT_SECONDS": "600",
+      "VLLM_FP8_USE_BLADNN": "1",
+      "VLLM_MOE_USE_BLADNN": "1",
+      "VLLM_GDN_USE_BLADNN": "0",
+      "VLLM_USE_V1": "1",
+      "VLLM_IS_HYBRID_MODEL": "1",
+      "VLLM_ENABLE_TORCH_COMPILE": "1",
+      "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
+      "VLLM_QUANTIZE_ROUTED_EXPERTS_ONLY": "1",
+      "VLLM_USE_FLASHINFER_SAMPLER": "0",
+      "VLLM_DP_MASTER_PORT": "9528",
+      "VLLM_RESPONSE_TIMEOUT": "300",
+      "VLLM_LOG_REQ_KV_LENS": "1",
+      "DS_LLM_GRACEFUL_SHUTDOWN_KEEP_SECONDS": "600",
+      "CUDA_VISIBLE_DEVICES": "2,3,4,5,6,7"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18082,
+      "served-model-name": "qwen35-27b-aituner",
+      "trust-remote-code": true,
+      "dtype": "bfloat16",
+      "gpu-memory-utilization": 0.9,
+      "enable-prefix-caching": true,
+      "mamba-cache-mode": "light",
+      "distributed-executor-backend": "mp",
+      "block-size": 64,
+      "enable-chunked-prefill": true,
+      "max-num-batched-tokens": 8192,
+      "disable-cascade-attn": true,
+      "max-model-len": 262144,
+      "speculative-config": "{\"method\":\"qwen3_next_vl_mtp\",\"num_speculative_tokens\":3}",
+      "mm-processor-cache-gb": 0,
+      "limit-mm-per-prompt": "{\"image\":256,\"video\":64}",
+      "compilation-config": "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"use_inductor\":false,\"pass_config\":{\"fuse_norm_quant\":false,\"fuse_act_quant\":false,\"fuse_attn_quant\":false}}",
+      "mamba-cache-dtype": "float32",
+      "skip-mm-profiling": true,
+      "quantization": "fp8",
+      "tensor-parallel-size": 1,
+      "disable-log-requests": true
+    },
+    "tunable_envs": [
+      "VLLM_ENABLE_TORCH_COMPILE"
+    ],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "block-size",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1
+      ]
+    },
+    "python_executable": "python3"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 32,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 1.0,
+    "early_stop_max_lag_s": 120.0,
+    "early_stop_max_elapsed_s": 900.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "linear_ms",
+      "intercept_ms": 4000,
+      "per_token_ms": 0.125
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.25,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325,
+    "inherit_incumbent_floor": true
+  },
+  "llm": {
+    "system_prompt": "Propose a single engine config patch that increases the maximum feasible sampling_u under the SLO target. Favor launch-safe changes grounded in the incumbent result and only propose knobs that plausibly improve throughput above the incumbent request rate.",
+    "max_history_trials": 8,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.4",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 180
+    }
+  }
+}
--- a/configs/examples/dash0_qwen27b_tp_ab.json
+++ b/configs/examples/dash0_qwen27b_tp_ab.json
@@ -0,0 +1,176 @@
+{
+  "study_id": "dash0-qwen27b-tp-ab-chat-0-8k",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "qwen3.5-27b-256k-0223-internal",
+    "served_model_name": "qwen35-27b-aituner"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "latest-release-on-dash0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18082,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 180,
+    "launch_args": [
+      "serve",
+      "/home/admin/resource/model/464482ce/qwen3.5-27b/256k-0223-internal"
+    ],
+    "base_envs": {
+      "VLLM_DISABLE_COMPILE_CACHE": "1",
+      "DS_LLM_IGNORE_WARMUP": "1",
+      "DS_LLM_IGNORE_CHECK_WARMUP": "1",
+      "VLLM_ENABLE_MODEL_RUNNER_WARMUP": "1",
+      "VLLM_GDN_USE_FUSED_QKVZBA_KERNEL": "0",
+      "PARAM_TOTAL_MAX": "262144",
+      "PARAM_IN_LENGTH_MAX": "262144",
+      "PARAM_MAX_LENGTH_MAX": "131072",
+      "DS_LLM_MAX_THINK_TOKENS": "81920",
+      "DS_LLM_GRACEFUL_SHUTDOWN_WAIT_SECONDS": "600",
+      "VLLM_FP8_USE_BLADNN": "1",
+      "VLLM_MOE_USE_BLADNN": "1",
+      "VLLM_GDN_USE_BLADNN": "0",
+      "VLLM_USE_V1": "1",
+      "VLLM_IS_HYBRID_MODEL": "1",
+      "VLLM_ENABLE_TORCH_COMPILE": "1",
+      "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
+      "VLLM_QUANTIZE_ROUTED_EXPERTS_ONLY": "1",
+      "VLLM_USE_FLASHINFER_SAMPLER": "0",
+      "VLLM_DP_MASTER_PORT": "9528",
+      "VLLM_RESPONSE_TIMEOUT": "300",
+      "VLLM_LOG_REQ_KV_LENS": "1",
+      "DS_LLM_GRACEFUL_SHUTDOWN_KEEP_SECONDS": "600",
+      "CUDA_VISIBLE_DEVICES": "2,3,4,5,6,7"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18082,
+      "served-model-name": "qwen35-27b-aituner",
+      "trust-remote-code": true,
+      "dtype": "bfloat16",
+      "gpu-memory-utilization": 0.9,
+      "enable-prefix-caching": true,
+      "mamba-cache-mode": "light",
+      "distributed-executor-backend": "mp",
+      "block-size": 64,
+      "enable-chunked-prefill": true,
+      "max-num-batched-tokens": 8192,
+      "disable-cascade-attn": true,
+      "max-model-len": 262144,
+      "speculative-config": "{\"method\":\"qwen3_next_vl_mtp\",\"num_speculative_tokens\":3}",
+      "mm-processor-cache-gb": 0,
+      "limit-mm-per-prompt": "{\"image\":256,\"video\":64}",
+      "compilation-config": "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"use_inductor\":false,\"pass_config\":{\"fuse_norm_quant\":false,\"fuse_act_quant\":false,\"fuse_attn_quant\":false}}",
+      "mamba-cache-dtype": "float32",
+      "skip-mm-profiling": true,
+      "quantization": "fp8",
+      "tensor-parallel-size": 1,
+      "disable-log-requests": true
+    },
+    "tunable_envs": [
+      "VLLM_ENABLE_TORCH_COMPILE"
+    ],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "block-size",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1
+      ]
+    },
+    "python_executable": "python3"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 32,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 1.0,
+    "early_stop_max_lag_s": 120.0,
+    "early_stop_max_elapsed_s": 900.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "linear_ms",
+      "intercept_ms": 4000,
+      "per_token_ms": 0.125
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.125,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325
+  },
+  "llm": {
+    "system_prompt": "Propose a single engine config patch that increases the maximum feasible sampling_u under the SLO target. Favor launch-safe changes grounded in the incumbent result and only propose knobs that plausibly improve throughput above the incumbent request rate.",
+    "max_history_trials": 8,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.4",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 180
+    }
+  }
+}
--- a/configs/examples/dash0_qwen30b_a3b_stopA_fulldata.json
+++ b/configs/examples/dash0_qwen30b_a3b_stopA_fulldata.json
@@ -0,0 +1,138 @@
+{
+  "study_id": "dash0-qwen30b-a3b-stopA-fulldata-chat-0-8k",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "Qwen/Qwen3-30B-A3B",
+    "served_model_name": "qwen3-30b-a3b-community"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "0.20.0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18230,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 900,
+    "launch_args": [
+      "serve",
+      "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
+    ],
+    "base_envs": {
+      "CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7",
+      "HOME": "/tmp/wjh",
+      "XDG_CACHE_HOME": "/tmp/wjh/.cache"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18230,
+      "served-model-name": "qwen3-30b-a3b-community",
+      "gpu-memory-utilization": 0.9,
+      "max-model-len": 16384,
+      "trust-remote-code": true,
+      "enable-prefix-caching": true
+    },
+    "tunable_envs": [],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "enable-expert-parallel",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "block-size",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ]
+    },
+    "python_executable": "/tmp/wjh/venvs/vllm-0.20.0-cu129/bin/python"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "completion_tokens_override": 128,
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 64,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 1.0,
+    "early_stop_max_lag_s": 120.0,
+    "early_stop_max_elapsed_s": 900.0
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "step_ms",
+      "buckets": [
+        {
+          "max_input_tokens": 4096,
+          "threshold_ms": 2000
+        },
+        {
+          "max_input_tokens": 32768,
+          "threshold_ms": 4000
+        },
+        {
+          "threshold_ms": 6000
+        }
+      ]
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.125,
+    "tolerance": 0.001,
+    "max_probes": 4,
+    "sample_seed": 20260325
+  },
+  "llm": {
+    "system_prompt": "Tune community vLLM 0.20.0 serving for Qwen3-30B-A3B. Start from the default vLLM engine configuration, use only launch-safe patches, and optimize request_rate_per_gpu under the configured SLO.",
+    "max_history_trials": 8,
+    "use_harness": false
+  }
+}
--- a/configs/examples/dash0_qwen30b_a3b_stopA_on.json
+++ b/configs/examples/dash0_qwen30b_a3b_stopA_on.json
@@ -0,0 +1,147 @@
+{
+  "study_id": "dash0-qwen30b-a3b-stopA-on-chat-0-8k",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "Qwen/Qwen3-30B-A3B",
+    "served_model_name": "qwen3-30b-a3b-community"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "0.20.0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18230,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 900,
+    "launch_args": [
+      "serve",
+      "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
+    ],
+    "base_envs": {
+      "CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7",
+      "HOME": "/tmp/wjh",
+      "XDG_CACHE_HOME": "/tmp/wjh/.cache"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18230,
+      "served-model-name": "qwen3-30b-a3b-community",
+      "gpu-memory-utilization": 0.9,
+      "max-model-len": 16384,
+      "trust-remote-code": true,
+      "enable-prefix-caching": true
+    },
+    "tunable_envs": [],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "enable-expert-parallel",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "block-size",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ]
+    },
+    "python_executable": "/tmp/wjh/venvs/vllm-0.20.0-cu129/bin/python"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "completion_tokens_override": 128,
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 64,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "replay_time_scale": 1.0,
+    "early_stop_max_lag_s": 120.0,
+    "early_stop_max_elapsed_s": 900.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "step_ms",
+      "buckets": [
+        {
+          "max_input_tokens": 4096,
+          "threshold_ms": 2000
+        },
+        {
+          "max_input_tokens": 32768,
+          "threshold_ms": 4000
+        },
+        {
+          "threshold_ms": 6000
+        }
+      ]
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 0.125,
+    "tolerance": 0.001,
+    "max_probes": 4,
+    "sample_seed": 20260325
+  },
+  "llm": {
+    "system_prompt": "Tune community vLLM 0.20.0 serving for Qwen3-30B-A3B. Start from the default vLLM engine configuration, use only launch-safe patches, and optimize request_rate_per_gpu under the configured SLO.",
+    "max_history_trials": 8,
+    "use_harness": false
+  }
+}
--- a/configs/examples/dash0_qwen30b_a3b_stopB_e2e.json
+++ b/configs/examples/dash0_qwen30b_a3b_stopB_e2e.json
@@ -0,0 +1,155 @@
+{
+  "study_id": "dash0-qwen30b-a3b-stopB-e2e-hi-chat-0-8k",
+  "hardware": {
+    "gpu_count": 8,
+    "gpu_model": "H20",
+    "host_candidates": [
+      "dash0"
+    ]
+  },
+  "model": {
+    "model_id": "Qwen/Qwen3-30B-A3B",
+    "served_model_name": "qwen3-30b-a3b-community"
+  },
+  "engine": {
+    "engine_name": "vllm",
+    "engine_version": "0.20.0",
+    "exec_path": "/usr/local/bin/vllm",
+    "cwd": "/home/admin/cpfs/wjh/aituner/aituner",
+    "host": "127.0.0.1",
+    "port": 18230,
+    "healthcheck_path": "/v1/models",
+    "ready_timeout_s": 900,
+    "request_timeout_s": 900,
+    "launch_args": [
+      "serve",
+      "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B"
+    ],
+    "base_envs": {
+      "CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7",
+      "HOME": "/tmp/wjh",
+      "XDG_CACHE_HOME": "/tmp/wjh/.cache"
+    },
+    "base_flags": {
+      "host": "127.0.0.1",
+      "port": 18230,
+      "served-model-name": "qwen3-30b-a3b-community",
+      "gpu-memory-utilization": 0.9,
+      "max-model-len": 16384,
+      "trust-remote-code": true,
+      "enable-prefix-caching": true
+    },
+    "tunable_envs": [],
+    "tunable_flags": [
+      "tensor-parallel-size",
+      "data-parallel-size",
+      "enable-expert-parallel",
+      "expert-parallel-size",
+      "gpu-memory-utilization",
+      "max-num-batched-tokens",
+      "max-num-seqs",
+      "block-size",
+      "enable-prefix-caching",
+      "enable-chunked-prefill"
+    ],
+    "topology_constraints": {
+      "require_tp_dp_product_equals_gpu_count": false,
+      "require_ep_size_leq_tp_dp_product": true,
+      "require_ep_size_divides_tp_dp_product": true,
+      "require_enable_expert_parallel_when_ep_gt_one": true,
+      "validate_cuda_graph_sizes_divisible_by_tp_when_tp_ep_reduce_scatter": true,
+      "allowed_tp_dp_products": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_tensor_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_data_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ],
+      "allowed_expert_parallel_sizes": [
+        1,
+        2,
+        4,
+        8
+      ]
+    },
+    "python_executable": "/tmp/wjh/venvs/vllm-0.20.0-cu129/bin/python"
+  },
+  "trace": {
+    "windows_path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json",
+    "window_id": "chat_w20260311_1000",
+    "completion_tokens_override": 128,
+    "u_field": "sampling_u",
+    "timestamp_field": "timestamp",
+    "max_concurrency": 64,
+    "input_length_filter": {
+      "min_input_tokens": 0,
+      "max_input_tokens": 8192
+    },
+    "max_requests_per_probe": 512,
+    "replay_time_scale": 0.1,
+    "early_stop_max_lag_s": 120.0,
+    "early_stop_max_elapsed_s": 900.0,
+    "adaptive_stop": {
+      "enabled": true,
+      "tau": 0.9,
+      "tau_c": 0.9,
+      "stable_checks": 3,
+      "max_checks": 20,
+      "min_fraction": 0.1,
+      "boundary_delta": 0.02
+    }
+  },
+  "slo": {
+    "target_pass_rate": 0.95,
+    "ttft_rule": {
+      "kind": "step_ms",
+      "buckets": [
+        {
+          "max_input_tokens": 4096,
+          "threshold_ms": 2000
+        },
+        {
+          "max_input_tokens": 32768,
+          "threshold_ms": 4000
+        },
+        {
+          "threshold_ms": 6000
+        }
+      ]
+    },
+    "tpot_rule": {
+      "kind": "fixed_ms",
+      "threshold_ms": 50
+    }
+  },
+  "search": {
+    "low": 0.0,
+    "high": 1.0,
+    "tolerance": 0.001,
+    "max_probes": 6,
+    "sample_seed": 20260325
+  },
+  "llm": {
+    "system_prompt": "Tune community vLLM 0.20.0 serving for Qwen3-30B-A3B. Start from the default vLLM engine configuration, use only launch-safe patches, and optimize request_rate_per_gpu under the configured SLO.",
+    "max_history_trials": 8,
+    "use_harness": true,
+    "endpoint": {
+      "provider": "codex",
+      "model": "gpt-5.4",
+      "stream": true,
+      "api_key_env": "OPENAI_API_KEY",
+      "timeout_s": 240
+    }
+  }
+}
--- a/configs/examples/stopb_27b_ab/p1_tp1.json
+++ b/configs/examples/stopb_27b_ab/p1_tp1.json
@@ -0,0 +1,13 @@
+{
+  "observation": "baseline TP1 (deployed flags)",
+  "diagnosis": "deterministic TP A/B point",
+  "config_patch": {
+    "env_patch": {},
+    "flag_patch": {}
+  },
+  "expected_effects": [
+    "measure peak request_rate_per_gpu at this topology"
+  ],
+  "why_not_previous_failures": "n/a",
+  "should_stop": false
+}
--- a/configs/examples/stopb_27b_ab/p2_tp2.json
+++ b/configs/examples/stopb_27b_ab/p2_tp2.json
@@ -0,0 +1,15 @@
+{
+  "observation": "TP2",
+  "diagnosis": "deterministic TP A/B point",
+  "config_patch": {
+    "env_patch": {},
+    "flag_patch": {
+      "tensor-parallel-size": 2
+    }
+  },
+  "expected_effects": [
+    "measure peak request_rate_per_gpu at this topology"
+  ],
+  "why_not_previous_failures": "n/a",
+  "should_stop": false
+}
--- a/configs/examples/stopb_27b_ab/p3_tp4.json
+++ b/configs/examples/stopb_27b_ab/p3_tp4.json
@@ -0,0 +1,15 @@
+{
+  "observation": "TP4",
+  "diagnosis": "deterministic TP A/B point",
+  "config_patch": {
+    "env_patch": {},
+    "flag_patch": {
+      "tensor-parallel-size": 4
+    }
+  },
+  "expected_effects": [
+    "measure peak request_rate_per_gpu at this topology"
+  ],
+  "why_not_previous_failures": "n/a",
+  "should_stop": false
+}
--- a/docs/harness-ablation/qwen27b-tp-sweep-20260616.md
+++ b/docs/harness-ablation/qwen27b-tp-sweep-20260616.md
@@ -0,0 +1,51 @@
+# Qwen3.5-27B TP sweep under length-aware TTFT SLO — 2026-06-16
+
+Branch `feat/two-stop`. Deterministic ground-truth A/B (proposal files, no LLM):
+TP1 vs TP2 vs TP4 on the dense Qwen3.5-27B (internal 256k, fp8, spec-decode) at
+0–8k chat, vLLM 0.11.1, H20, `replay_time_scale=1.0` (no smoke), Stop-A enabled,
+pinned to GPUs 2–7.
+
+**SLO**: TTFT ≤ `4000 + 0.125·L_in` ms (= 4s + L_in/8k), TPOT ≤ 50 ms, pass ≥ 95%.
+
+## Result
+
+| config | best_u | raw req/s | req/s/GPU | pass | saturated |
+| --- | --- | --- | --- | --- | --- |
+| TP1 | 0.00195 | 0.065 | **0.065** | 1.00 | no |
+| TP2 | 0.0195 | 0.585 | **0.2925** | 0.96 | no |
+| TP4 | 0.123 | 3.63 | **≥0.908** | 0.98 | **yes (best_u≈high=0.125)** |
+
+- **Per-GPU throughput rises sharply with TP for the dense 27B**: TP2 = 4.5× TP1,
+  TP4 ≥ 14× TP1. Opposite of the MoE Qwen3-30B-A3B (TP1 best per-GPU) — confirms the
+  dense-vs-MoE distinction.
+- **Mechanism**: TP1 is TPOT-bound — one H20 cannot decode a 27B under 50 ms/token
+  once the batch grows, so it saturates at ~0.065 req/s/GPU. Loosening TTFT (2s→4-5s)
+  did *not* change TP1 (still 0.065), confirming TPOT — not TTFT — is TP1's binding
+  constraint. Each TP doubling speeds decode+prefill enough to more than recover the
+  added GPUs.
+- **TP4 saturated** the offered-load ceiling (`best_u=0.123 ≈ 0.125`): still feasible
+  after ~the whole trace, so 0.908 is a lower bound. True peak (and TP8) need a
+  raised `search.high` to measure.
+
+## Process findings (fed back into the harness)
+
+- **Bug fixed**: a request exceeding `request_timeout_s` raised a raw `TimeoutError`
+  mid-stream that escaped `_run_one_request` and crashed the whole trial; now wrapped
+  as `HttpClientError` (failed request, not failed trial). Commit `2fcaf80`.
+- **Open gap**: killing a `study tune` run orphans the `VLLM::EngineCore` workers
+  (SIGTERM/SIGKILL of the loop doesn't tear down the engine), which twice left leaked
+  GPU memory on GPUs 0/1 (dead PIDs still pinning KV, only clearable via root
+  `nvidia-smi --gpu-reset`). Fix: SIGTERM handler in the CLI loop + make
+  `_terminate_process_tree` match `EngineCore` workers, not just `vllm serve`.
+- Experiment hygiene: scale=1.0 makes each probe take real arrival time; `search.high`
+  must bracket the config's boundary (too wide wastes probes on a low-capacity config;
+  too low saturates a high-capacity one), and `request_timeout_s` must be modest so
+  overloaded probes drain fast.
+
+## Next
+
+- Re-measure TP4 (and TP8) with `search.high` raised (e.g. 0.5) to find the true peak
+  per-GPU and the TP knee.
+- Run the Stop-B agentic loop on this 27B stack: unlike the 30B (baseline already
+  optimal), here the loop should climb TP1→TP2→TP4 and stop — a real improving
+  trajectory (the original Phase-5 "A" goal).
--- a/docs/harness-ablation/stop-a-validation-20260615.md
+++ b/docs/harness-ablation/stop-a-validation-20260615.md
@@ -0,0 +1,121 @@
+# Stop-A validation (Phase 3) — 2026-06-15
+
+Branch `feat/two-stop`. Stop-A = truncate each probe's replay once the offered
+L-C-A of the replayed prefix converges to the full set (pure L-C-A criterion +
+C-gate). This note records the CPU calibration and the GPU fidelity check.
+
+## 1. Calibration (CPU, no serving)
+
+`scripts/stop_a_calibration.py` on the dash0 0321 10:00–10:10 windows:
+
+| dim | chat (19239 req, hit≈7%) | coder (2451 req, structured reuse) |
+| --- | --- | --- |
+| A | ≥0.95 by frac 0.10 | fast |
+| L | ≥0.96 from frac 0.05 | 0.05=0.75 (heavy tail) → ≥0.94 by 0.20 |
+| **C (slowest)** | noisy, dips (0.50→0.885, 0.55→0.835), stable ≥0.92 only ~0.85 | smooth, stable ≥0.92 by ~0.70 |
+
+Stop fraction (τ_L=τ_A=0.90, W=3):
+
+| τ_c | chat | coder |
+| --- | --- | --- |
+| 0.85 | 0.45 (273s) | 0.45 (255s) |
+| 0.90 | 0.70 (423s) | 0.55 (318s) |
+| 0.92 | 0.85 (513s) | 0.70 (411s) |
+
+Findings:
+- **C is the slowest dimension in both workloads** — reproduces paper §5.2 / Fig 9.
+- **What makes C hard to call converged is signal *noise*, not reuse magnitude.**
+  Low-reuse chat has a sparse/spiky ideal-hit-length series, so its C similarity
+  oscillates and is *harder* to stabilize than the structured, higher-reuse coder.
+  Consequence: a strict τ_c (0.92) gives chat only ~15% saving. A more robust C
+  feature for the low-reuse regime is future work.
+
+## 2. GPU fidelity check (Qwen3-30B-A3B, vLLM 0.11.1, H20)
+
+One full-window run (`adaptive_stop` disabled, `replay_time_scale=1.0`, window
+`chat_w20260311_1000`, 0–8k, out=128), then `scripts/stop_a_validate.py`
+recomputes each probe's convergence prefix and compares the truncated verdict to
+the full verdict — so a single GPU run validates truncation fidelity (no second run).
+
+Trial result: best feasible `sampling_u=0.078125`, request_rate **2.30 req/s**,
+pass_rate 0.973.
+
+Per-probe verdict (τ=0.9):
+
+| τ_c | verdict matches | mean replay saved |
+| --- | --- | --- |
+| 0.85 | 3/4 | 54% |
+| 0.90 | 3/4 | 52% |
+| 0.92 | 3/4 | 38% |
+
+The mismatch is the same probe at every τ_c — the feasibility knee `0.08594`:
+
+```
+thresh    full_pass  prefix_pass  full_feas  prefix_feas
+0.08594   0.946      0.956–0.961  False      True   <- mismatch
+0.07812   0.973      0.987–0.990  True       True
+0.06250   0.986      1.000        True       True
+0.09375   0.268      0.49–0.54    False      False
+```
+
+## 3. Interpretation
+
+- **Stop-A works and saves ~50% of replay** (vs the full 600 s window) while
+  preserving 3/4 probe verdicts. (The paper's ~70% is vs a 30-min fixed baseline;
+  our baseline is the 600 s window, so the percentages are not directly comparable.)
+- **The one failure is a boundary false-positive at the feasibility knee.** At
+  `0.08594` the full window is 0.946 (just below the 0.95 SLO) but the prefix is
+  0.956–0.961 (just above): the *second half* of the window degraded — engine-state
+  drift (KV fill / fragmentation / later-arriving harder requests) that the
+  *offered* L-C-A cannot see. The C-gate did not help because offered-C had
+  converged; the divergence is in the measured pass-rate, not in C.
+- If Stop-A were enabled, the binary search would accept `0.08594`, overestimating
+  the peak sustainable rate by one binary step (~10%).
+
+**This is the boundary jitter we accepted when choosing the pure-L-C-A criterion.**
+The data now argues for revisiting the previously-declined **SLO-boundary guard**:
+keep replaying while the measured pass-rate is within ±δ of the target, even after
+L-C-A converges. It targets exactly this knee case at low extra cost (it only
+extends replay on probes sitting on the feasibility boundary). Recommend adding it
+as a small Stop-A enhancement before enabling Stop-A in production studies.
+
+## 4. SLO-boundary guard (implemented + validated)
+
+Added `trace.adaptive_stop.boundary_delta` (default 0.02): when a truncated probe's
+measured pass-rate lands within ±δ of the SLO target, re-measure on the full window
+and use that verdict. Re-ran the same config with `adaptive_stop` enabled
+(τ=0.9, τ_c=0.90, δ=0.02):
+
+| threshold | feasible | pass | selected | replayed | boundary_extended |
+| --- | --- | --- | --- | --- | --- |
+| 0.06250 | True | 1.000 | 1086 | 487 (45%) | — |
+| 0.09375 | False | 0.444 | 1656 | 822 (50%) | — |
+| 0.07812 | True | 0.994 | 1378 | 682 (49%) | — |
+| 0.08594 | **False** | 0.947 | 1523 | **1523 (100%)** | **True** |
+
+Result: best feasible `sampling_u=0.078125` (rate 2.30 req/s) — **identical to the
+full-replay baseline**. The guard fired on exactly the one knee probe and
+re-measured it to the correct infeasible verdict; the other three probes truncated
+to ~45–50%. Net replayed 3514/5643 requests ≈ **38% replay saved on this trial
+while recovering the correct peak rate** (no one-step overestimate).
+
+**Conclusion: Stop-A with the boundary guard is correct (verdict matches full
+replay) and still saves replay time. Safe to enable.** Configs:
+`dash0_qwen30b_a3b_stopA_fulldata.json` (OFF baseline) and
+`dash0_qwen30b_a3b_stopA_on.json` (ON).
+
+## Repro
+
+```
+# calibration
+PYTHONPATH=src python3 scripts/stop_a_calibration.py \
+  --jsonl <DIR>/qwen_chat_blksz_64_032109-032111.jsonl --block-size 64 \
+  --window-start 3600 --window-end 4200 --gpu-count 8 --label chat
+# GPU run + fidelity
+PYTHONPATH=src python3 -m aituner.cli study tune \
+  --spec configs/examples/dash0_qwen30b_a3b_stopA_fulldata.json \
+  --store-root .aituner/stopA-fulldata --max-trials 1
+PYTHONPATH=src python3 scripts/stop_a_validate.py \
+  --spec configs/examples/dash0_qwen30b_a3b_stopA_fulldata.json \
+  --store-root .aituner/stopA-fulldata --tau 0.9 --tau-c 0.90
+```
--- a/docs/harness-ablation/stop-b-e2e-20260615.md
+++ b/docs/harness-ablation/stop-b-e2e-20260615.md
@@ -0,0 +1,86 @@
+# Stop-B end-to-end validation (Phase 5) — 2026-06-15
+
+Branch `feat/two-stop`. Real agentic loop on dash0: Qwen3-30B-A3B / vLLM 0.11.1 /
+8×H20, `gpt-5.4` (via codex/prism) proposing configs, Stop-A enabled to accelerate
+each evaluation, `use_harness=True` so the Stop-B deterministic validator + LLM-stop
+veto are active. Config `dash0_qwen30b_a3b_stopB_e2e.json`, `search.high=1.0`,
+`max_probes=6`, `--max-trials 8`.
+
+## Two stop paths exercised
+
+**Run A (`search.high=0.125`)** — the default config already saturates the offered-load
+search range, so Stop-B fired immediately via the **search-high-saturation** path:
+`stop_authorized_by: validator`, reason *"the incumbent's highest measured probe is
+feasible and within the binary-search resolution of search.high."* Correct
+measurement-ceiling stop (no point proposing configs when the load range, not the
+config, is the bound).
+
+**Run B (`search.high=1.0`)** — forces a real multi-iteration search:
+
+| trial | TP | DP | EP | feasible | raw req/s | **req/s/GPU** | source |
+| --- | --- | --- | --- | --- | --- | --- | --- |
+| 0001 | 1 | 1 | 1 | ✓ | 2.90 | **2.900** | baseline |
+| 0002 | 2 | 1 | 1 | ✓ | 4.42 | 2.208 | harness TP-seed |
+| 0003 | 2 | 1 | 2 | ✗ launch-fail | — | — | harness (EP) |
+| 0004 | 1 | 2 | 1 | ✓ | 4.42 | 2.208 | LLM (after veto) |
+| 0005 | 2 | 2 | 1 | ✓ | 8.37 | 2.092 | harness |
+| 0006 | 2 | 2 | 2 | ✗ launch-fail | — | — | harness (EP) |
+| 0007 | 4 | 1 | 1 | ✓ | 8.37 | 2.092 | LLM |
+| (0008) | — | — | — | **STOP** | — | — | LLM stop, honored after veto budget |
+
+Incumbent: **trial-0001 (TP1), 2.90 req/s/GPU — never beaten.**
+
+> **⚠️ The per-GPU trajectory above is NOT a valid benchmark — it validates only
+> the Stop-B *mechanics*.** Two confounds:
+> 1. **Trace-ceiling saturation.** TP2·DP2 and TP4 reached `best_sampling_u≈0.98`
+>    (still feasible after consuming ~the whole window), so their *true* peak
+>    per-GPU is higher than the 2.09 shown — we ran out of offered load to push
+>    them to their boundary. Only TP1 (u=0.31), TP2 (u=0.48) and DP2 (u=0.48)
+>    found real boundaries. The `sampling_u` axis maxes at the full trace, so any
+>    config that sustains more than the window's offered rate cannot be measured.
+> 2. **Smoke regime.** This run inherited `replay_time_scale=0.1` +
+>    `max_requests_per_probe=512` (README: convergence test, *not* a benchmark) —
+>    compressed arrivals distort A and the 512 cap imposes a ~8.4 req/s ceiling.
+>
+> The below-ceiling TP1 (2.90) > TP2 (2.21) ordering *may* be real for this model
+> (Qwen3-30B-A3B is an MoE with ~3B active params → little compute per token → TP
+> adds all-reduce overhead with little benefit), which differs from the dense
+> Qwen3.5-27B where TP2 wins. But this run cannot establish it. A valid benchmark
+> needs `scale=1.0`, no cap, and enough offered-load headroom that strong configs
+> are not trace-saturated — see the 27B TP A/B follow-up.
+
+## Phase-5 acceptance
+
+- **No regression.** The primary metric `request_rate_per_gpu` stayed 2.90 the whole
+  run. Scaling TP/DP raised *raw* throughput (4.42, 8.37) but lowered per-GPU
+  efficiency (2.21, 2.09); the loop correctly kept the TP1 baseline as incumbent and
+  never adopted a worse-per-GPU config. (Matches the paper: long-prompt, low-reuse
+  chat prefers small TP for per-GPU efficiency.)
+- **Stop-B authority validated live.** At trial 4 the LLM tried to stop
+  (`should_stop=true`); the deterministic validator **vetoed** it
+  (`validator_did_not_authorize_stop`, `continue_harness_guided_search`), forcing one
+  more confirmation (DP2, which also failed to beat baseline). After the budget, the
+  LLM's later, well-justified stop was honored (`stop_authorized_by:
+  llm_after_veto_budget`). The bounded veto behaved exactly as designed.
+- **Auditable reason chain.** Every stop/veto carries a diagnosis grounded in the
+  measured evidence (e.g. *"increasing TP 1→2 lowers per-GPU efficiency even though
+  token latency improves … EP is explicitly blocked by launch-failure evidence"*).
+- **Launch-failure robustness.** Two EP configs (trial-0003, 0006) failed to launch
+  under vLLM 0.11.1; the harness recorded them as hard-negative evidence and the LLM
+  explicitly stopped proposing EP.
+
+## Notes / limitations
+
+- For this workload the baseline (TP1) is already per-GPU optimal, so iterations-to-
+  *best* = 1; the remaining trials are the loop *confirming* no config beats baseline
+  before stopping. A workload with an under-tuned default would show an improving
+  trajectory; this run validates the stop/no-regression machinery, not a tuning win.
+- The final stop came via `llm_after_veto_budget` (validator vetoed once, then
+  deferred), not a pure deterministic validator stop — because the deterministic
+  conditions (3-within-2%, saturation, validation-exhausted) did not cleanly fire
+  when every trial was a distinct config with a distinct per-GPU rate. The validator
+  acted as the *guard* (preventing premature stop), which is its designed role.
+- 7 trials > the paper's 3–6 average, inflated by the wider search range, 2 EP
+  launch-failures, and the veto. Acceptable for a validation run.
+- LLM token: the non-interactive shell lacks `OPENAI_API_KEY`; export it from the
+  codex `auth.json` (`~/.codex/auth.json`) before the run.
--- a/docs/harness-ablation/stop-b-e2e-27b-20260616.md
+++ b/docs/harness-ablation/stop-b-e2e-27b-20260616.md
@@ -0,0 +1,64 @@
+# Stop-B end-to-end on dense Qwen3.5-27B (the improving trajectory) — 2026-06-16
+
+Branch `feat/two-stop`. Real `gpt-5.4` agentic loop (codex/prism), Stop-A enabled,
+length-aware TTFT SLO (4s + L_in/8k, TPOT ≤ 50 ms), vLLM 0.11.1, H20, GPUs 2–7,
+`replay_time_scale=1.0`, `search.high=0.25`, `inherit_incumbent_floor=true`.
+Config `dash0_qwen27b_stopB_loop.json`. Companion to the 30B run
+(`stop-b-e2e-20260616.md`); together they cover all Stop-B behaviors.
+
+## Trajectory (incumbent = TP4 @ 1.00 req/s/GPU)
+
+| iter | proposed by | config | per_gpu | adopted? |
+| --- | --- | --- | --- | --- |
+| 1 | baseline | TP1 | 0.123 | incumbent |
+| 2 | gpt-5.4 | TP2 | 0.2925 (2.4×) | ✅ new incumbent |
+| 3 | gpt-5.4 | TP4 | **1.0012 (8.1×)** | ✅ new incumbent |
+| 4 | gpt-5.4 | TP4 + chunked-prefill + mbt=16384 | 0.942 | ❌ **worse → rejected** |
+| 5 | gpt-5.4 | TP2 + DP2 | (loop stopped before completing) | — |
+
+(Run stopped manually after iter-4 — see "Why stopped" below. Incumbent preserved
+at TP4.)
+
+## What this demonstrates (the piece the 30B run could not)
+
+- **A genuine improving climb.** `gpt-5.4` + the harness raised per-GPU throughput
+  TP1 → TP2 → TP4 (0.123 → 0.29 → 1.00, 8.1×), each step a correctly-diagnosed real
+  gain: TP1 is TPOT-bound, so the agent scaled tensor-parallelism, then — once
+  topology was won — pivoted to **runtime tuning on the winning family** (chunked
+  prefill + larger batched tokens).
+- **No regression.** The runtime tweak (iter-4) measured *below* plain TP4
+  (0.942 < 1.00), and the harness correctly **kept TP4 as the incumbent** rather than
+  adopting the worse config — the core Stop-B guarantee, shown live.
+- Combined with the 30B run (search-high-saturation `validator`-authorized stop +
+  premature-LLM-stop veto), every Stop-B behavior is now validated end-to-end:
+  improving climb, correct bottleneck-driven proposals, no regression, deterministic
+  stop authority, and the LLM-stop veto.
+
+## Process wins / findings
+
+- **SIGTERM teardown fix validated in practice.** This loop was stopped with a plain
+  SIGTERM and the engine + EngineCore workers torn down cleanly — GPUs 2–7 freed, no
+  orphan, no leaked memory (contrast: the pre-fix runs twice leaked GPU0/1). Commit
+  `b17b213`.
+- **Timeout-as-failed-request fix** (`2fcaf80`) held — no trial crashed on
+  request timeouts this run.
+
+## Why stopped (efficiency finding — feeds next round)
+
+The loop was stopped after iter-4 rather than run to an explicit Stop-B firing,
+because each TP4-family trial took ~3 h: at `scale=1.0`, infeasible high-θ probes
+each run to the **`early_stop_max_elapsed_s=900` cap** (`probe_elapsed_s>900`), and
+the primary+fallback binary search doubles the probe count. Stop-A truncates a
+*converged* replay but does not shortcut an *overloaded* probe that simply runs out
+the clock. **For a practical agentic loop at scale=1.0, lower `early_stop_max_elapsed_s`
+(≈300 s)** so overloaded probes die fast; consider also having an infeasible-and-
+overloaded probe early-stop on a fast lag/throughput signal rather than the elapsed
+cap. The convergence itself was already evident (iter-4's runtime tweak and the
+queued TP2+DP2 were not beating TP4).
+
+## Next
+
+- Lower the elapsed cap and (optionally) re-run to capture the explicit Stop-B stop
+  on this 27B stack.
+- Land the deferred items: more robust C feature for the low-reuse regime; Stop-C
+  cross-day retune trigger; §7 baselines (SCOOT/naive/community).
--- a/scripts/stop_a_validate.py
+++ b/scripts/stop_a_validate.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Validate Stop-A truncation fidelity from a full-replay trial's probe_details.
+
+Given a completed trial that replayed the full window (adaptive_stop disabled), for
+each probe recompute the L-C-A convergence prefix and compare the feasibility
+verdict / pass-rate of the truncated prefix against the full probe. This answers:
+"would Stop-A have changed the measured peak-sustainable-rate?" using only the one
+full run (no second GPU run needed).
+
+Example:
+  PYTHONPATH=src python3 scripts/stop_a_validate.py \
+    --spec configs/examples/dash0_qwen30b_a3b_stopA_fulldata.json \
+    --store-root .aituner/stopA-fulldata --tau 0.9 --tau-c 0.90
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from aituner.lca import find_convergence_prefix, resolve_length_mode
+from aituner.spec import load_study_spec
+from aituner.trace import load_trace_requests, select_requests_for_threshold
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--spec", type=Path, required=True)
+    ap.add_argument("--store-root", type=Path, required=True)
+    ap.add_argument("--tau", type=float, default=0.9)
+    ap.add_argument("--tau-c", type=float, default=0.90)
+    ap.add_argument("--stable-checks", type=int, default=3)
+    ap.add_argument("--target-pass-rate", type=float, default=0.95)
+    args = ap.parse_args()
+
+    study = load_study_spec(args.spec)
+    window, requests = load_trace_requests(study, study_spec_path=args.spec)
+    mode = resolve_length_mode(request_mode=study.trace.request_mode)
+    gpu_count = study.hardware.gpu_count
+
+    detail_files = sorted(args.store_root.glob("*/trials/*/probe_details.jsonl"))
+    if not detail_files:
+        print(f"no probe_details.jsonl under {args.store_root}")
+        return 1
+
+    print(f"target_pass_rate={args.target_pass_rate} tau={args.tau} tau_c={args.tau_c}")
+    print(
+        "thresh   n_full  stop_idx  frac  full_pass  prefix_pass  "
+        "full_feas  prefix_feas  verdict_match"
+    )
+    mismatches = 0
+    total = 0
+    saved_fractions = []
+    for detail_file in detail_files:
+        with detail_file.open(encoding="utf-8") as handle:
+            for line in handle:
+                line = line.strip()
+                if not line:
+                    continue
+                probe = json.loads(line)
+                threshold = float(probe["threshold"])
+                outcomes = probe.get("outcomes") or []
+                # arrival-ordered outcomes that carry an arrival_s and verdict
+                ordered = sorted(
+                    (o for o in outcomes if o.get("arrival_s") is not None),
+                    key=lambda o: float(o["arrival_s"]),
+                )
+                n = len(ordered)
+                if n == 0:
+                    continue
+                selected = select_requests_for_threshold(requests, threshold=threshold)
+                cp = find_convergence_prefix(
+                    selected, window, gpu_count=gpu_count, length_mode=mode,
+                    tau=args.tau, tau_c=args.tau_c, stable_checks=args.stable_checks,
+                )
+                # Map the convergence prefix fraction onto the replayed outcomes.
+                stop_n = max(1, min(n, round(cp.fraction * n)))
+                full_pass = sum(1 for o in ordered if o.get("evaluation")) / n
+                prefix_pass = sum(1 for o in ordered[:stop_n] if o.get("evaluation")) / stop_n
+                full_feas = full_pass >= args.target_pass_rate
+                prefix_feas = prefix_pass >= args.target_pass_rate
+                match = full_feas == prefix_feas
+                total += 1
+                mismatches += 0 if match else 1
+                saved_fractions.append(1.0 - cp.fraction)
+                print(
+                    f"{threshold:.5f}  {n:6d}  {stop_n:7d}  {cp.fraction:.2f}  "
+                    f"{full_pass:.3f}      {prefix_pass:.3f}        "
+                    f"{str(full_feas):5s}      {str(prefix_feas):5s}        {match}"
+                )
+    if total:
+        avg_saved = sum(saved_fractions) / len(saved_fractions)
+        print(
+            f"\nverdict matches: {total - mismatches}/{total}  "
+            f"mean replay saved: {avg_saved*100:.0f}%"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/src/aituner/cli.py
+++ b/src/aituner/cli.py
@@ -376,6 +376,8 @@ def cmd_study_tune(args: argparse.Namespace) -> int:
                    "stop_authorized_by": (
                        "validator"
                        if (is_harness_stop or authorized)
+                        else "file_proposal"
+                        if proposal_source is not None
                        else "llm_after_veto_budget"
                    ),
                    "diagnosis": proposal.diagnosis,
--- a/src/aituner/http_client.py
+++ b/src/aituner/http_client.py
@@ -179,6 +179,9 @@ def chat_completion(
    except urllib.error.HTTPError as exc:
        detail = exc.read().decode("utf-8", errors="replace")
        raise HttpClientError(f"llm_completion failed: {exc.code} {detail}") from exc
+    except OSError as exc:
+        # TimeoutError (socket.timeout), URLError, ConnectionError all subclass OSError.
+        raise HttpClientError(f"llm_completion failed: {exc}") from exc


 def stream_text_completion(
@@ -232,6 +235,8 @@ def stream_text_completion(
    except urllib.error.HTTPError as exc:
        detail = exc.read().decode("utf-8", errors="replace")
        raise HttpClientError(f"stream_text_completion failed: {exc.code} {detail}") from exc
+    except OSError as exc:
+        raise HttpClientError(f"stream_text_completion failed: {exc}") from exc
    return "".join(parts)


@@ -293,6 +298,10 @@ def stream_chat_completion(
    except urllib.error.HTTPError as exc:
        detail = exc.read().decode("utf-8", errors="replace")
        raise HttpClientError(f"stream_chat_completion failed: {exc.code} {detail}") from exc
+    except OSError as exc:
+        # A request that exceeds request_timeout_s raises TimeoutError mid-stream;
+        # treat it as a failed request (SLO miss), not a crashed trial.
+        raise HttpClientError(f"stream_chat_completion failed: {exc}") from exc
    ttft_ms = None if first_token_at is None else (first_token_at - start) * 1000.0
    if completion_tokens is None and chunk_token_count > 0:
        completion_tokens = chunk_token_count
--- a/src/aituner/lca.py
+++ b/src/aituner/lca.py
@@ -373,12 +373,13 @@ def _prefix_profile(
    length_mode: str,
 ) -> WorkloadProfile:
    prefix = requests[:index]
+    start = float(prefix[0].arrival_s) if prefix else float(window.window_start)
    end = float(prefix[-1].arrival_s) if prefix else float(window.window_start)
    prefix_window = WindowRecord(
        window_id=window.window_id,
        trace_path=window.trace_path,
        trace_type=window.trace_type,
-        window_start=window.window_start,
+        window_start=start,
        window_end=end,
        source_payload=window.source_payload,
    )
--- a/src/aituner/slo.py
+++ b/src/aituner/slo.py
@@ -29,6 +29,9 @@ def _rule_threshold_ms(rule: ThresholdRule, prompt_tokens: int | None) -> float:
    if rule.kind == "fixed_ms":
        assert rule.threshold_ms is not None
        return rule.threshold_ms
+    if rule.kind == "linear_ms":
+        assert rule.intercept_ms is not None and rule.per_token_ms is not None
+        return float(rule.intercept_ms) + float(rule.per_token_ms) * float(prompt_tokens or 0)
    if rule.kind != "step_ms":
        raise ValueError(f"Unsupported threshold rule: {rule.kind}")
    prompt = float(prompt_tokens or 0)
--- a/src/aituner/spec.py
+++ b/src/aituner/spec.py
@@ -335,6 +335,7 @@ class AdaptiveStopSpec:
    stable_checks: int = 3
    max_checks: int = 20
    min_fraction: float = 0.1
+    boundary_delta: float = 0.02

    @classmethod
    def from_dict(cls, data: Any) -> "AdaptiveStopSpec":
@@ -357,13 +358,23 @@ class AdaptiveStopSpec:
        min_fraction = _require_float(
            m.get("min_fraction", 0.1), context="trace.adaptive_stop.min_fraction"
        )
+        boundary_delta = _require_float(
+            m.get("boundary_delta", 0.02), context="trace.adaptive_stop.boundary_delta"
+        )
        for name, value in (("tau", tau), ("tau_c", tau_c), ("min_fraction", min_fraction)):
            if not 0.0 < value <= 1.0:
                raise SpecError(f"trace.adaptive_stop.{name} must be in (0, 1].")
+        if not 0.0 <= boundary_delta < 1.0:
+            raise SpecError("trace.adaptive_stop.boundary_delta must be in [0, 1).")
        if stable_checks <= 0 or max_checks <= 0:
            raise SpecError(
                "trace.adaptive_stop.stable_checks and max_checks must be > 0."
            )
+        if stable_checks > max_checks:
+            raise SpecError(
+                "trace.adaptive_stop.stable_checks must be <= max_checks, "
+                "otherwise convergence can never be detected."
+            )
        return cls(
            enabled=enabled,
            tau=tau,
@@ -371,6 +382,7 @@ class AdaptiveStopSpec:
            stable_checks=stable_checks,
            max_checks=max_checks,
            min_fraction=min_fraction,
+            boundary_delta=boundary_delta,
        )


@@ -492,6 +504,8 @@ class ThresholdRule:
    kind: str
    threshold_ms: float | None = None
    buckets: list[dict[str, float]] = field(default_factory=list)
+    intercept_ms: float | None = None
+    per_token_ms: float | None = None

    @classmethod
    def from_dict(cls, data: Mapping[str, Any], *, context: str) -> "ThresholdRule":
@@ -503,6 +517,18 @@ class ThresholdRule:
                    data.get("threshold_ms"), context=f"{context}.threshold_ms"
                ),
            )
+        if kind == "linear_ms":
+            # threshold = intercept_ms + per_token_ms * input_tokens
+            # e.g. "4s + L_in/8k" -> intercept_ms=4000, per_token_ms=0.125
+            intercept_ms = _require_float(
+                data.get("intercept_ms"), context=f"{context}.intercept_ms"
+            )
+            per_token_ms = _require_float(
+                data.get("per_token_ms"), context=f"{context}.per_token_ms"
+            )
+            if intercept_ms < 0 or per_token_ms < 0:
+                raise SpecError(f"{context}.intercept_ms/per_token_ms must be >= 0.")
+            return cls(kind=kind, intercept_ms=intercept_ms, per_token_ms=per_token_ms)
        if kind == "step_ms":
            raw = data.get("buckets")
            if not isinstance(raw, list) or not raw:
--- a/src/aituner/worker.py
+++ b/src/aituner/worker.py
@@ -210,6 +210,50 @@ def _probe_outcome_details(
    }


+_SIGTERM_NOT_INSTALLED = object()
+
+
+def _install_sigterm_as_keyboardinterrupt() -> Any:
+    """Make SIGTERM raise KeyboardInterrupt so the engine-teardown finally runs.
+
+    When `study tune` is killed, a default SIGTERM skips the finally blocks and
+    orphans the vLLM engine (and its EngineCore workers) on the GPUs. Converting
+    SIGTERM to KeyboardInterrupt lets _terminate_process_tree run. Only installable
+    from the main thread; returns the previous handler (or a sentinel).
+    """
+    if threading.current_thread() is not threading.main_thread():
+        return _SIGTERM_NOT_INSTALLED
+
+    def _handler(signum: int, frame: Any) -> None:
+        raise KeyboardInterrupt()
+
+    try:
+        return signal.signal(signal.SIGTERM, _handler)
+    except (ValueError, OSError):
+        return _SIGTERM_NOT_INSTALLED
+
+
+def _restore_sigterm(previous: Any) -> None:
+    if previous is _SIGTERM_NOT_INSTALLED:
+        return
+    if threading.current_thread() is not threading.main_thread():
+        return
+    try:
+        signal.signal(signal.SIGTERM, previous)
+    except (ValueError, OSError):
+        pass
+
+
+def _ignore_sigterm_if_main() -> None:
+    """Ignore SIGTERM during teardown so a second signal cannot orphan the engine."""
+    if threading.current_thread() is not threading.main_thread():
+        return
+    try:
+        signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    except (ValueError, OSError):
+        pass
+
+
 def _adaptive_replay_set(
    selected: list[TraceRequest],
    *,
@@ -249,6 +293,29 @@ def _adaptive_replay_set(
    return replay, certificate


+def _should_extend_on_boundary(
+    *,
+    pass_rate: float,
+    target_pass_rate: float,
+    certificate: dict[str, Any] | None,
+    truncated: bool,
+    boundary_delta: float,
+) -> bool:
+    """SLO-boundary guard: re-measure on the full window when a truncated probe
+    lands within +/- boundary_delta of the SLO target.
+
+    Offered-L-C-A convergence cannot see engine-state drift in the window's tail,
+    so a near-boundary truncated verdict is untrustworthy. This fires only on
+    probes sitting on the feasibility knee, so non-boundary probes keep the Stop-A
+    time saving.
+    """
+    if certificate is None or not certificate.get("converged"):
+        return False
+    if not truncated or boundary_delta <= 0:
+        return False
+    return abs(float(pass_rate) - float(target_pass_rate)) <= float(boundary_delta)
+
+
 def _best_feasible_probe_record(probe_history: list[dict[str, Any]]) -> dict[str, Any] | None:
    feasible = [
        item
@@ -545,6 +612,7 @@ def run_trial(trial_spec_path: Path) -> dict[str, Any]:
            )

        process = launch_process()
+        previous_sigterm = _install_sigterm_as_keyboardinterrupt()
        probe_history: list[dict[str, Any]] = []
        failure_stage = "engine_launch"
        try:
@@ -563,18 +631,36 @@ def run_trial(trial_spec_path: Path) -> dict[str, Any]:
                    selected, study=study, window=window
                )
                restart_after_early_stop = study.trace.restart_engine_after_early_stop
-                outcomes, early_stopped, early_stop_reason = _replay_requests(
-                    replay_set,
-                    base_url=recipe.base_url,
-                    timeout_s=recipe.request_timeout_s,
-                    max_concurrency=study.trace.max_concurrency,
-                    target_pass_rate=study.slo.target_pass_rate,
-                    max_lag_s=study.trace.early_stop_max_lag_s,
-                    max_elapsed_s=study.trace.early_stop_max_elapsed_s,
-                    evaluate_outcome=lambda outcome: evaluate_request(outcome, study.slo),
-                    drain_inflight_on_early_stop=not restart_after_early_stop,
-                )
+
+                def _run(reqs: list[TraceRequest]) -> tuple[list[RequestOutcome], bool, str]:
+                    return _replay_requests(
+                        reqs,
+                        base_url=recipe.base_url,
+                        timeout_s=recipe.request_timeout_s,
+                        max_concurrency=study.trace.max_concurrency,
+                        target_pass_rate=study.slo.target_pass_rate,
+                        max_lag_s=study.trace.early_stop_max_lag_s,
+                        max_elapsed_s=study.trace.early_stop_max_elapsed_s,
+                        evaluate_outcome=lambda outcome: evaluate_request(outcome, study.slo),
+                        drain_inflight_on_early_stop=not restart_after_early_stop,
+                    )
+
+                outcomes, early_stopped, early_stop_reason = _run(replay_set)
                evaluations, summary = summarize_evaluations(outcomes, study.slo)
+                if _should_extend_on_boundary(
+                    pass_rate=summary["slo_pass_rate"],
+                    target_pass_rate=study.slo.target_pass_rate,
+                    certificate=adaptive_stop_certificate,
+                    truncated=len(replay_set) < len(selected),
+                    boundary_delta=study.trace.adaptive_stop.boundary_delta,
+                ):
+                    # On the feasibility knee the truncated verdict is untrustworthy;
+                    # re-measure the full window and use that result.
+                    replay_set = selected
+                    outcomes, early_stopped, early_stop_reason = _run(selected)
+                    evaluations, summary = summarize_evaluations(outcomes, study.slo)
+                    if adaptive_stop_certificate is not None:
+                        adaptive_stop_certificate["boundary_extended"] = True
                probe_details = _probe_outcome_details(
                    threshold=threshold,
                    selected=replay_set,
@@ -785,4 +871,6 @@ def run_trial(trial_spec_path: Path) -> dict[str, Any]:
            StudyStore.write_json(Path(trial.result_path), result)
            return result
        finally:
+            _ignore_sigterm_if_main()
            _terminate_process_tree(process, timeout_s=30.0, marker_env=trial_marker_env)
+            _restore_sigterm(previous_sigterm)
--- a/tests/test_core_flow.py
+++ b/tests/test_core_flow.py
@@ -16,6 +16,7 @@ from aituner.cli import main as cli_main
 from aituner.compare import _aggregate_summary, load_compare_spec, run_compare
 from aituner.engine import build_launch_recipe
 from aituner.http_client import (
+    HttpClientError,
    StreamMetrics,
    _auth_headers,
    _openai_url,
@@ -44,6 +45,7 @@ from aituner.spec import (
    ConfigPatch,
    LLMEndpointSpec,
    Proposal,
+    SloSpec,
    SpecError,
    StudyState,
    TrialSummary,
@@ -53,6 +55,9 @@ from aituner.store import StudyStore
 from aituner.trace import load_trace_requests, summarize_window
 from aituner.worker import (
    _adaptive_replay_set,
+    _install_sigterm_as_keyboardinterrupt,
+    _restore_sigterm,
+    _should_extend_on_boundary,
    _best_feasible_probe_record,
    _latency_summary,
    _run_one_request,
@@ -476,6 +481,128 @@ class CoreFlowTests(unittest.TestCase):
        self.assertIsNone(no_cert)
        self.assertEqual(len(passthrough), len(requests))

+    def test_boundary_guard_extends_only_near_the_slo_knee(self) -> None:
+        converged = {"converged": True}
+        # Truncated, converged, pass-rate on the knee -> re-measure full.
+        self.assertTrue(
+            _should_extend_on_boundary(
+                pass_rate=0.961, target_pass_rate=0.95, certificate=converged,
+                truncated=True, boundary_delta=0.02,
+            )
+        )
+        self.assertTrue(
+            _should_extend_on_boundary(
+                pass_rate=0.946, target_pass_rate=0.95, certificate=converged,
+                truncated=True, boundary_delta=0.02,
+            )
+        )
+        # Clearly feasible / clearly infeasible -> trust the truncated verdict.
+        self.assertFalse(
+            _should_extend_on_boundary(
+                pass_rate=0.99, target_pass_rate=0.95, certificate=converged,
+                truncated=True, boundary_delta=0.02,
+            )
+        )
+        self.assertFalse(
+            _should_extend_on_boundary(
+                pass_rate=0.50, target_pass_rate=0.95, certificate=converged,
+                truncated=True, boundary_delta=0.02,
+            )
+        )
+        # Not truncated, not converged, guard disabled, or no certificate -> no extend.
+        self.assertFalse(
+            _should_extend_on_boundary(
+                pass_rate=0.95, target_pass_rate=0.95, certificate=converged,
+                truncated=False, boundary_delta=0.02,
+            )
+        )
+        self.assertFalse(
+            _should_extend_on_boundary(
+                pass_rate=0.95, target_pass_rate=0.95, certificate={"converged": False},
+                truncated=True, boundary_delta=0.02,
+            )
+        )
+        self.assertFalse(
+            _should_extend_on_boundary(
+                pass_rate=0.95, target_pass_rate=0.95, certificate=converged,
+                truncated=True, boundary_delta=0.0,
+            )
+        )
+        self.assertFalse(
+            _should_extend_on_boundary(
+                pass_rate=0.95, target_pass_rate=0.95, certificate=None,
+                truncated=True, boundary_delta=0.02,
+            )
+        )
+
+    def test_linear_ms_ttft_rule_scales_with_input_length(self) -> None:
+        slo = SloSpec.from_dict(
+            {
+                "target_pass_rate": 0.95,
+                "ttft_rule": {"kind": "linear_ms", "intercept_ms": 4000, "per_token_ms": 0.125},
+                "tpot_rule": {"kind": "fixed_ms", "threshold_ms": 50},
+            }
+        )
+
+        def ev(prompt_tokens: int, ttft_ms: float):
+            return evaluate_request(
+                RequestOutcome(
+                    request_id="r",
+                    success=True,
+                    ttft_ms=ttft_ms,
+                    tpot_ms=10.0,
+                    prompt_tokens=prompt_tokens,
+                    completion_tokens=8,
+                ),
+                slo,
+            )
+
+        # threshold = 4000 + 0.125*L_in : 8k->5000ms, 0->4000ms
+        self.assertTrue(ev(8000, 4900).passed)
+        self.assertFalse(ev(8000, 5100).passed)
+        self.assertTrue(ev(0, 3900).passed)
+        self.assertFalse(ev(0, 4100).passed)
+
+    def test_streaming_socket_timeout_is_a_failed_request_not_a_crash(self) -> None:
+        # A request that exceeds request_timeout_s raises TimeoutError mid-stream;
+        # it must surface as HttpClientError (a failed request), never escape to
+        # crash the trial.
+        with mock.patch(
+            "aituner.http_client._urlopen", side_effect=TimeoutError("timed out")
+        ):
+            with self.assertRaises(HttpClientError):
+                stream_chat_completion(
+                    base_url="http://127.0.0.1:1/v1",
+                    body={"messages": [{"role": "user", "content": "hi"}], "stream": True},
+                    timeout_s=0.5,
+                )
+            outcome = _run_one_request(
+                TraceRequest(
+                    row_id="r",
+                    arrival_s=0.0,
+                    sampling_u=1.0,
+                    body={"messages": [{"role": "user", "content": "hi"}], "stream": True},
+                    prompt_tokens_hint=10,
+                    completion_tokens_hint=None,
+                ),
+                base_url="http://127.0.0.1:1/v1",
+                timeout_s=0.5,
+            )
+        self.assertFalse(outcome.success)
+        self.assertIn("timed out", outcome.error)
+
+    def test_sigterm_is_converted_to_keyboardinterrupt(self) -> None:
+        # So a killed `study tune` runs the engine-teardown finally instead of
+        # orphaning the vLLM EngineCore workers on the GPUs.
+        import signal as _signal
+
+        previous = _install_sigterm_as_keyboardinterrupt()
+        try:
+            with self.assertRaises(KeyboardInterrupt):
+                _signal.raise_signal(_signal.SIGTERM)
+        finally:
+            _restore_sigterm(previous)
+
    def test_lca_similarity_matrix_separates_different_profiles(self) -> None:
        window = WindowRecord(
            window_id="base",
Author	SHA1	Message	Date
Gahow Wang	f2ff0faebd	Document Stop-B end-to-end on dense 27B: the improving climb + no-regression Real gpt-5.4 agentic loop raised per-GPU TP1 0.123 -> TP2 0.2925 -> TP4 1.0012 (8.1x), each a correctly-diagnosed real gain; then a TP4 runtime tweak measured 0.942 < 1.00 and was correctly rejected (no regression). With the 30B run (validator stop + LLM-stop veto), all Stop-B behaviors are now validated end-to-end. The SIGTERM-teardown fix was validated in practice (clean engine teardown, no GPU leak on stop). Efficiency finding: at scale=1.0, infeasible high-theta probes burn the 900s elapsed cap, so a practical loop needs a lower cap; this is why the run was stopped after iter-4 rather than driven to an explicit Stop-B firing. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-16 18:07:00 +08:00
Gahow Wang	4a64196a99	Add 27B Stop-B agentic-loop config (harness-driven, GPUs 2-7) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-16 09:08:46 +08:00
Gahow Wang	b17b213575	Tear down the engine on SIGTERM instead of orphaning it Killing `study tune` with a default SIGTERM skipped the finally blocks, leaving the vLLM engine and its EngineCore workers (which inherit the AITUNER_* marker env) alive on the GPUs — twice leaking GPU memory that needed a root reset. Install a SIGTERM handler in run_trial that raises KeyboardInterrupt so _terminate_process_tree runs, ignore SIGTERM during teardown so a second signal can't re-orphan it, and restore the prior handler afterward. Main-thread-guarded; unit-tested. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-16 09:08:06 +08:00
Gahow Wang	93ce339d61	Document 27B TP sweep: per-GPU rises sharply with TP (dense), opposite of MoE Under the length-aware TTFT SLO (4s + L_in/8k), dense Qwen3.5-27B per-GPU throughput: TP1=0.065, TP2=0.2925 (4.5x), TP4>=0.908 (>=14x, ceiling-saturated). TP1 is TPOT-bound (one H20 can't decode a 27B under 50ms/token once batched); loosening TTFT didn't move TP1, confirming TPOT is the binding constraint. Opposite of MoE 30B-A3B where TP1 was best per-GPU. Validates the harness + length-aware SLO produce meaningful, non-saturated measurements (TP1/TP2). TP4 saturated -> lower bound. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-16 01:54:40 +08:00
Gahow Wang	b1b74318f6	Pin 27B A/B to GPUs 2-7 (route around leaked GPU0/1 memory) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 23:01:22 +08:00
Gahow Wang	2fcaf80450	Wrap socket/timeout errors in HTTP client as HttpClientError stream_chat_completion (and the LLM stream/chat paths) only caught HTTPError, so a request exceeding request_timeout_s raised a raw TimeoutError mid-stream that escaped _run_one_request (which only catches HttpClientError), propagated through the probe, and crashed the whole trial ("failed: timed out"). A timed-out request is a failed request (SLO miss), not a trial crash. Catch OSError (covers TimeoutError, URLError, ConnectionError) after HTTPError and wrap it. Exposed by lowering request_timeout_s to 180s on the 27B run. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 22:58:28 +08:00
Gahow Wang	3541065675	Speed up 27B TP A/B: request_timeout 180s, search.high 0.125 The wide 0.5 range made TP1 (low-capacity) waste many infeasible high-theta probes, and the 900s request timeout made overloaded probes drain hung requests for 15min each. Cap drain at 180s and bound the search to where the boundaries actually are. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 22:40:42 +08:00
Gahow Wang	7678c7d5e8	Switch 27B TP A/B to length-aware TTFT SLO (4s + L_in/8k), widen search Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 20:35:23 +08:00
Gahow Wang	ed2bbe0323	Add linear_ms SLO rule (length-aware TTFT budget) threshold_ms = intercept_ms + per_token_ms * input_tokens. Lets the TTFT target scale with prefill work, e.g. "4s + L_in/8k" => intercept_ms=4000, per_token_ms=0.125 (4s base, +1s per 8k input tokens). slo + spec + test. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 20:35:23 +08:00
Gahow Wang	77af4ded2a	Flag Stop-B e2e per-GPU trajectory as non-benchmark (saturation + smoke regime) The reported trajectory validates the Stop-B mechanics only. TP2-DP2/TP4 saturated the trace ceiling (best_sampling_u~0.98) so their per-GPU peak is underestimated, and the run used the smoke regime (scale=0.1 + 512 cap). The TP1>TP2 ordering may be real for the small-active MoE but this run cannot establish it; the 27B TP A/B is the valid follow-up. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 18:40:38 +08:00
Gahow Wang	4f45b546a1	Add 27B TP A/B (deterministic ground-truth: does TP2 beat TP1 per-GPU) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 18:39:54 +08:00
Gahow Wang	90c3eb51c8	Document Stop-B end-to-end validation (Phase 5) Real gpt-5.4 agentic loop on Qwen3-30B-A3B/H20 with Stop-A enabled. Validates both Stop-B paths: search-high-saturation (validator-authorized immediate stop) and multi-iteration convergence. The TP1 baseline stays the per-GPU incumbent (2.90 req/s/GPU); TP/DP scaling raises raw throughput but lowers per-GPU efficiency and is correctly never adopted (no regression). The Phase-4 authority model is exercised live: a premature LLM stop is vetoed (validator_did_not_authorize_stop), then a later justified stop is honored after the veto budget. EP launch-failures handled as hard-negative evidence. Auditable reason chains throughout. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 17:58:44 +08:00
Gahow Wang	0b6beafeb8	Phase 5: widen search.high to 1.0 to force multi-iteration Stop-B convergence Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 17:12:32 +08:00
Gahow Wang	d4aff81691	Add Stop-B end-to-end config (agentic loop, Stop-A enabled) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 17:05:39 +08:00
Gahow Wang	f31e9ccfd5	Record Stop-A boundary-guard A/B: correct verdict, ~38% replay saved With the guard enabled the binary search recovers best sampling_u=0.078125 (rate 2.30 req/s), identical to the full-replay baseline. The guard fired on exactly the one feasibility-knee probe (0.08594, re-measured full -> infeasible); the other three probes truncated to ~45-50%. Net ~38% replay saved on the trial with no peak-rate overestimate. Stop-A + boundary guard is safe to enable. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 16:57:53 +08:00
Gahow Wang	03e556f0ab	Add Stop-A ON config (adaptive_stop enabled + boundary guard) for A/B Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 16:25:24 +08:00
Gahow Wang	dfc823f972	Add Stop-A SLO-boundary guard When a truncated probe's measured pass-rate lands within trace.adaptive_stop. boundary_delta of the SLO target, re-measure on the full window and use that verdict. Offered-L-C-A convergence cannot see engine-state drift in the window tail, so a near-knee truncated verdict is untrustworthy (validated: prefix 0.96 vs full 0.946 at threshold 0.08594). The guard fires only on feasibility-knee probes, so non-boundary probes keep the Stop-A saving. Default delta=0.02. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 16:25:24 +08:00
Gahow Wang	9f52812753	Document Stop-A validation: calibration + GPU fidelity check CPU calibration (chat vs coder) reproduces the paper's C-slowest ordering and shows C-convergence difficulty is driven by signal noise (low-reuse chat) not reuse magnitude. GPU fidelity check on Qwen3-30B-A3B: truncating at the L-C-A convergence prefix saves ~52% replay (tau_c=0.90) with 3/4 probe verdicts preserved; the one mismatch is a boundary false-positive at the feasibility knee (prefix 0.96 vs full 0.946), caused by second-half engine-state drift the offered L-C-A cannot see. Argues for revisiting the SLO-boundary guard before enabling. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 16:03:16 +08:00
Gahow Wang	958739027a	Fix Stop-A validation config: system vllm, cap max-model-len Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 15:22:48 +08:00
Gahow Wang	0f57ee96a9	Drop LLM endpoint from Stop-A full-data config (baseline-only run) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 15:19:46 +08:00
Gahow Wang	43125f48cf	Address review of two-stop branch - lca._prefix_profile: anchor the prefix window to the prefix's own first arrival so the A-rate is measured over the prefix span (matches the design intent; no-op for the 0-based canonical pipeline). - cli study tune: label file-originated stops as file_proposal rather than llm_after_veto_budget (the veto never applies to file proposals). - spec.AdaptiveStopSpec: reject stable_checks > max_checks (would make convergence undetectable and silently disable Stop-A). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 15:19:08 +08:00
Gahow Wang	3af1d84ac0	Add Stop-A full-data validation config (real-time replay, no cap) A single-config baseline run with adaptive_stop disabled and replay_time_scale=1.0, so per-request probe_details capture the full 600s window for offline analysis of whether truncating at the L-C-A convergence prefix preserves the feasibility verdict. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-06-15 15:15:12 +08:00