diff --git a/docs/harness-ablation/prefill-scheduler-normalized-design-20260629.md b/docs/harness-ablation/prefill-scheduler-normalized-design-20260629.md index 8712c9f..734599e 100644 --- a/docs/harness-ablation/prefill-scheduler-normalized-design-20260629.md +++ b/docs/harness-ablation/prefill-scheduler-normalized-design-20260629.md @@ -65,6 +65,9 @@ target_mbt = sqrt(current_mbt * prompt_tokens_p95) - `_prefill_scheduler_candidate_actions` - 输出 `prefill-scheduler-interaction` family。 - `score_factors` 显式记录 current/target `prefill_quantum_ratio`，方便后续实验解释。 + - 当 scheduler dimension 还没有被 materialized config 覆盖时，加入 + `uncovered_scheduler_dimension_bonus`，让该 family 在 topology settled 后优先于 + `gpu-memory-utilization` 这类 resource micro-tuning。 ## 为什么不是 rule-based hack @@ -101,12 +104,30 @@ target_mbt = sqrt(current_mbt * prompt_tokens_p95) - `adjust_admission_pressure_with_chunked_prefill` 3. 测试改为验证 normalized direction 和 scale sensitivity，而不是固定 absolute value。 +### 2026-06-29 远端 review feedback + +在 dash1 用 `36c301c` 启动 case3 bad-runtime 重跑后，trial-0003 的 +candidate-set 已经出现 `prefill-scheduler-interaction`： + +```text +action_id = seed_chunked_prefill_quantum +patch = enable-chunked-prefill=true, max-num-batched-tokens=8192 +ratio = target prefill_quantum_ratio ~= 1.05 +``` + +但初始 scoring 仍让 `raise_gpu_memory_utilization` 排在前面。这说明 family +接入是正确的，但排序仍偏向 resource micro-tuning。随后实现加入 +`uncovered_scheduler_dimension_bonus`：当 topology frontier 已覆盖、prefill scheduler +dimension 还没有被 materialized config 测过时，优先测试 scheduler hypothesis， +避免重复旧 harness 先爬 GMU 的失败轨迹。 + ## 单元验证 新增/更新的测试覆盖： - long-tail TTFT 下，过大的 `prefill_quantum_ratio` 会下降； - prompt length scale 变大时，下一步 MBT target 也变大； +- topology frontier 已覆盖后，未覆盖的 scheduler dimension 排在 GMU 微调之前； - short prompt workload 不激活 prefill scheduler family； - 原有 prefill stop guard 仍不允许在有 high-value candidate 时停止； - normalized full-config no-repeat 语义不变。 @@ -138,4 +159,3 @@ PYTHONPATH=src python3 -m unittest discover -s tests - candidate family sequence； - `prefill_quantum_ratio_current -> target` 的方向是否与 bottleneck evidence 一致； - 是否出现 repeated normalized full-config signature。 - diff --git a/src/aituner/harness.py b/src/aituner/harness.py index fb4050c..1353853 100644 --- a/src/aituner/harness.py +++ b/src/aituner/harness.py @@ -1738,11 +1738,18 @@ def _prefill_scheduler_candidate_actions( relief = 0.38 if admission_step is not None: relief += 0.06 - score = relief * max(confidence, 0.35) + _information_gain(bottleneck_hypotheses, "runtime") + 0.08 + coverage_bonus = 0.0 + if quantum_step["direction"] == "seed" or not current_chunked: + coverage_bonus = 0.28 + elif quantum_step["target"] is not None or admission_step is not None: + coverage_bonus = 0.14 + information_gain = _information_gain(bottleneck_hypotheses, "runtime") + score = relief * max(confidence, 0.35) + information_gain + 0.08 + coverage_bonus factors = { "expected_bottleneck_relief": round(relief, 4), "bottleneck_confidence": round(confidence, 4), - "information_gain": round(_information_gain(bottleneck_hypotheses, "runtime"), 4), + "information_gain": round(information_gain, 4), + "uncovered_scheduler_dimension_bonus": round(coverage_bonus, 4), "launch_safety": 0.08, "regression_risk": 0.06 if current_chunked else 0.1, "prefill_quantum_ratio_current": ( diff --git a/tests/test_core_flow.py b/tests/test_core_flow.py index c2fe5da..6308296 100644 --- a/tests/test_core_flow.py +++ b/tests/test_core_flow.py @@ -3407,6 +3407,110 @@ class CoreFlowTests(unittest.TestCase): self.assertGreater(targets[1], targets[0]) + def test_prefill_scheduler_coverage_precedes_gmu_microtune(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + study_path = _write_study_assets( + tmp_path, + engine_overrides={ + "base_flags": { + "host": "127.0.0.1", + "port": 8000, + "tensor-parallel-size": 2, + "data-parallel-size": 1, + "gpu-memory-utilization": 0.7, + "max-num-seqs": 8, + }, + "tunable_flags": [ + "tensor-parallel-size", + "data-parallel-size", + "gpu-memory-utilization", + "max-num-batched-tokens", + "max-num-seqs", + "enable-chunked-prefill", + ], + "topology_constraints": { + "allowed_tensor_parallel_sizes": [2, 4], + "allowed_data_parallel_sizes": [1], + "allowed_tp_dp_products": [2, 4], + }, + }, + trace_overrides={"max_concurrency": 64}, + ) + + def write_result(name: str, request_rate: float) -> Path: + path = tmp_path / f"{name}.json" + path.write_text( + json.dumps( + { + "status": "completed", + "best_sampling_u": 0.5, + "best_request_rate": request_rate, + "best_pass_rate": 0.95, + "probes": [ + { + "threshold": 0.5, + "feasible": True, + "payload": { + "request_rate": request_rate, + "pass_rate": 0.95, + "latency_summary": { + "failed_reason_counts": {"ttft_ms>4000.0": 24} + }, + }, + } + ], + } + ), + encoding="utf-8", + ) + return path + + study = load_study_spec(study_path) + state = StudyState( + study_id=study.study_id, + best_trial_id="trial-0001", + best_parallel_size=2, + best_request_rate=4.05, + best_request_rate_per_gpu=2.025, + trials=[ + TrialSummary( + trial_id="trial-0001", + status="completed", + parallel_size=2, + best_request_rate=4.05, + best_request_rate_per_gpu=2.025, + result_path=str(write_result("trial-0001", 4.05)), + config_patch={"env_patch": {}, "flag_patch": {}}, + ), + TrialSummary( + trial_id="trial-0002", + status="completed", + parallel_size=4, + best_request_rate=8.0, + best_request_rate_per_gpu=2.0, + result_path=str(write_result("trial-0002", 8.0)), + config_patch={ + "env_patch": {}, + "flag_patch": {"tensor-parallel-size": 4}, + }, + ), + ], + ) + + context = build_harness_context( + study=study, + window_summary={"prompt_tokens_p95": 7774, "prompt_tail_ratio_p95_p50": 3.0}, + state=state, + ) + action = context["experiment_plan"]["next_action"] + self.assertEqual(action["knob_family"], "prefill-scheduler-interaction") + self.assertEqual(action["action_id"], "seed_chunked_prefill_quantum") + self.assertGreater( + action["score_factors"]["uncovered_scheduler_dimension_bonus"], + 0.0, + ) + def test_prefill_scheduler_not_active_for_short_prompt_workload(self) -> None: with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp)