diff --git a/docs/action-aware-constraint-pilot-v2-protocol-20260714.md b/docs/action-aware-constraint-pilot-v2-protocol-20260714.md new file mode 100644 index 0000000..d51ca29 --- /dev/null +++ b/docs/action-aware-constraint-pilot-v2-protocol-20260714.md @@ -0,0 +1,35 @@ +# Action-aware constraint pilot v2 amendment + +Status: **FROZEN AFTER V1 CONTROLLER STOP AND BEFORE V2 GPU RUNS**. + +Date: 2026-07-14 (Asia/Singapore). + +The V1 configuration matrix and every scientific gate remain unchanged. V2 +fixes one controller bug and reruns every configuration and repetition fresh. + +## V1 controller failure + +The MNS64/MBBT2048 burn-in completed all 510 requests in 61.259 seconds, below +the frozen 90-second operational limit. However, the controller accidentally +assigned the preceding 16-request warm-up result to `burnin_result`; its state +therefore recorded 4.376 seconds and evaluated the wrong object. + +The first measured replay was terminated after 75 seconds, before it produced +a result. No target configuration had started. V1 consumed +0.3184109431 H20-hours and contributes no action label or telemetry to V2. + +## V2 correction and regression gate + +- The completed burn-in `run_client()` return value is assigned to + `burnin_result`. +- A dedicated `burnin_gate()` rejects any non-anchor object, any request count + other than 510, and elapsed time above 90 seconds. +- Unit tests explicitly pass a warm-up object and require rejection, then test + accepted and over-limit burn-ins. +- All five configs and 15 measured runs use a new run root; no V0/V1 runtime + artifact is reused. + +V0 and V1 together consumed 0.7043978426 H20-hours. V2's incremental hard cap +is therefore 7.2956021574 H20-hours, preserving the original global 8.0 +H20-hour cap. The authoritative accounting file is +`runs/action-aware-v0/prior-attempts-v2.json`. diff --git a/runs/action-aware-v0/pilot-manifest-v2.json b/runs/action-aware-v0/pilot-manifest-v2.json new file mode 100644 index 0000000..47908c7 --- /dev/null +++ b/runs/action-aware-v0/pilot-manifest-v2.json @@ -0,0 +1,227 @@ +{ + "budget": { + "expected_h20_hours": [ + 6.0, + 7.2 + ], + "expected_wall_minutes": [ + 90, + 110 + ], + "global_hard_cap_h20_hours": 8.0, + "hard_cap_h20_hours": 7.295602157380846, + "prior_attempt_artifact": "/home/admin/cpfs/wjh/aituner/aituner-action-aware-20260714/runs/action-aware-v0/prior-attempts-v2.json", + "prior_attempt_h20_hours": 0.7043978426191542, + "safety_h20_hours": 0.25, + "session_estimate_h20_hours": 1.35 + }, + "burnin": { + "anchor": 0.18919793755240089, + "arrival_order_sha256": "6c0ac4cb9a30ef501eeeacc8e6cc631c345e976db5ccf530ea5a1ec706d62a24", + "input_length_order_sha256": "7939cc20e1a00d1031d27d71508789f38decbbbb6ea59a1df18b2ec342fd2ef8", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "84f4809acbc8acd3b1d14dfa357134a1dc0b9287341624b33f598dafeef54dc7", + "selected_count": 510, + "study": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/studies/burnin-tp4.json", + "study_sha256": "5d6c2098042909a863efd3112818fbee9bafe96f22898ac98b66846dbe1fef0f" + }, + "configs": [ + { + "id": "b_base", + "mbbt": 2048, + "mns": 64, + "repetition_order": [ + 1, + 2, + 3 + ] + }, + { + "id": "a_base", + "mbbt": 8192, + "mns": 16, + "repetition_order": [ + 2, + 3, + 1 + ] + }, + { + "id": "shared", + "mbbt": 8192, + "mns": 64, + "repetition_order": [ + 3, + 1, + 2 + ] + }, + { + "id": "b_mns", + "mbbt": 2048, + "mns": 128, + "repetition_order": [ + 1, + 3, + 2 + ] + }, + { + "id": "a_mbbt", + "mbbt": 16384, + "mns": 16, + "repetition_order": [ + 2, + 1, + 3 + ] + } + ], + "engine": { + "burnin_max_elapsed_s": 90.0, + "client_timeout_s": 450.0, + "disable_slo_early_stop": true, + "duration_s": 300.0, + "tp": 4 + }, + "gates": { + "material_kv_usage": 0.9, + "minimum_exclusive_fraction": 0.1, + "minimum_exclusive_ratio": 5.0, + "minimum_relative_winner_margin": 0.1, + "phase_fractions": [ + 0.25, + 0.5, + 0.75, + 1.0 + ] + }, + "regimes": { + "A": { + "actions": { + "mbbt": "a_mbbt", + "mns": "shared" + }, + "source": "a_base" + }, + "B": { + "actions": { + "mbbt": "shared", + "mns": "b_mns" + }, + "source": "b_base" + } + }, + "repetitions": { + "1": { + "merged_trace": { + "bytes": 337429767, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep1.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9420, + "sha256": "68983266aa0e66aa589562f7c08edbd966f9ba4405e20c105adb43777d2dfbf5", + "source_sha256": [ + "b242d1d9086df3accab57b4c92445d5edd581e12f47e12cea227aa63964c6930", + "d23b549f7b69af3647308677bbf76f818a3c226a1c98f9a9f93f09ceee46be87" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low1.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high1.jsonl" + ] + }, + "selection": { + "anchor": 0.48686986110831465, + "arrival_order_sha256": "c2ad99986ce558da5901a9c5ec0a00bd69f198c981d8779235f2773a5c87f1c0", + "input_length_order_sha256": "9442bfebdc3fab5062dc1f4d688dc28c02afe3fd806c56dd8159f0ac7e6d0b94", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep1-tp4.json", + "study_sha256": "ecfff96e33d458eb1e3b9a6d24386f00cc6f1b19ff926e2ec6320b3f671a7ae3" + }, + "2": { + "merged_trace": { + "bytes": 337509330, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep2.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9457, + "sha256": "f38e8938f6a481fc6725b71b21aa04ff7eaf79783cdfd6e41aa2f074156f00c2", + "source_sha256": [ + "4cbb0baac082bd54af562ce2f39104c5c23b4671672da365a67b1e8c146adf9f", + "bb0bcd2564a88000f435f12feb21c7c902eafc9ea5fe916adfe9d1eae47f3f9a" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low2.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high2.jsonl" + ] + }, + "selection": { + "anchor": 0.4825698948735577, + "arrival_order_sha256": "b9fc12cf3f86bc8a79bee65296e65aa2b8bf2aeca46b2887094c669adcbb9a00", + "input_length_order_sha256": "d8d4bd6fc8ba852a45605b673b6b3e4f33b58f459e69f2a032d226ee175b074e", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep2-tp4.json", + "study_sha256": "d92a576db031db24bb58f354ea725d7f7567cb76699d387117ac5a6c9317bbb9" + }, + "3": { + "merged_trace": { + "bytes": 337450256, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep3.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9431, + "sha256": "3094084b0bb20cc02eecf465091a5c919b4e5b112f704cdc36a563d1efdcee46", + "source_sha256": [ + "1f7ececb142f9a363d2d1ca25eb7b8488b2cc319a51b55faa384f2a3d51f2142", + "6f326234791e1cff4ff866bface0d097d0d6e3844eebb1c97653d8e9c35e9397" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low3.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high3.jsonl" + ] + }, + "selection": { + "anchor": 0.48664343020532463, + "arrival_order_sha256": "efce7339e22d3618cb4d55e6b55bfddb2c563c18faba2a992d5829c13e3f55e9", + "input_length_order_sha256": "0792b05fff6729fbd92ab2bb4cb6d31bea7799e232ad42772936bc06efbafb54", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep3-tp4.json", + "study_sha256": "fb8ffe256dace32f4ca8a8d49b662d98c3b69b94ecc8fa826e43068b238884ab" + } + }, + "sanity": { + "invariants": { + "all_repetition_orders_are_permutations": true, + "five_unique_configs": true, + "same_load_all_repetitions": true, + "shared_endpoint_reused_by_both_regimes": true, + "three_disjoint_repetitions": true + }, + "red_flags": [] + }, + "schema": "action-aware-constraint-pilot-manifest-v1", + "source": { + "base_manifest": "/home/gahow/phd/aituner/runs/intervention-response-v2/pilot-manifest-v3.json", + "base_manifest_sha256": "273db1181dcc9d6b64439650d0642ebe553b12e6aa9adebfbe3758a7977e5611", + "source_trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260312_1000.jsonl", + "source_trace_sha256": "875ba869775deb78086477919f03b322da14e2673c7d070e26528c4190912757", + "window_id": "chat_w20260312_1000" + }, + "status": "PASS" +} diff --git a/runs/action-aware-v0/pilot_controller.py b/runs/action-aware-v0/pilot_controller.py index 0492116..14668f9 100644 --- a/runs/action-aware-v0/pilot_controller.py +++ b/runs/action-aware-v0/pilot_controller.py @@ -334,6 +334,30 @@ def validate_result( raise RuntimeError(f"selection hash mismatch {result_key}: {role}") +def burnin_gate( + result: Mapping[str, Any], + *, + expected_count: int, + maximum_elapsed_s: float, +) -> dict[str, Any]: + if result.get("kind") != "anchor": + raise RuntimeError("burnin gate received a non-anchor result") + if int(result["selection"]["count"]) != expected_count: + raise RuntimeError("burnin gate received the wrong request set") + elapsed_s = float(result["interval"]["elapsed_s"]) + summary = { + "elapsed_s": elapsed_s, + "pass_rate": float(result["pass_rate"]), + "feasible": bool(result["feasible"]), + } + if elapsed_s > maximum_elapsed_s: + raise RuntimeError( + f"burnin throughput gate failed: {elapsed_s:.3f}s > " + f"{maximum_elapsed_s:.3f}s" + ) + return summary + + def run_client( *, entry: dict[str, Any], @@ -458,7 +482,7 @@ def execute_session( first = manifest["repetitions"][str(config["repetition_order"][0])] session_state["status"] = "warmup" atomic_json(state_path, state) - burnin_result = run_client( + run_client( entry=entry, config=config, role="warmup", @@ -472,7 +496,7 @@ def execute_session( session_state["status"] = "burnin" atomic_json(state_path, state) burnin = manifest["burnin"] - run_client( + burnin_result = run_client( entry=entry, config=config, role="burnin", @@ -482,19 +506,12 @@ def execute_session( state=state, timeout_s=float(manifest["engine"]["client_timeout_s"]), ) - burnin_elapsed_s = float(burnin_result["interval"]["elapsed_s"]) - session_state["burnin"] = { - "elapsed_s": burnin_elapsed_s, - "pass_rate": float(burnin_result["pass_rate"]), - "feasible": bool(burnin_result["feasible"]), - } + session_state["burnin"] = burnin_gate( + burnin_result, + expected_count=int(burnin["selected_count"]), + maximum_elapsed_s=float(manifest["engine"]["burnin_max_elapsed_s"]), + ) atomic_json(state_path, state) - burnin_limit_s = float(manifest["engine"]["burnin_max_elapsed_s"]) - if burnin_elapsed_s > burnin_limit_s: - raise RuntimeError( - f"burnin throughput gate failed: {burnin_elapsed_s:.3f}s > " - f"{burnin_limit_s:.3f}s" - ) session_state["status"] = "measured" atomic_json(state_path, state) for repetition in config["repetition_order"]: diff --git a/runs/action-aware-v0/prior-attempts-v2.json b/runs/action-aware-v0/prior-attempts-v2.json new file mode 100644 index 0000000..3faf060 --- /dev/null +++ b/runs/action-aware-v0/prior-attempts-v2.json @@ -0,0 +1,24 @@ +{ + "global_hard_cap_h20_hours": 8.0, + "invariants": { + "all_gpus_idle_after_each_stop": true, + "no_completed_measured_runs": true, + "no_prior_runtime_data_reused": true + }, + "prior_attempt_h20_hours": 0.7043978426191542, + "schema": "action-aware-prior-attempts-v2", + "stops": [ + { + "artifact": "/home/admin/cpfs/wjh/action-aware-constraint-v0-20260714/operational-stop-v0.json", + "h20_hours": 0.38598689953486126, + "reason": "MBBT256 burn-in remained throughput-backlogged", + "stage": "burnin" + }, + { + "artifact": "/home/admin/cpfs/wjh/action-aware-constraint-v1-20260714/operational-stop-v1.json", + "h20_hours": 0.31841094308429296, + "reason": "controller passed the warmup result to the burn-in gate", + "stage": "first measured run in flight; zero measured results completed" + } + ] +} diff --git a/runs/action-aware-v0/test_pilot.py b/runs/action-aware-v0/test_pilot.py index c46af88..c2867d9 100644 --- a/runs/action-aware-v0/test_pilot.py +++ b/runs/action-aware-v0/test_pilot.py @@ -201,6 +201,32 @@ def main() -> None: "--max-num-batched-tokens 2048" in revised_plan["sessions"][0]["commands"]["server"] ) + accepted_burnin = { + "kind": "anchor", + "selection": {"count": 510}, + "interval": {"elapsed_s": 61.25}, + "pass_rate": 0.5, + "feasible": False, + } + assert controller.burnin_gate( + accepted_burnin, expected_count=510, maximum_elapsed_s=90.0 + )["elapsed_s"] == 61.25 + warmup = copy.deepcopy(accepted_burnin) + warmup["kind"] = "warmup" + try: + controller.burnin_gate(warmup, expected_count=510, maximum_elapsed_s=90.0) + except RuntimeError as error: + assert "non-anchor" in str(error) + else: + raise AssertionError("warmup incorrectly passed the burnin gate") + slow = copy.deepcopy(accepted_burnin) + slow["interval"]["elapsed_s"] = 91.0 + try: + controller.burnin_gate(slow, expected_count=510, maximum_elapsed_s=90.0) + except RuntimeError as error: + assert "throughput gate failed" in str(error) + else: + raise AssertionError("slow burnin incorrectly passed the throughput gate") print("action-aware constraint pilot: PASS")