From 7fd956355072399b1b56946ecd8007da8f9d2fb0 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 17:52:21 +0800 Subject: [PATCH] Replace undrainable telemetry load with fresh rerun --- ...sponse-v2-phase-aware-protocol-20260714.md | 7 +- ...-response-v3-two-load-protocol-20260714.md | 73 +++++++++++++++++++ .../intervention-response-v2/analyze_pilot.py | 10 +-- .../pilot_controller.py | 32 +++++--- .../intervention-response-v2/prepare_pilot.py | 49 +++++++++---- .../intervention-response-v2/test_analysis.py | 14 +++- 6 files changed, 150 insertions(+), 35 deletions(-) create mode 100644 docs/intervention-response-v3-two-load-protocol-20260714.md diff --git a/docs/intervention-response-v2-phase-aware-protocol-20260714.md b/docs/intervention-response-v2-phase-aware-protocol-20260714.md index a29e535..4c72d37 100644 --- a/docs/intervention-response-v2-phase-aware-protocol-20260714.md +++ b/docs/intervention-response-v2-phase-aware-protocol-20260714.md @@ -1,9 +1,14 @@ # Phase-aware telemetry intervention-response v2 protocol -Status: **FROZEN BEFORE THE SYSTEMATIC 10% DECILE AUDIT**. +Status: **INVALID OPERATIONAL ATTEMPT; SUPERSEDED BY V3**. Date: 2026-07-14 (Asia/Singapore). +The first MNS=16 session timed out while draining the 3.125 requests/s/GPU +workload after its 300-second arrival window. It produced no high-load result, +and no MNS=64 endpoint was run. No comparative conclusion is drawn from this +attempt; see `intervention-response-v3-two-load-protocol-20260714.md`. + ## Correction to v0/v1 The 5/10-second analyses tested an ultra-early verifier. They did not test diff --git a/docs/intervention-response-v3-two-load-protocol-20260714.md b/docs/intervention-response-v3-two-load-protocol-20260714.md new file mode 100644 index 0000000..f5f83b7 --- /dev/null +++ b/docs/intervention-response-v3-two-load-protocol-20260714.md @@ -0,0 +1,73 @@ +# Phase-aware telemetry intervention-response v3 protocol + +Status: **FROZEN AFTER A NON-COMPARATIVE OPERATIONAL FAILURE AND BEFORE V3 RUNS**. + +Date: 2026-07-14 (Asia/Singapore). + +## Why v2 was invalid + +The first v2 session completed the 300-second arrival windows at 1.5 and 2.125 +requests/s/GPU. At 3.125 requests/s/GPU, MNS=16 could not drain the admitted +requests before the 450-second client timeout. The session produced no result +and no MNS=64 action endpoint was run. V2 is therefore an invalid operational +attempt, not evidence for or against the telemetry hypothesis. + +This failure was observed before any MNS action comparison. V3 excludes only +the unmeasurable overload point and reruns every retained point on fresh +servers; it does not reuse the completed v2 low/mid results. + +## Question and hypothesis + +Question: after enough replay time for queue, batch, and KV state to develop, +does an MNS intervention create telemetry responses that exceed workload-repeat +noise, and does any such response predict whether the intervention repairs the +full-run SLO outcome better than external prefix outcomes alone? + +Hypothesis: increasing MNS from 16 to 64 has little value at the 1.5 +requests/s/GPU control load but can repair the 2.125 requests/s/GPU pressure +load. Queue, running-set, batch, or KV telemetry should expose the difference +at stable replay phases. Label balance is an assumption to test, not a fact. + +## Frozen setup + +- Solo placement on dash0 GPU 0-3: 4 NVIDIA H20 GPUs, Qwen3-30B-A3B, patched + vLLM `0.24.1.dev3+opprof`, fixed TP=4. +- Action: MNS `16 -> 64`; all other engine and workload parameters fixed. +- Workload: `chat_w20260312_1000`, replay-time scale 0.5, hence 300 seconds. +- Loads per GPU: 1.5 control and 2.125 pressure requests/s. The failed 3.125 + overload point is excluded from V3 and retained only as a failure artifact. +- Three disjoint request bands. Each MNS action pair has exact request, + arrival, and input-length hashes. Endpoint order is A/B, B/A, A/B; load + order is low/mid, mid/low, low/mid. +- Every session starts a fresh server, then runs the accepted 16-request long + warm-up and bounded burn-in before measured runs. +- SLO-unrecoverable early stop is disabled. Measured results must cover the + full 300-second arrival window and must not be early-stopped. +- Cumulative checkpoints are 10%, 25%, 50%, 75%, and 100%; non-overlapping + quarter blocks are also reported. +- A Layer-1 interval is complete only if timestamps are monotonic and its + start, end, and maximum internal record gaps are each at most one second. +- Incremental V3 cap is the unused portion of the original 8 H20-hour cap. + The exact prior cost and V3 cap are machine-recorded in the manifest. + +## Frozen gates + +Data validity requires six uncensored sessions, six action pairs, eight +same-config repeat pairs, exact action-pair hashes, full request accounting, +zero Layer-1 drops, continuous coverage, all stream/footer invariants, no +co-resident compute process, idle GPUs before and after sessions, nonnegative +counters, bounded ratios, non-identical per-config state, and monotonic request +coverage across checkpoints. Any red flag stops analysis. + +Mechanism evidence requires at least two telemetry features to exceed the +unchanged v1 repeat-noise thresholds at the same pair of adjacent checkpoints. +Those features must have a direction consistent in both retained load regimes. + +Decision evidence additionally requires at least two positive and two negative +full-run action-efficacy labels, valid leave-one-repetition-out folds, and at +least one phase-stable mechanism feature whose balanced accuracy is at least +0.75 and at least 0.15 above the best external prefix-outcome feature at two +adjacent predeclared checkpoints from 25% onward. + +V3 remains a development mechanism pilot. Even `OPEN_E2E_POLICY_TEST` opens a +held-out tuning-policy experiment; it is not itself a paper performance claim. diff --git a/runs/intervention-response-v2/analyze_pilot.py b/runs/intervention-response-v2/analyze_pilot.py index 7cd539a..0586680 100644 --- a/runs/intervention-response-v2/analyze_pilot.py +++ b/runs/intervention-response-v2/analyze_pilot.py @@ -15,10 +15,10 @@ from typing import Any, Iterable, Mapping HERE = Path(__file__).resolve().parent P1_PATH = HERE.parent / "intervention-response-v0" / "analyze_p1.py" -SCHEMA = "intervention-response-phase-aware-pilot-analysis-v2" -EXPECTED_ACTION_PAIRS = 9 -EXPECTED_REPEAT_PAIRS = 12 -MIN_EFFICACY_CLASS = 3 +SCHEMA = "intervention-response-phase-aware-pilot-analysis-v3" +EXPECTED_ACTION_PAIRS = 6 +EXPECTED_REPEAT_PAIRS = 8 +MIN_EFFICACY_CLASS = 2 MAX_LAYER1_GAP_S = 1.0 @@ -520,7 +520,7 @@ def cumulative_coverage_gate(windows: list[dict[str, Any]]) -> dict[str, Any]: def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": + if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v3": raise ValueError("unexpected phase-aware pilot manifest schema") fractions = [float(value) for value in manifest["checkpoints"]["fractions"]] seconds = [float(value) for value in manifest["checkpoints"]["seconds"]] diff --git a/runs/intervention-response-v2/pilot_controller.py b/runs/intervention-response-v2/pilot_controller.py index cb089ff..fe41848 100644 --- a/runs/intervention-response-v2/pilot_controller.py +++ b/runs/intervention-response-v2/pilot_controller.py @@ -12,7 +12,7 @@ import subprocess import sys import time from pathlib import Path -from typing import Any +from typing import Any, Mapping HERE = Path(__file__).resolve().parent @@ -22,8 +22,8 @@ sys.path.insert(0, str(PHASE6)) import opprof_phase6_controller as base # noqa: E402 -SCHEMA = "intervention-response-phase-aware-pilot-state-v2" -SESSION_ESTIMATE_H20_HOURS = 1.25 +SCHEMA = "intervention-response-phase-aware-pilot-state-v3" +SESSION_ESTIMATE_H20_HOURS = 1.0 SAFETY_H20_HOURS = 0.20 CLIENT_TIMEOUT_S = 450.0 @@ -55,7 +55,7 @@ def configure(args: argparse.Namespace, manifest: dict[str, Any]) -> None: base.MODEL = args.model base.CLIENT = args.client base.GPU_LIMIT = float(manifest["budget"]["hard_cap_h20_hours"]) - base.MARKER = "intervention-response-phase-aware-v2" + base.MARKER = "intervention-response-phase-aware-v3" base.CELLS = { f"tp4_mns{mns}": {"tp": 4, "mns": int(mns)} for mns in manifest["engine"]["mns_endpoints"] @@ -63,10 +63,15 @@ def configure(args: argparse.Namespace, manifest: dict[str, Any]) -> None: def validate_inputs(args: argparse.Namespace, manifest: dict[str, Any]) -> None: - if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": + if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v3": raise RuntimeError("unexpected phase-aware pilot manifest schema") if manifest.get("status") != "PASS": raise RuntimeError("phase-aware pilot manifest did not pass preflight") + if abs( + float(manifest["budget"]["session_estimate_h20_hours"]) + - SESSION_ESTIMATE_H20_HOURS + ) > 1e-12: + raise RuntimeError("controller and manifest session-cost estimates disagree") failed_invariants = [ name for name, passed in manifest.get("sanity", {}).get("invariants", {}).items() @@ -95,6 +100,13 @@ def validate_inputs(args: argparse.Namespace, manifest: dict[str, Any]) -> None: raise RuntimeError(f"phase-aware pilot input paths missing: {missing}") +def warmup_selection(repetition: Mapping[str, Any]) -> Mapping[str, Any]: + return max( + repetition["selections"].values(), + key=lambda selection: float(selection["offered_req_s_per_gpu"]), + ) + + def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str, Any]: sessions = [] for index, session in enumerate(manifest["sessions"]): @@ -102,13 +114,13 @@ def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str entry = {"cell": cell, "gpus": (0, 1, 2, 3), "port": 8950 + index} repetition = manifest["repetitions"][str(session["replicate"])] session_root = args.run_root / "sessions" / str(session["session"]) - high = repetition["selections"]["high"] + warmup = warmup_selection(repetition) commands = { "server": base.server_command(cell, entry["gpus"], entry["port"]), "warmup": client_command( entry, study=repetition["study"], - anchor=float(high["anchor"]), + anchor=float(warmup["anchor"]), output=session_root / "warmup", warmup=True, ), @@ -145,7 +157,7 @@ def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str } ) return { - "schema": "intervention-response-phase-aware-pilot-dry-run-v2", + "schema": "intervention-response-phase-aware-pilot-dry-run-v3", "status": "PASS", "manifest": str(args.manifest), "run_root": str(args.run_root), @@ -423,14 +435,14 @@ def execute_session( failure: Exception | None = None try: base.wait_ready(entry) - high = repetition["selections"]["high"] + warmup = warmup_selection(repetition) session_state["status"] = "warmup" save_state(state_path, state) run_client( entry=entry, role="warmup", study=repetition["study"], - selection=high, + selection=warmup, output=entry["dir"] / "warmup", state=state, warmup=True, diff --git a/runs/intervention-response-v2/prepare_pilot.py b/runs/intervention-response-v2/prepare_pilot.py index e2d0266..14c5a00 100644 --- a/runs/intervention-response-v2/prepare_pilot.py +++ b/runs/intervention-response-v2/prepare_pilot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Prepare the uncensored 300-second TP4 matched pilot on dash0.""" +"""Prepare the uncensored 300-second TP4 two-load matched pilot on dash0.""" from __future__ import annotations @@ -20,18 +20,18 @@ from aituner.spec import load_study_spec # noqa: E402 from aituner.trace import load_trace_requests, select_requests_for_threshold # noqa: E402 -SCHEMA = "intervention-response-phase-aware-pilot-manifest-v2" +SCHEMA = "intervention-response-phase-aware-pilot-manifest-v3" TP = 4 MNS_ENDPOINTS = (16, 64) REPLAY_TIME_SCALE = 0.5 EXPECTED_DURATION_S = 300.0 SAFETY_DEADLINE_S = 360.0 -LOADS_REQ_S_GPU = {"low": 1.5, "mid": 2.125, "high": 3.125} +LOADS_REQ_S_GPU = {"low": 1.5, "mid": 2.125} REPLICATE_ROLE_PAIRS = (("low1", "high1"), ("low2", "high2"), ("low3", "high3")) LOAD_ORDERS = { - 1: ("low", "mid", "high"), - 2: ("high", "low", "mid"), - 3: ("mid", "high", "low"), + 1: ("low", "mid"), + 2: ("mid", "low"), + 3: ("low", "mid"), } SESSION_ORDER = ( (1, 16), @@ -118,7 +118,7 @@ def private_request_id( *, source_sha256: str, line_number: int, original_id: str ) -> str: payload = f"{source_sha256}:{line_number}:{original_id}".encode() - return f"phase-v2-{hashlib.sha256(payload).hexdigest()}" + return f"phase-v3-{hashlib.sha256(payload).hexdigest()}" def merge_role_traces(sources: tuple[Path, Path], target: Path) -> dict[str, Any]: @@ -159,7 +159,7 @@ def materialize_study( source: Path, target: Path, *, replicate: int, trace_override: Path ) -> Path: payload = json.loads(source.read_text(encoding="utf-8")) - payload["study_id"] = f"phase-aware-telemetry-v2-rep{replicate}" + payload["study_id"] = f"phase-aware-telemetry-v3-rep{replicate}" payload["hardware"]["host_candidates"] = ["dash0"] payload["engine"]["engine_version"] = "0.24.1.dev3+opprof" trace = payload["trace"] @@ -174,7 +174,7 @@ def materialize_study( def build_manifest( - *, base_manifest_path: Path, private_root: Path + *, base_manifest_path: Path, private_root: Path, prior_attempt_state: Path ) -> dict[str, Any]: base = json.loads(base_manifest_path.read_text(encoding="utf-8")) if base.get("schema") != "fidelity-prefix-pilot-manifest-v1": @@ -223,7 +223,7 @@ def build_manifest( selections[level] = record selected_ids_by_level[level] = {request.row_id for request in selected} selection_hashes.append(record["request_id_order_sha256"]) - selected_ids_by_replicate[replicate] = selected_ids_by_level["high"] + selected_ids_by_replicate[replicate] = selected_ids_by_level["mid"] repetitions[str(replicate)] = { "source_roles": list(roles), "merged_trace": merged_traces[str(replicate)], @@ -250,8 +250,8 @@ def build_manifest( invariants = { "three_repetitions": len(repetitions) == 3, "six_sessions": len(sessions) == 6, - "load_levels_three": all( - len(item["selections"]) == 3 for item in repetitions.values() + "load_levels_two": all( + len(item["selections"]) == 2 for item in repetitions.values() ), "selection_hashes_unique": len(selection_hashes) == len(set(selection_hashes)), "selection_sets_disjoint_across_repetitions": all( @@ -267,6 +267,13 @@ def build_manifest( ), } red_flags = [name for name, passed in invariants.items() if not passed] + prior_state = json.loads(prior_attempt_state.read_text(encoding="utf-8")) + if prior_state.get("status") != "failed": + raise ValueError("prior three-load attempt was not recorded as failed") + prior_h20_hours = float(prior_state["gpu_hours_total"]) + incremental_cap_h20_hours = 8.0 - prior_h20_hours + if incremental_cap_h20_hours < 6.5: + raise ValueError("insufficient global H20-hour budget for the two-load rerun") return { "schema": SCHEMA, "status": "PASS" if not red_flags else "STOP", @@ -276,6 +283,10 @@ def build_manifest( "window_id": base["source"]["window_id"], "source_trace": base["source"]["trace"], "source_trace_sha256": base["source"]["trace_sha256"], + "prior_attempt_state": str(prior_attempt_state), + "prior_attempt_state_sha256": sha256_file(prior_attempt_state), + "prior_attempt_h20_hours": prior_h20_hours, + "prior_attempt_failure": prior_state["failures"], }, "engine": { "tp": TP, @@ -294,9 +305,12 @@ def build_manifest( "seconds": [30.0, 75.0, 150.0, 225.0, 300.0], }, "budget": { - "hard_cap_h20_hours": 8.0, - "expected_wall_minutes": [95, 120], - "expected_h20_hours": [6.3, 8.0], + "global_hard_cap_h20_hours": 8.0, + "prior_attempt_h20_hours": prior_h20_hours, + "hard_cap_h20_hours": incremental_cap_h20_hours, + "session_estimate_h20_hours": 1.0, + "expected_wall_minutes": [75, 95], + "expected_h20_hours": [4.8, incremental_cap_h20_hours], }, "sanity": { "red_flags": red_flags, @@ -312,9 +326,12 @@ def main() -> None: parser.add_argument("--base-manifest", type=Path, required=True) parser.add_argument("--private-root", type=Path, required=True) parser.add_argument("--public-manifest", type=Path, required=True) + parser.add_argument("--prior-attempt-state", type=Path, required=True) args = parser.parse_args() manifest = build_manifest( - base_manifest_path=args.base_manifest, private_root=args.private_root + base_manifest_path=args.base_manifest, + private_root=args.private_root, + prior_attempt_state=args.prior_attempt_state, ) atomic_json(args.public_manifest, manifest) print( diff --git a/runs/intervention-response-v2/test_analysis.py b/runs/intervention-response-v2/test_analysis.py index 03ef524..5a0c816 100644 --- a/runs/intervention-response-v2/test_analysis.py +++ b/runs/intervention-response-v2/test_analysis.py @@ -95,8 +95,8 @@ def main() -> None: source_sha256="b" * 64, line_number=1, original_id="1" ) controller = load_controller_module() - assert math.isclose(controller.remaining_projection(6, 0), 7.7) - assert math.isclose(controller.remaining_projection(6, 5), 1.45) + assert math.isclose(controller.remaining_projection(6, 0), 6.2) + assert math.isclose(controller.remaining_projection(6, 5), 1.2) parsed = controller.parser().parse_args( [ "--manifest", @@ -193,7 +193,15 @@ def main() -> None: ] ) assert coverage_gate["red_flags"] == [] - print("phase-aware intervention response v2 analysis: PASS") + assert controller.warmup_selection( + { + "selections": { + "low": {"offered_req_s_per_gpu": 1.5}, + "mid": {"offered_req_s_per_gpu": 2.125}, + } + } + )["offered_req_s_per_gpu"] == 2.125 + print("phase-aware intervention response v3 analysis: PASS") if __name__ == "__main__":