diff --git a/docs/opprof/oracle-gap-protocol.md b/docs/opprof/oracle-gap-protocol.md index d537f9e..a929088 100644 --- a/docs/opprof/oracle-gap-protocol.md +++ b/docs/opprof/oracle-gap-protocol.md @@ -1,11 +1,29 @@ # Static-policy oracle-gap protocol -Status: **FROZEN WITH A-OG-1 AMENDMENT**. +Status: **FROZEN WITH A-OG-1 AND A-OG-2 AMENDMENTS**. Date frozen: 2026-07-13 (Asia/Singapore). Existing Phase-3 measurements were inspected only to choose the workload pair and rate brackets. They are exploratory calibration data, not primary observations in this protocol. +### A-OG-2 — retry one transport-invalid attempt (after 49 scored trials) + +The first attempt at `P06-C01-r2.2-rep0` produced one local +`ClientOSError` 3.27 ms after admission, with HTTP status 0, no first token, +and no output. The other 399 requests completed successfully, every successful +request produced exactly 512 tokens, the server stayed healthy, and all other +client invariants passed. The controller rejected the attempt before creating +a `score.json`; no TTFT/TPOT result from this attempt was used to choose the +recovery rule. + +This is a measurement-transport failure, not an observed server SLO outcome. +The entire attempt is content-hashed and moved under `invalid-attempts/`, then +the identical logical trial (phase, config, rate, repetition, derived manifest, +seeds, timeline, and SLO) is run once on a fresh C01 server. All 49 existing +scores remain immutable. A second client transport failure in the retry is a +stop condition and makes the experiment inconclusive; it must not be retried +again. No grid, policy, threshold, order, or oracle calculation changes. + ### A-OG-1 — extend the P06 upper bracket (after trial 33) The controller stopped as registered after C00/P06 remained SLO-feasible at diff --git a/scripts/oracle_gap/run_frontier.py b/scripts/oracle_gap/run_frontier.py index 998c59a..92952c9 100644 --- a/scripts/oracle_gap/run_frontier.py +++ b/scripts/oracle_gap/run_frontier.py @@ -20,7 +20,11 @@ from analyze import CONFIGS, PHASES, atomic_json, score_trial, summarize_trials SCHEMA = 1 -AMENDMENT = "A-OG-1" +AMENDMENT = "A-OG-2" +AMENDMENT_REASON = ( + "retry one quarantined P06-C01-r2.2-rep0 transport-invalid attempt " + "without changing its logical trial" +) REMOTE_ROOT = Path("/home/admin/cpfs/wjh/oracle-gap-20260713") RUN_ROOT = REMOTE_ROOT / "runs" STATE = RUN_ROOT / "controller-state.json" @@ -235,6 +239,7 @@ def fingerprint() -> dict[str, Any]: def resume_compatible(old: dict[str, Any], current: dict[str, Any]) -> bool: immutable = ( + "analyzer_sha256", "p5_client_sha256", "p3_client_sha256", "vllm_commit", @@ -245,7 +250,21 @@ def resume_compatible(old: dict[str, Any], current: dict[str, Any]) -> bool: "config_details", "base_rates", ) - return all(old.get(key) == current.get(key) for key in immutable) + if not all(old.get(key) == current.get(key) for key in immutable): + return False + old_extensions = old.get("up_extensions") + current_extensions = current.get("up_extensions") + if old_extensions == current_extensions: + return True + # The only earlier compatible transition was A-OG-1, which appended the + # five registered P06 anchors after the original 2.3-rps list. + if not isinstance(old_extensions, dict) or not isinstance(current_extensions, dict): + return False + return ( + old_extensions.get("P01") == current_extensions.get("P01") + and list(old_extensions.get("P06", [])) + [2.4, 2.5, 2.6, 2.8, 3.0] + == current_extensions.get("P06") + ) def ensure_provenance(current: dict[str, Any]) -> None: @@ -650,10 +669,7 @@ def execute(resume: bool) -> None: { "id": AMENDMENT, "accepted_at": time.time(), - "reason": ( - "C00/P06 feasible through the original 2.3-rps " - "upper extension" - ), + "reason": AMENDMENT_REASON, "completed_trials_before": len( state.get("completed_trials", []) ), diff --git a/tests/test_oracle_gap.py b/tests/test_oracle_gap.py index ebf8975..d661c7a 100644 --- a/tests/test_oracle_gap.py +++ b/tests/test_oracle_gap.py @@ -9,7 +9,7 @@ ORACLE_GAP = Path(__file__).resolve().parents[1] / "scripts/oracle_gap" sys.path.insert(0, str(ORACLE_GAP)) from analyze import score_trial, summarize_trials # noqa: E402 -from run_frontier import UP_EXTENSIONS, resume_compatible # noqa: E402 +from run_frontier import AMENDMENT, UP_EXTENSIONS, resume_compatible # noqa: E402 def _request( @@ -153,14 +153,19 @@ def test_a_og_1_extends_only_mutable_resume_fields() -> None: "manifests": {"P01": "a", "P06": "b"}, "runtime": "runtime", "driver": "driver", + "analyzer_sha256": "analyzer", "config_details": {"C00": {}}, "base_rates": {"P01": [26], "P06": [1.4]}, + "up_extensions": {"P01": [38.0], "P06": [2.1, 2.2, 2.3]}, } amended = { **immutable, "controller_sha256": "new", "repo_commit": "new", - "up_extensions": {"P06": [2.4]}, + "up_extensions": { + "P01": [38.0], + "P06": [2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.8, 3.0], + }, } assert UP_EXTENSIONS["P06"][-5:] == (2.4, 2.5, 2.6, 2.8, 3.0) @@ -168,3 +173,26 @@ def test_a_og_1_extends_only_mutable_resume_fields() -> None: assert not resume_compatible( immutable, {**amended, "manifests": {"P01": "changed"}} ) + + +def test_a_og_2_requires_the_amended_grid_to_stay_fixed() -> None: + old = { + "analyzer_sha256": "analyzer", + "p5_client_sha256": "p5", + "p3_client_sha256": "p3", + "vllm_commit": "vllm", + "model": "model", + "manifests": {"P01": "a", "P06": "b"}, + "runtime": "runtime", + "driver": "driver", + "config_details": {"C00": {}}, + "base_rates": {"P01": [26], "P06": [1.4]}, + "up_extensions": {"P01": [38.0], "P06": [2.1, 2.2, 2.3, 2.4]}, + } + + assert AMENDMENT == "A-OG-2" + assert resume_compatible(old, {**old, "controller_sha256": "new"}) + assert not resume_compatible( + old, + {**old, "up_extensions": {"P01": [38.0], "P06": [2.1, 2.2, 2.3, 2.5]}}, + )