From 9c8570f36b1753d1b77c7cf1b0d16ecc360cad33 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 03:34:49 +0800 Subject: [PATCH] Report held-out active intervention result --- ...active-intervention-v0-results-20260715.md | 106 ++++++++++++ .../trace13-results.json | 151 ++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 docs/active-intervention-v0-results-20260715.md create mode 100644 runs/active-intervention-v0/trace13-results.json diff --git a/docs/active-intervention-v0-results-20260715.md b/docs/active-intervention-v0-results-20260715.md new file mode 100644 index 0000000..d4fc98c --- /dev/null +++ b/docs/active-intervention-v0-results-20260715.md @@ -0,0 +1,106 @@ +# Active intervention v0: held-out trace-13 result + +Date: 2026-07-15 (Asia/Singapore) + +Decision: **close the passive-telemetry treatment-effect route**. The held-out +campaign produced no telemetry-induced action change, measurement reduction, or +GPU-cost reduction. It did show that the engine state contained the correct +action-specific mechanism; the current feature model failed to use it. + +## Headline result + +The outcome-only and telemetry policies both measured the source for 300 +seconds, selected `joint=(MNS64,MBBT8192)`, and produced the same complete +acquisition order. Both reached the exact finite-surface oracle after the +first intervention at a reconstructed all-in lower-bound cost of 2.4284 +H20-hours. Telemetry GPU-cost reduction was therefore exactly 0%, below the +10% confirmation trigger and 20% contribution gate. No actual early-stop +confirmation was launched. + +The complete annotation campaign cost 5.0379 H20-hours, below the 6.0 H20-hour +hard cap. It ran 12 uncensored real-GPU outcomes: four configs, three disjoint +request partitions, and a fresh server per config. + +## Exact response surface + +Median normalized SLO-goodput was: + +| Config | Rep values | Median | +|---|---|---:| +| `MNS32,MBBT4096` source | 0.40091 / 0.39788 / 0.42061 | 0.40091 | +| `MNS64,MBBT4096` | 1.00000 / 0.99970 / 1.00000 | 1.00000 | +| `MNS32,MBBT8192` | 0.44394 / 0.41515 / 0.42606 | 0.42606 | +| `MNS64,MBBT8192` joint | 1.00000 / 1.00000 / 1.00000 | 1.00000 | + +Increasing MNS alone was sufficient and joint was redundant. Increasing MBBT +alone improved the median by only 0.02515, versus 0.59909 for MNS. This is a +strong non-additive action response, not a setting where independently tuning +the knobs and merging their improvements is valid. + +## What the telemetry actually said + +Across 41,086 source scheduler records, 93.12% of steps had waiting work, +85.36% were MNS-exclusive binding, 1.11% were MBBT-exclusive, mean running-slot +utilization was 97.39%, mean token-budget utilization was 15.69%, mean KV usage +was 2.75%, and there were no preemptions. + +The intervention transition agreed with that state: + +| Config | Waiting | MNS-exclusive | MBBT-exclusive | Median goodput | +|---|---:|---:|---:|---:| +| source | 93.12% | 85.36% | 1.11% | 0.40091 | +| MNS only | 5.38% | 0% | 5.38% | 1.00000 | +| MBBT only | 91.19% | 91.09% | 0.04% | 0.42606 | +| joint | 0.89% | 0% | 0.89% | 1.00000 | + +Thus this experiment does **not** support the claim that engine telemetry lacks +tuning information. It rejects the narrower claim that adding passive state +summaries to the current small-data ridge policy converts that information into +lower tuning cost. + +## Why the learned policy failed + +At 300 seconds, the telemetry model predicted joint, MNS, and MBBT effects of +0.35190, 0.26118, and 0.09686. The actual median effects were 0.59909, +0.59909, and 0.02515. Telemetry therefore made the nonexistent joint-over-MNS +gap larger: 0.09072 predicted versus 0 actual; the outcome-only model predicted +0.03188. + +The failure has three concrete causes: + +1. The six training decisions contain no joint intervention. The + `delta_product` feature has no support, so joint ranking is extrapolation. +2. Passive raw summaries do not represent the counterfactual scheduler work + unlocked by each action. Capacity-normalized MNS pressure was visible, but + the model was not structurally required to map it to MNS marginal value. +3. The policy maximizes predicted effect. It does not identify the smallest + epsilon-optimal intervention or price unsupported action complexity. + +## Research implication + +Do not retain the harness or the passive telemetry model as a contribution. +The next defensible route is engine-native, action-conditional counterfactual +instrumentation: at a real scheduling state, shadow-replay the exact scheduler +decision under an MNS relaxation, MBBT relaxation, and their joint relaxation, +then expose the incremental queued work admitted by each action. Real paired +interventions calibrate how those one-step shadow effects map to E2E SLO +goodput. This is distinct from a hand-written cap-to-knob rule and from a +full-system simulator: it reuses the exact live queue, scheduler, and cache +state while simulating only the local decision boundary. + +That route should be evaluated against outcome-only search, the present passive +telemetry model, a cap-hit expert rule, and a full simulator. The paper-level +gate remains at least 20% measured H20-hour reduction to a 2%-oracle config on +task-held-out workloads with at most 2% regret. + +## Sanity + +Surface outcomes: n=12, min=0.39788, max=1.0, distinct=8. Session costs: n=4, +min=1.1702, max=1.3566 H20-hours, distinct=4. Scheduler-record counts: n=4, +min=37,001, max=41,348, distinct=4. All counters and costs were non-negative; +all ratios were in `[0,1]`; request hashes matched; all 12 runs were uncensored; +the controller and four sessions completed; and config outcomes were not all +identical. No red flags were found. + +Machine-readable summary: `runs/active-intervention-v0/trace13-results.json`. +Raw immutable root: `/home/admin/cpfs/wjh/active-intervention-prospective-20260715`. diff --git a/runs/active-intervention-v0/trace13-results.json b/runs/active-intervention-v0/trace13-results.json new file mode 100644 index 0000000..b5c3e01 --- /dev/null +++ b/runs/active-intervention-v0/trace13-results.json @@ -0,0 +1,151 @@ +{ + "schema": "active-intervention-trace13-result-summary-v0", + "status": "STOP_NO_PROSPECTIVE_GPU_COST_SIGNAL", + "provenance": { + "aituner_commit": "39b767e384fc49da53b99ad06a3e2ca1b6ac37d6", + "vllm_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05", + "manifest_sha256": "3bd25ae0ca040729a6351635f14447b3c789d4d86fb0fe4d65940735ad225a78", + "policy_sha256": "4f096d3a5f8c38771e956dfd576dd6cd5d5691286ab258028dbddbe07b20078c", + "decision_sha256": "86b7089151c480e18b5ae6ed65c4e4a3e11159dd0c16d5851312d4d9196d5ca2", + "controller_state_sha256": "d10d3cbe5ce20eacc1392f52e79d16ae23b5e15301123d5e85e48d53ae676cda", + "audit_sha256": "bd95b45e5d4cb93b5ad2f722b7dc96553d64eb8a5d5aae3a82f29c2d015fe3f6", + "remote_root": "/home/admin/cpfs/wjh/active-intervention-prospective-20260715" + }, + "cost": { + "annotation_campaign_h20_hours": 5.0379046784506905, + "hard_cap_h20_hours": 6.0, + "outcome_only_cost_to_acceptable_h20_hours_lower_bound": 2.4284364508172893, + "telemetry_cost_to_acceptable_h20_hours_lower_bound": 2.4284364508172893, + "telemetry_gpu_cost_reduction_fraction": 0.0, + "session_h20_hours": { + "source_mns32_mbbt4096": 1.3566088432735868, + "mns64_mbbt4096": 1.256969277858734, + "mns32_mbbt8192": 1.254111782974667, + "joint_mns64_mbbt8192": 1.1702147743437026 + } + }, + "policy_comparison": { + "outcome_only": { + "measurement_cutoff_s": 300.0, + "selected_action": "joint", + "intervention_order": ["joint", "mns", "mbbt", "noop"] + }, + "telemetry": { + "measurement_cutoff_s": 300.0, + "selected_action": "joint", + "intervention_order": ["joint", "mns", "mbbt", "noop"] + }, + "action_changed": false, + "measurement_changed": false, + "confirmation_trigger": false, + "contribution_gate": false + }, + "surface": { + "source_mns32_mbbt4096": { + "normalized_slo_goodput": [ + 0.40090909090909094, + 0.3978787878787879, + 0.42060606060606065 + ], + "median": 0.40090909090909094 + }, + "mns64_mbbt4096": { + "normalized_slo_goodput": [1.0, 0.9996969696969698, 1.0], + "median": 1.0 + }, + "mns32_mbbt8192": { + "normalized_slo_goodput": [ + 0.44393939393939397, + 0.41515151515151516, + 0.4260606060606061 + ], + "median": 0.42606060606060603 + }, + "joint_mns64_mbbt8192": { + "normalized_slo_goodput": [1.0, 1.0, 1.0], + "median": 1.0 + } + }, + "selected_checkpoint_prediction": { + "actual_median_effect": { + "noop": 0.0, + "mns": 0.5990909090909091, + "mbbt": 0.02515151515151509, + "joint": 0.5990909090909091 + }, + "outcome_only_predicted_effect": { + "noop": 0.0, + "mns": 0.2886250281729182, + "mbbt": 0.17933598309437812, + "joint": 0.3205015384324615 + }, + "telemetry_predicted_effect": { + "noop": 0.0, + "mns": 0.26117798146236215, + "mbbt": 0.09686132563074483, + "joint": 0.35190199346536294 + }, + "actual_joint_minus_mns": 0.0, + "outcome_only_joint_minus_mns": 0.0318765102595433, + "telemetry_joint_minus_mns": 0.09072401200300079 + }, + "engine_mechanism": { + "source_mns32_mbbt4096": { + "scheduler_records": 41086, + "waiting_fraction": 0.9312174463320839, + "mns_exclusive_fraction": 0.8536484447256973, + "mbbt_exclusive_fraction": 0.01114734946210388, + "both_fraction": 0.06642165214428272, + "running_utilization_mean": 0.9738878510928297, + "token_utilization_mean": 0.15694342925509905, + "kv_usage_mean": 0.027507593814715858, + "preemptions": 0 + }, + "mns64_mbbt4096": { + "scheduler_records": 37001, + "waiting_fraction": 0.053809356503878275, + "mns_exclusive_fraction": 0.0, + "mbbt_exclusive_fraction": 0.053809356503878275, + "running_utilization_mean": 0.5410364753655307, + "token_utilization_mean": 0.17425695631536986, + "kv_usage_mean": 0.030549112146415616, + "preemptions": 0 + }, + "mns32_mbbt8192": { + "scheduler_records": 41348, + "waiting_fraction": 0.9119425365192996, + "mns_exclusive_fraction": 0.9108542130211861, + "mbbt_exclusive_fraction": 0.0003627744993711909, + "running_utilization_mean": 0.9652567838831383, + "token_utilization_mean": 0.0779606414231039, + "kv_usage_mean": 0.027355900366122385, + "preemptions": 0 + }, + "joint_mns64_mbbt8192": { + "scheduler_records": 40416, + "waiting_fraction": 0.0088826207442597, + "mns_exclusive_fraction": 0.0, + "mbbt_exclusive_fraction": 0.0088826207442597, + "running_utilization_mean": 0.49403392220902614, + "token_utilization_mean": 0.07978070546782215, + "kv_usage_mean": 0.028003770774946098, + "preemptions": 0 + } + }, + "sanity": { + "surface_outcomes": {"n": 12, "min": 0.3978787878787879, "max": 1.0, "distinct_n": 8}, + "session_h20_hours": {"n": 4, "min": 1.1702147743437026, "max": 1.3566088432735868, "distinct_n": 4}, + "scheduler_records": {"n": 4, "min": 37001, "max": 41348, "distinct_n": 4}, + "invariants": { + "controller_complete": true, + "four_sessions_complete": true, + "twelve_surface_outcomes": true, + "ratios_bounded": true, + "nonnegative_counts_and_costs": true, + "surface_not_all_identical": true, + "request_hashes_match": true, + "no_censored_runs": true + }, + "red_flags": [] + } +}