gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enforce phase-stable telemetry pilot gates

Browse Source
This commit is contained in:
Gahow Wang
2026-07-14 17:35:58 +08:00
parent 2afc6eeb8d
commit c0b40af24f
3 changed files with 292 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
runs/intervention-response-v2/analyze_pilot.py
View File

@@ -19,6 +19,7 @@ SCHEMA = "intervention-response-phase-aware-pilot-analysis-v2"
EXPECTED_ACTION_PAIRS = 9
EXPECTED_REPEAT_PAIRS = 12
MIN_EFFICACY_CLASS = 3
MAX_LAYER1_GAP_S = 1.0
def _load_p1():
@@ -54,6 +55,7 @@ def _trial_record(
requests_path: Path,
state: dict[str, float],
outcome: dict[str, float],
telemetry_coverage: dict[str, float],
) -> dict[str, Any]:
return {
"trial_id": str(result_path.relative_to(run_root)),
@@ -74,9 +76,34 @@ def _trial_record(
"early_stopped": bool(result["early_stopped"]),
"state": state,
"outcome": outcome,
"telemetry_coverage": telemetry_coverage,
}
def telemetry_coverage(
records: list[dict[str, Any]], *, start_ns: int, end_ns: int
) -> dict[str, float]:
layer1 = [record for record in records if "step_index" in record]
timestamps = [int(record["submit_mono_ns"]) for record in layer1]
if timestamps != sorted(timestamps):
raise ValueError("Layer-1 timestamps are not monotonic")
selected = [timestamp for timestamp in timestamps if start_ns <= timestamp <= end_ns]
if not selected:
raise ValueError("Layer-1 coverage interval contains no records")
internal_gaps = [
(right - left) / 1e9
for left, right in zip(selected, selected[1:], strict=False)
]
coverage = {
"start_gap_s": (selected[0] - start_ns) / 1e9,
"end_gap_s": (end_ns - selected[-1]) / 1e9,
"max_internal_gap_s": max(internal_gaps, default=0.0),
}
if any(value > MAX_LAYER1_GAP_S for value in coverage.values()):
raise ValueError(f"Layer-1 coverage gap exceeds {MAX_LAYER1_GAP_S}s: {coverage}")
return coverage
def validate_result_against_manifest(
*,
result: Mapping[str, Any],
@@ -147,11 +174,16 @@ def load_interval_trials(
start_ns = int(result["interval"]["start_mono_ns"])
for interval in intervals_s:
start_s, end_s = interval
interval_start_ns = start_ns + int(start_s * 1e9)
interval_end_ns = start_ns + int(end_s * 1e9)
coverage = telemetry_coverage(
stream, start_ns=interval_start_ns, end_ns=interval_end_ns
)
state = P1.V0.flatten_state(
P1.summarize_engine(
stream,
start_ns=start_ns + int(start_s * 1e9),
end_ns=start_ns + int(end_s * 1e9),
start_ns=interval_start_ns,
end_ns=interval_end_ns,
request_count=int(result["selection"]["count"]),
)
)
@@ -166,6 +198,7 @@ def load_interval_trials(
requests_path=requests_path,
state=state,
outcome=outcome,
telemetry_coverage=coverage,
)
)
return by_interval, streams
@@ -221,6 +254,31 @@ def analyze_window(
* float(trial["outcome"]["completed_over_admitted"])
for trial in trials
]
state_vectors = {
tuple(round(float(trial["state"][feature]), 12) for feature in P1.V0.ALL_FEATURES)
for trial in trials
}
per_cell_vectors: dict[str, set[tuple[float, ...]]] = defaultdict(set)
for trial in trials:
per_cell_vectors[str(trial["cell"])].add(
tuple(
round(float(trial["state"][feature]), 12)
for feature in P1.V0.ALL_FEATURES
)
)
ratio_features = (
"prefill_token_fraction",
"kv_usage_mean",
"kv_usage_max",
"graph_none_share",
"graph_full_share",
"graph_padding_fraction",
)
nonnegative_features = tuple(
feature
for feature in P1.V0.ALL_FEATURES
if feature != "kv_usage_end_minus_start"
)
action_metadata = [
{
"group": pair["group"],
@@ -243,6 +301,25 @@ def analyze_window(
for value in pair[key].values()
),
"all_results_uncensored": all(not trial["early_stopped"] for trial in trials),
"state_vectors_not_all_identical": len(state_vectors) > 1,
"per_cell_state_vectors_not_all_identical": all(
len(vectors) > 1 for vectors in per_cell_vectors.values()
),
"ratios_bounded": all(
0.0 <= float(trial["state"][feature]) <= 1.0
for trial in trials
for feature in ratio_features
),
"nonnegative_counters": all(
float(trial["state"][feature]) >= 0.0
for trial in trials
for feature in nonnegative_features
),
"layer1_boundary_and_internal_gaps_bounded": all(
value <= MAX_LAYER1_GAP_S
for trial in trials
for value in trial["telemetry_coverage"].values()
),
}
return {
"start_s": start_s,
@@ -255,6 +332,21 @@ def analyze_window(
"action_pairs": len(actions),
"repeat_pairs": len(repeats),
"actions": action_metadata,
"trial_sanity": [
{
"trial_id": trial["trial_id"],
"cell": trial["cell"],
"level": trial["level"],
"replicate": trial["replicate"],
"admitted_fraction": trial["outcome"]["admitted_fraction"],
"completed_fraction": (
trial["outcome"]["admitted_fraction"]
* trial["outcome"]["completed_over_admitted"]
),
"telemetry_coverage": trial["telemetry_coverage"],
}
for trial in trials
],
"response_statistics": response,
"qualifying_response_features": response_qualifying,
"efficacy": {
@@ -290,6 +382,20 @@ def stable_adjacent_features(windows: list[dict[str, Any]]) -> dict[str, list[st
return result
def stable_adjacent_efficacy_features(
windows: list[dict[str, Any]],
) -> dict[str, list[str]]:
eligible = [window for window in windows if window["end_fraction"] >= 0.25]
result = {}
for left, right in zip(eligible, eligible[1:], strict=False):
key = f"{left['end_fraction']:.2f}->{right['end_fraction']:.2f}"
result[key] = sorted(
set(left["efficacy"]["telemetry_qualifying_features"])
& set(right["efficacy"]["telemetry_qualifying_features"])
)
return result
def consistent_load_regimes(
windows: list[dict[str, Any]], stable: dict[str, list[str]]
) -> dict[str, Any]:
@@ -322,6 +428,96 @@ def consistent_load_regimes(
return result
def mechanism_gate(
stable: Mapping[str, list[str]], load_consistency: Mapping[str, Mapping[str, Any]]
) -> dict[str, Any]:
by_transition = {}
for transition, features in stable.items():
qualifying = sorted(
feature
for feature in features
if load_consistency[f"{transition}:{feature}"]["passes_two_regimes"]
)
by_transition[transition] = qualifying
passing_transitions = sorted(
transition
for transition, features in by_transition.items()
if len(features) >= 2
)
return {
"minimum_features": 2,
"by_transition": by_transition,
"passing_transitions": passing_transitions,
"passes": bool(passing_transitions),
}
def controller_gate(
run_root: Path, manifest: Mapping[str, Any]
) -> dict[str, Any]:
path = run_root / "controller-state.json"
state = json.loads(path.read_text(encoding="utf-8"))
expected_sessions = {str(session["session"]) for session in manifest["sessions"]}
actual_sessions = set(state.get("sessions", {}))
session_invariants = [
passed
for session in state.get("sessions", {}).values()
for passed in session.get("validation", {}).get("invariants", {}).values()
]
invariants = {
"controller_complete": state.get("status") == "complete",
"completed_session_count": int(state.get("completed_sessions", -1))
== len(expected_sessions),
"exact_session_set": actual_sessions == expected_sessions,
"all_sessions_complete": all(
session.get("status") == "complete"
for session in state.get("sessions", {}).values()
),
"no_controller_failures": not state.get("failures"),
"under_h20_hour_cap": float(state.get("gpu_hours_total", math.inf))
<= float(manifest["budget"]["hard_cap_h20_hours"]),
"all_stream_validation_invariants_pass": bool(session_invariants)
and all(session_invariants),
}
return {
"path": str(path.resolve()),
"sha256": sha256_file(path),
"gpu_hours_total": float(state.get("gpu_hours_total", math.nan)),
"invariants": invariants,
"red_flags": [name for name, passed in invariants.items() if not passed],
}
def cumulative_coverage_gate(windows: list[dict[str, Any]]) -> dict[str, Any]:
trajectories: dict[str, list[tuple[float, float]]] = defaultdict(list)
trial_sets = []
for window in windows:
trial_sets.append({str(trial["trial_id"]) for trial in window["trial_sanity"]})
for trial in window["trial_sanity"]:
trajectories[str(trial["trial_id"])].append(
(
float(trial["admitted_fraction"]),
float(trial["completed_fraction"]),
)
)
invariants = {
"same_trials_at_every_checkpoint": bool(trial_sets)
and all(trial_set == trial_sets[0] for trial_set in trial_sets[1:]),
"admitted_fraction_monotonic": all(
all(right[0] + 1e-12 >= left[0] for left, right in zip(values, values[1:]))
for values in trajectories.values()
),
"completed_fraction_monotonic": all(
all(right[1] + 1e-12 >= left[1] for left, right in zip(values, values[1:]))
for values in trajectories.values()
),
}
return {
"invariants": invariants,
"red_flags": [name for name, passed in invariants.items() if not passed],
}
def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2":
@@ -354,33 +550,34 @@ def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str
]
stable = stable_adjacent_features(cumulative)
load_consistency = consistent_load_regimes(cumulative, stable)
mechanism = mechanism_gate(stable, load_consistency)
mechanism_features = sorted(
{
key.split(":", 1)[1]
for key, item in load_consistency.items()
if item["passes_two_regimes"]
feature
for transition in mechanism["passing_transitions"]
for feature in mechanism["by_transition"][transition]
}
)
full = cumulative[-1]
efficacy_features = sorted(
set.intersection(
*(
set(window["efficacy"]["telemetry_qualifying_features"])
for window in cumulative
if window["end_fraction"] >= 0.25
)
)
efficacy_stable = stable_adjacent_efficacy_features(cumulative)
efficacy_candidates = sorted(
{feature for features in efficacy_stable.values() for feature in features}
)
efficacy_features = sorted(set(efficacy_candidates) & set(mechanism_features))
controller = controller_gate(run_root, manifest)
coverage = cumulative_coverage_gate(cumulative)
red_flags = sorted(
{
flag
for window in [*cumulative, *quarter_blocks]
for flag in window["sanity"]["red_flags"]
}
| set(controller["red_flags"])
| set(coverage["red_flags"])
)
if red_flags:
decision = "STOP_DATA_INVALID"
elif not mechanism_features:
elif not mechanism["passes"]:
decision = "STOP_NO_PHASE_STABLE_RESPONSE"
elif not full["efficacy"]["label_balance_sufficient"]:
decision = "MECHANISM_ONLY_NO_LABEL_BALANCE"
@@ -394,9 +591,12 @@ def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str
"decision": decision,
"claim_boundary": "Development mechanism pilot; not a held-out paper claim.",
"mechanism_features": mechanism_features,
"mechanism_gate": mechanism,
"stable_adjacent_features": stable,
"load_consistency": load_consistency,
"stable_incremental_efficacy_features": efficacy_features,
"stable_incremental_efficacy_candidates": efficacy_candidates,
"stable_adjacent_efficacy_features": efficacy_stable,
"cumulative": cumulative,
"quarter_blocks": quarter_blocks,
"provenance": {
@@ -410,6 +610,8 @@ def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str
"sanity": {
"streams": len(streams),
"stream_bytes": numeric(item["bytes"] for item in streams),
"controller": controller,
"cumulative_coverage": coverage,
"red_flags": red_flags,
},
}