521 lines
19 KiB
Python
521 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Audit whether a controlled knob change produces identifiable telemetry deltas.
|
|
|
|
This is a development-only feasibility audit. It compares adjacent MNS
|
|
interventions at an identical TP, offered-load anchor, and request sequence
|
|
against same-config primary/confirmation repeat noise. It does not claim that
|
|
the observed response is causal or that it improves an end-to-end tuner.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
from statistics import median
|
|
from typing import Any, Iterable, Mapping
|
|
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
COMMON_STATE_DIR = HERE.parent / "telemetry-residual"
|
|
sys.path.insert(0, str(COMMON_STATE_DIR))
|
|
|
|
from common_state import load_jsonl, summarize_engine # noqa: E402
|
|
|
|
|
|
SCHEMA = "intervention-response-audit-v0"
|
|
HORIZONS_S = (5.0, 10.0)
|
|
GATE_FEATURES = (
|
|
"scheduler_steps_per_s",
|
|
"decode_batch_size.mean",
|
|
"prefill_token_fraction",
|
|
"queue_waiting_mean",
|
|
"queue_running_mean",
|
|
"kv_usage_mean",
|
|
"graph_padding_fraction",
|
|
)
|
|
ALL_FEATURES = (
|
|
"scheduler_steps_per_s",
|
|
"batch_size.mean",
|
|
"batch_tokens.mean",
|
|
"decode_batch_size.mean",
|
|
"prefill_token_fraction",
|
|
"queue_waiting_mean",
|
|
"queue_running_mean",
|
|
"preemptions",
|
|
"kv_usage_mean",
|
|
"kv_usage_max",
|
|
"kv_usage_end_minus_start",
|
|
"graph_none_share",
|
|
"graph_full_share",
|
|
"graph_padding_fraction",
|
|
)
|
|
EXPECTED_ACTION_PAIRS = 17
|
|
MIN_REPEAT_PAIRS = 20
|
|
MIN_STABLE_FEATURES = 2
|
|
MIN_SIGN_CONSISTENCY = 0.75
|
|
MIN_EFFECT_TO_NOISE = 2.0
|
|
MIN_ABOVE_NOISE_P95_FRACTION = 0.5
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for chunk in iter(lambda: source.read(1 << 20), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def numeric(values: Iterable[float]) -> dict[str, Any]:
|
|
finite = [float(value) for value in values]
|
|
if not finite:
|
|
raise ValueError("numeric summary requires at least one value")
|
|
if any(not math.isfinite(value) for value in finite):
|
|
raise ValueError("numeric summary received a non-finite value")
|
|
return {
|
|
"n": len(finite),
|
|
"min": min(finite),
|
|
"max": max(finite),
|
|
"distinct_n": len(set(finite)),
|
|
}
|
|
|
|
|
|
def quantile(values: Iterable[float], probability: float) -> float:
|
|
ordered = sorted(float(value) for value in values)
|
|
if not ordered:
|
|
raise ValueError("quantile requires at least one value")
|
|
if not 0.0 <= probability <= 1.0:
|
|
raise ValueError("quantile probability must be in [0, 1]")
|
|
position = probability * (len(ordered) - 1)
|
|
lower = math.floor(position)
|
|
upper = math.ceil(position)
|
|
if lower == upper:
|
|
return ordered[lower]
|
|
weight = position - lower
|
|
return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
|
|
|
|
|
|
def flatten_state(summary: Mapping[str, Any]) -> dict[str, float]:
|
|
common = summary["common"]
|
|
engine = summary["engine_only"]
|
|
state = {
|
|
"scheduler_steps_per_s": float(common["scheduler_steps_per_s"]),
|
|
"batch_size.mean": float(common["batch_size"]["mean"]),
|
|
"batch_tokens.mean": float(common["batch_tokens"]["mean"]),
|
|
"decode_batch_size.mean": float(common["decode_batch_size"]["mean"]),
|
|
"prefill_token_fraction": float(common["prefill_token_fraction"]),
|
|
"queue_waiting_mean": float(common["queue_waiting_mean"]),
|
|
"queue_running_mean": float(common["queue_running_mean"]),
|
|
"preemptions": float(common["preemptions"]),
|
|
"kv_usage_mean": float(engine["kv_usage_mean"]),
|
|
"kv_usage_max": float(engine["kv_usage_max"]),
|
|
"kv_usage_end_minus_start": float(engine["kv_usage_end_minus_start"]),
|
|
"graph_none_share": float(engine["graph_none_share"]),
|
|
"graph_full_share": float(engine["graph_full_share"]),
|
|
"graph_padding_fraction": float(engine["graph_padding_fraction"]),
|
|
}
|
|
if set(state) != set(ALL_FEATURES):
|
|
raise ValueError("flattened state does not match the frozen feature set")
|
|
if any(not math.isfinite(value) for value in state.values()):
|
|
raise ValueError("flattened state contains a non-finite value")
|
|
return state
|
|
|
|
|
|
def _trial_role(path: Path) -> str:
|
|
return "confirmation" if path.parent.name.startswith("confirm-") else "primary"
|
|
|
|
|
|
def load_trials(
|
|
raw_root: Path,
|
|
*,
|
|
horizons_s: tuple[float, ...] = HORIZONS_S,
|
|
) -> tuple[dict[float, list[dict[str, Any]]], list[dict[str, Any]]]:
|
|
by_horizon = {horizon: [] for horizon in horizons_s}
|
|
stream_provenance = []
|
|
for cell_dir in sorted(path for path in raw_root.iterdir() if path.is_dir()):
|
|
streams = sorted((cell_dir / "opprof").glob("*.jsonl"))
|
|
if len(streams) != 1:
|
|
raise ValueError(f"{cell_dir}: expected exactly one Layer-1 stream")
|
|
stream = streams[0]
|
|
records = load_jsonl(stream)
|
|
stream_provenance.append(
|
|
{
|
|
"path": str(stream),
|
|
"sha256": sha256_file(stream),
|
|
"bytes": stream.stat().st_size,
|
|
}
|
|
)
|
|
result_paths = sorted(cell_dir.glob("anchor-*/result.json"))
|
|
result_paths.extend(sorted(cell_dir.glob("confirm-*-anchor-*/result.json")))
|
|
for result_path in result_paths:
|
|
result = json.loads(result_path.read_text(encoding="utf-8"))
|
|
start_ns = int(result["interval"]["start_mono_ns"])
|
|
elapsed_s = float(result["interval"]["elapsed_s"])
|
|
for horizon_s in horizons_s:
|
|
if elapsed_s < horizon_s:
|
|
raise ValueError(
|
|
f"{result_path}: elapsed {elapsed_s} is shorter than {horizon_s}s"
|
|
)
|
|
state = flatten_state(
|
|
summarize_engine(
|
|
records,
|
|
start_ns=start_ns,
|
|
end_ns=start_ns + int(horizon_s * 1e9),
|
|
request_count=int(result["selection"]["count"]),
|
|
)
|
|
)
|
|
by_horizon[horizon_s].append(
|
|
{
|
|
"trial_id": str(result_path.relative_to(raw_root)),
|
|
"result_sha256": sha256_file(result_path),
|
|
"role": _trial_role(result_path),
|
|
"cell": str(result["cell"]),
|
|
"study_sha256": str(result["study_sha256"]),
|
|
"tp": int(result["tp"]),
|
|
"mns": int(result["mns"]),
|
|
"anchor": float(result["anchor"]),
|
|
"request_hash": str(
|
|
result["selection"]["request_id_order_sha256"]
|
|
),
|
|
"request_count": int(result["selection"]["count"]),
|
|
"early_stopped": bool(result["early_stopped"]),
|
|
"full_pass_rate": float(result["pass_rate"]),
|
|
"full_feasible": bool(result["feasible"]),
|
|
"state": state,
|
|
}
|
|
)
|
|
return by_horizon, stream_provenance
|
|
|
|
|
|
def _group_key(trial: Mapping[str, Any]) -> tuple[Any, ...]:
|
|
return (
|
|
trial["study_sha256"],
|
|
trial["tp"],
|
|
trial["anchor"],
|
|
trial["request_hash"],
|
|
)
|
|
|
|
|
|
def _delta(source: Mapping[str, Any], target: Mapping[str, Any]) -> dict[str, float]:
|
|
return {
|
|
feature: float(target["state"][feature]) - float(source["state"][feature])
|
|
for feature in ALL_FEATURES
|
|
}
|
|
|
|
|
|
def _pair(source: Mapping[str, Any], target: Mapping[str, Any], kind: str) -> dict[str, Any]:
|
|
if _group_key(source) != _group_key(target):
|
|
raise ValueError("pair endpoints do not share workload identity")
|
|
return {
|
|
"kind": kind,
|
|
"group": {
|
|
"study_sha256": source["study_sha256"],
|
|
"tp": source["tp"],
|
|
"anchor": source["anchor"],
|
|
"request_hash": source["request_hash"],
|
|
},
|
|
"source": {
|
|
"trial_id": source["trial_id"],
|
|
"cell": source["cell"],
|
|
"mns": source["mns"],
|
|
"early_stopped": source["early_stopped"],
|
|
"full_pass_rate": source["full_pass_rate"],
|
|
"full_feasible": source["full_feasible"],
|
|
},
|
|
"target": {
|
|
"trial_id": target["trial_id"],
|
|
"cell": target["cell"],
|
|
"mns": target["mns"],
|
|
"early_stopped": target["early_stopped"],
|
|
"full_pass_rate": target["full_pass_rate"],
|
|
"full_feasible": target["full_feasible"],
|
|
},
|
|
"delta_state": _delta(source, target),
|
|
"descriptive_full_outcome": {
|
|
"delta_pass_rate": target["full_pass_rate"] - source["full_pass_rate"],
|
|
"feasibility_transition": (
|
|
f"{str(source['full_feasible']).lower()}->"
|
|
f"{str(target['full_feasible']).lower()}"
|
|
),
|
|
},
|
|
}
|
|
|
|
|
|
def build_pairs(
|
|
trials: list[dict[str, Any]],
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
primary_groups: dict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list)
|
|
primary_by_cell_anchor: dict[tuple[Any, ...], dict[str, Any]] = {}
|
|
confirmations = []
|
|
for trial in trials:
|
|
if trial["role"] == "primary":
|
|
primary_groups[_group_key(trial)].append(trial)
|
|
primary_by_cell_anchor[
|
|
(trial["cell"], trial["anchor"], trial["request_hash"])
|
|
] = trial
|
|
else:
|
|
confirmations.append(trial)
|
|
|
|
actions = []
|
|
for group in primary_groups.values():
|
|
ordered = sorted(group, key=lambda item: item["mns"])
|
|
for source, target in zip(ordered, ordered[1:], strict=False):
|
|
if target["mns"] == source["mns"] * 2:
|
|
actions.append(_pair(source, target, "mns_increase"))
|
|
|
|
repeats = []
|
|
for confirmation in confirmations:
|
|
key = (
|
|
confirmation["cell"],
|
|
confirmation["anchor"],
|
|
confirmation["request_hash"],
|
|
)
|
|
primary = primary_by_cell_anchor.get(key)
|
|
if primary is None:
|
|
raise ValueError(f"{confirmation['trial_id']}: missing matched primary")
|
|
if primary["mns"] != confirmation["mns"]:
|
|
raise ValueError("repeat endpoints changed MNS")
|
|
repeats.append(_pair(primary, confirmation, "same_config_repeat"))
|
|
return actions, repeats
|
|
|
|
|
|
def response_statistics(
|
|
actions: list[dict[str, Any]],
|
|
repeats: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
statistics = {}
|
|
for feature in ALL_FEATURES:
|
|
action = [float(pair["delta_state"][feature]) for pair in actions]
|
|
noise = [float(pair["delta_state"][feature]) for pair in repeats]
|
|
action_abs = [abs(value) for value in action]
|
|
noise_abs = [abs(value) for value in noise]
|
|
positive = sum(value > 1e-12 for value in action)
|
|
negative = sum(value < -1e-12 for value in action)
|
|
zero = len(action) - positive - negative
|
|
nonzero = positive + negative
|
|
sign_consistency = max(positive, negative) / nonzero if nonzero else 0.0
|
|
action_median = median(action_abs)
|
|
noise_median = median(noise_abs)
|
|
noise_p95 = quantile(noise_abs, 0.95)
|
|
effect_to_noise = (
|
|
action_median / noise_median
|
|
if noise_median > 0
|
|
else (math.inf if action_median > 0 else 0.0)
|
|
)
|
|
above_noise = sum(value > noise_p95 for value in action_abs) / len(action_abs)
|
|
qualifies = (
|
|
feature in GATE_FEATURES
|
|
and sign_consistency >= MIN_SIGN_CONSISTENCY
|
|
and effect_to_noise >= MIN_EFFECT_TO_NOISE
|
|
and above_noise >= MIN_ABOVE_NOISE_P95_FRACTION
|
|
)
|
|
statistics[feature] = {
|
|
"action_delta": numeric(action),
|
|
"repeat_delta": numeric(noise),
|
|
"action_abs_median": action_median,
|
|
"repeat_abs_median": noise_median,
|
|
"repeat_abs_p95": noise_p95,
|
|
"effect_to_repeat_median": (
|
|
effect_to_noise if math.isfinite(effect_to_noise) else None
|
|
),
|
|
"effect_to_repeat_median_is_infinite": math.isinf(effect_to_noise),
|
|
"action_signs": {
|
|
"positive": positive,
|
|
"negative": negative,
|
|
"zero": zero,
|
|
"consistency": sign_consistency,
|
|
},
|
|
"action_above_repeat_p95_fraction": above_noise,
|
|
"gate_feature": feature in GATE_FEATURES,
|
|
"qualifies": qualifies,
|
|
}
|
|
return statistics
|
|
|
|
|
|
def analyze_horizon(trials: list[dict[str, Any]], horizon_s: float) -> dict[str, Any]:
|
|
actions, repeats = build_pairs(trials)
|
|
feature_statistics = response_statistics(actions, repeats)
|
|
qualifying = sorted(
|
|
feature for feature, item in feature_statistics.items() if item["qualifies"]
|
|
)
|
|
all_values = [
|
|
value
|
|
for trial in trials
|
|
for value in trial["state"].values()
|
|
]
|
|
action_vectors = {
|
|
tuple(round(float(pair["delta_state"][feature]), 12) for feature in ALL_FEATURES)
|
|
for pair in actions
|
|
}
|
|
pair_invariants = {
|
|
"expected_action_pair_count": len(actions) == EXPECTED_ACTION_PAIRS,
|
|
"sufficient_repeat_pair_count": len(repeats) >= MIN_REPEAT_PAIRS,
|
|
"all_pair_hashes_match": all(
|
|
pair["group"]["request_hash"] for pair in [*actions, *repeats]
|
|
),
|
|
"all_values_finite": all(math.isfinite(value) for value in all_values),
|
|
"state_vectors_not_all_identical": len(action_vectors) > 1,
|
|
"ratios_bounded": all(
|
|
0.0 <= trial["state"][feature] <= 1.0
|
|
for trial in trials
|
|
for feature in (
|
|
"prefill_token_fraction",
|
|
"kv_usage_mean",
|
|
"kv_usage_max",
|
|
"graph_none_share",
|
|
"graph_full_share",
|
|
"graph_padding_fraction",
|
|
)
|
|
),
|
|
"nonnegative_counters": all(
|
|
trial["state"][feature] >= 0.0
|
|
for trial in trials
|
|
for feature in (
|
|
"scheduler_steps_per_s",
|
|
"batch_size.mean",
|
|
"batch_tokens.mean",
|
|
"decode_batch_size.mean",
|
|
"queue_waiting_mean",
|
|
"queue_running_mean",
|
|
"preemptions",
|
|
)
|
|
),
|
|
}
|
|
red_flags = [name for name, passed in pair_invariants.items() if not passed]
|
|
pass_deltas = [
|
|
pair["descriptive_full_outcome"]["delta_pass_rate"] for pair in actions
|
|
]
|
|
transitions = defaultdict(int)
|
|
for pair in actions:
|
|
transitions[pair["descriptive_full_outcome"]["feasibility_transition"]] += 1
|
|
return {
|
|
"horizon_s": horizon_s,
|
|
"actions": actions,
|
|
"repeats": repeats,
|
|
"feature_statistics": feature_statistics,
|
|
"qualifying_features": qualifying,
|
|
"descriptive_full_outcome": {
|
|
"delta_pass_rate": numeric(pass_deltas),
|
|
"positive": sum(value > 1e-12 for value in pass_deltas),
|
|
"negative": sum(value < -1e-12 for value in pass_deltas),
|
|
"zero": sum(abs(value) <= 1e-12 for value in pass_deltas),
|
|
"feasibility_transitions": dict(sorted(transitions.items())),
|
|
"limitation": (
|
|
"Full outcomes may use different elapsed durations when a trial "
|
|
"early-stopped; they are descriptive and are not a gate input."
|
|
),
|
|
},
|
|
"sanity": {
|
|
"trials": len(trials),
|
|
"action_pairs": len(actions),
|
|
"repeat_pairs": len(repeats),
|
|
"distinct_action_vectors": len(action_vectors),
|
|
"invariants": pair_invariants,
|
|
"red_flags": red_flags,
|
|
},
|
|
}
|
|
|
|
|
|
def audit(
|
|
*,
|
|
metrics_path: Path,
|
|
raw_root: Path,
|
|
output_path: Path,
|
|
) -> dict[str, Any]:
|
|
trials_by_horizon, streams = load_trials(raw_root)
|
|
horizons = {
|
|
str(int(horizon)): analyze_horizon(trials, horizon)
|
|
for horizon, trials in sorted(trials_by_horizon.items())
|
|
}
|
|
red_flags = sorted(
|
|
{
|
|
red_flag
|
|
for horizon in horizons.values()
|
|
for red_flag in horizon["sanity"]["red_flags"]
|
|
}
|
|
)
|
|
stable_features = sorted(
|
|
set.intersection(
|
|
*(set(horizon["qualifying_features"]) for horizon in horizons.values())
|
|
)
|
|
)
|
|
if red_flags:
|
|
decision = "STOP_DATA_INVALID"
|
|
elif len(stable_features) < MIN_STABLE_FEATURES:
|
|
decision = "STOP_NO_IDENTIFIABLE_RESPONSE"
|
|
else:
|
|
decision = "OPEN_MATCHED_PILOT"
|
|
payload = {
|
|
"schema": SCHEMA,
|
|
"status": "COMPLETE",
|
|
"decision": decision,
|
|
"claim_boundary": (
|
|
"Development-only identifiability gate. Passing opens a controlled "
|
|
"real-GPU pilot; it does not establish tuning benefit or causality."
|
|
),
|
|
"frozen_gate": {
|
|
"horizons_s": list(HORIZONS_S),
|
|
"expected_action_pairs": EXPECTED_ACTION_PAIRS,
|
|
"minimum_repeat_pairs": MIN_REPEAT_PAIRS,
|
|
"minimum_stable_features": MIN_STABLE_FEATURES,
|
|
"minimum_sign_consistency": MIN_SIGN_CONSISTENCY,
|
|
"minimum_effect_to_repeat_median": MIN_EFFECT_TO_NOISE,
|
|
"minimum_action_above_repeat_p95_fraction": (
|
|
MIN_ABOVE_NOISE_P95_FRACTION
|
|
),
|
|
"gate_features": list(GATE_FEATURES),
|
|
},
|
|
"stable_qualifying_features": stable_features,
|
|
"horizons": horizons,
|
|
"provenance": {
|
|
"analysis_script": str(Path(__file__).resolve()),
|
|
"analysis_script_sha256": sha256_file(Path(__file__).resolve()),
|
|
"phase6_metrics": str(metrics_path.resolve()),
|
|
"phase6_metrics_sha256": sha256_file(metrics_path),
|
|
"raw_root": str(raw_root.resolve()),
|
|
"streams": streams,
|
|
},
|
|
"sanity": {
|
|
"stream_count": len(streams),
|
|
"stream_bytes": numeric(item["bytes"] for item in streams),
|
|
"red_flags": red_flags,
|
|
},
|
|
}
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
|
return payload
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--metrics", type=Path, required=True)
|
|
parser.add_argument("--raw-root", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
payload = audit(
|
|
metrics_path=args.metrics,
|
|
raw_root=args.raw_root,
|
|
output_path=args.output,
|
|
)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"decision": payload["decision"],
|
|
"stable_qualifying_features": payload[
|
|
"stable_qualifying_features"
|
|
],
|
|
"sanity": payload["sanity"],
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|