diff --git a/docs/intervention-response-v2-phase-aware-protocol-20260714.md b/docs/intervention-response-v2-phase-aware-protocol-20260714.md new file mode 100644 index 0000000..d87025b --- /dev/null +++ b/docs/intervention-response-v2-phase-aware-protocol-20260714.md @@ -0,0 +1,84 @@ +# Phase-aware telemetry intervention-response v2 protocol + +Status: **FROZEN BEFORE THE SYSTEMATIC 10% DECILE AUDIT**. + +Date: 2026-07-14 (Asia/Singapore). + +## Correction to v0/v1 + +The 5/10-second analyses tested an ultra-early verifier. They did not test +whether telemetry observed after the engine has developed queue, batch, and KV +state can guide tuning. The P1 replay lasts 60 seconds after time scaling, and +the 5/10-second prefixes contain only a small fraction of its requests. + +V2 therefore replaces absolute cutoffs with replay phase. The old audits and +their negative decisions remain immutable, but their claim is narrowed to the +first 5/10 seconds. + +## Historical corrective audit + +The historical audit is development-only and cannot become confirmatory after +the horizon concern was observed. + +- Infer each trial's intended replay duration as selected requests divided by + offered requests per second. All trials must agree. +- Find every complete 10% replay decile supported by every trial. Analyze all + such deciles; selecting only the best horizon is forbidden. +- At each decile report both: + - cumulative state from replay start to the checkpoint; and + - the non-overlapping 10%-wide state block ending at the checkpoint. +- Report admitted/completed request coverage, response-versus-repeat statistics, + telemetry versus external-outcome efficacy, and per-feature trajectory drift. +- Reuse the frozen v1 action and repeat pairs and the frozen response and + incremental-efficacy thresholds. These thresholds are descriptive in V2; + passing one post-hoc horizon does not open a contribution claim. + +If early stopping prevents complete observation of the replay phases, the +historical decision is `REQUIRES_UNCENSORED_PHASE_AWARE_PILOT`, independent of +which early decile looks best. + +## Uncensored matched pilot + +The pilot is a mechanism gate, not paper evidence. + +- Hardware/engine/model: solo placement on dash0, 4 NVIDIA H20 GPUs, patched + vLLM `0.24.1.dev3+opprof`, Qwen3-30B-A3B, fixed `TP=4`. +- Action: `MNS 16 -> 64`; topology, model, engine build, workload, arrival + sequence, offered load, and all other settings remain fixed. +- Workload: `chat_w20260312_1000` at replay-time scale `0.5`, hence 300 seconds. +- Offered loads per GPU: `1.5`, `2.125`, and `3.125` requests/s. These supply a + low control and the two already-observed P1 pressure regimes. +- Repetitions: three disjoint session bands, exact request sequence matched + across action endpoints. Endpoint order alternates `A/B`, `B/A`, `A/B`; + load order is counter-rotated across repetitions. +- A fresh server receives the accepted long-request warm-up and a bounded + burn-in before each measured session. +- SLO-unrecoverable early stop is disabled. Every run must observe the full + 300-second arrival window; a separate 360-second safety deadline may mark a + run invalid but cannot manufacture a full-run label. +- Cumulative checkpoints: 10%, 25%, 50%, 75%, and 100%, or 30/75/150/225/300 + seconds. Quarter blocks are analyzed separately from cumulative means. +- Placement is serialized. Co-location remains forbidden because Phase 6 + observed material co-location-induced outcome shifts. +- Hard cap: 8 H20-hours including startup, warm-up, burn-in, invalid attempts, + and cleanup. + +## Gates + +Data validity requires complete 300-second Layer-1 coverage, zero dropped +records, exact request/arrival/length hashes across action endpoints, monotonic +timestamps, full request accounting, idle GPUs before and after each session, +and no co-resident GPU process. + +Mechanism evidence requires at least two telemetry features whose matched action +response exceeds same-config repeat noise at two consecutive checkpoints under +the unchanged v1 response thresholds. The same features must have a consistent +direction in at least two of the three load regimes. + +Decision evidence additionally requires both action-efficacy classes and an +instrumentation feature that reaches leave-one-repetition-out balanced accuracy +at least 0.75 and exceeds the best external prefix outcome by at least 0.15. +Without label balance the pilot can adjudicate mechanism evidence only. + +No H20 run is launched if the local analyzer/tests, manifest preflight, GPU +probe, command dry-run, projected cost, or cleanup plan fails. diff --git a/runs/intervention-response-v2/analyze_existing.py b/runs/intervention-response-v2/analyze_existing.py new file mode 100644 index 0000000..3d3fe85 --- /dev/null +++ b/runs/intervention-response-v2/analyze_existing.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +"""Audit telemetry responses over every uncensored replay decile. + +This corrective analysis keeps the frozen P1 pairs and thresholds, but replaces +the absolute 5/10-second cutoff with cumulative and non-overlapping 10%-of-trace +windows. It deliberately reports every common decile instead of selecting the +best-looking horizon. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +from pathlib import Path +from statistics import median +from typing import Any, Iterable, Mapping + + +HERE = Path(__file__).resolve().parent +P1_PATH = HERE.parent / "intervention-response-v0" / "analyze_p1.py" +SCHEMA = "intervention-response-phase-aware-existing-v2" +DECILE_FRACTION = 0.1 +MAX_DECILES = 10 + + +def _load_p1(): + spec = importlib.util.spec_from_file_location("intervention_response_p1", P1_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +P1 = _load_p1() + + +def numeric(values: Iterable[float | int]) -> dict[str, Any]: + finite = [float(value) for value in values] + result = P1.V0.numeric(finite) + result["median"] = median(finite) + return result + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def trial_directories(run_root: Path) -> list[Path]: + result = [] + for cell in sorted((run_root / "cells").iterdir()): + if not cell.is_dir(): + continue + for candidate in sorted(cell.iterdir()): + if candidate.is_dir() and P1.RUN_PATTERN.match(candidate.name): + result.append(candidate) + if not result: + raise ValueError("P1 run root contains no measured trial directories") + return result + + +def load_metadata(run_root: Path) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + metadata = [] + streams = [] + for cell in sorted((run_root / "cells").iterdir()): + if not cell.is_dir(): + continue + stream_paths = sorted((cell / "opprof").glob("*.jsonl")) + if len(stream_paths) != 1: + raise ValueError(f"{cell}: expected one Layer-1 stream") + stream_path = stream_paths[0] + streams.append( + { + "cell": cell.name, + "path": str(stream_path.resolve()), + "sha256": sha256_file(stream_path), + "bytes": stream_path.stat().st_size, + } + ) + for run_dir in trial_directories(run_root): + match = P1.RUN_PATTERN.match(run_dir.name) + assert match is not None + level, replicate_text = match.groups() + result_path = run_dir / "result.json" + requests_path = run_dir / "requests.jsonl" + result = json.loads(result_path.read_text(encoding="utf-8")) + selected = int(result["selection"]["count"]) + offered = float(result["selection"]["offered_req_s"]) + if selected <= 0 or offered <= 0.0: + raise ValueError(f"{result_path}: invalid selected count or offered rate") + metadata.append( + { + "trial_id": str(result_path.relative_to(run_root)), + "cell": str(result["cell"]), + "tp": int(result["tp"]), + "mns": int(result["mns"]), + "level": level, + "replicate": int(replicate_text), + "elapsed_s": float(result["interval"]["elapsed_s"]), + "trace_duration_s": round(selected / offered, 9), + "early_stopped": bool(result["early_stopped"]), + "request_count": selected, + "result_sha256": sha256_file(result_path), + "requests_sha256": sha256_file(requests_path), + } + ) + return metadata, streams + + +def common_decile_fractions( + *, trace_duration_s: float, minimum_elapsed_s: float +) -> tuple[float, ...]: + if trace_duration_s <= 0.0 or minimum_elapsed_s <= 0.0: + raise ValueError("trace duration and elapsed time must be positive") + supported = min( + MAX_DECILES, + int(math.floor((minimum_elapsed_s / trace_duration_s) * 10.0 + 1e-12)), + ) + return tuple( + round(index * DECILE_FRACTION, 10) for index in range(1, supported + 1) + ) + + +def _trial_record( + *, + run_root: Path, + run_dir: Path, + result: Mapping[str, Any], + state: dict[str, float], + outcome: dict[str, float], +) -> dict[str, Any]: + match = P1.RUN_PATTERN.match(run_dir.name) + assert match is not None + level, replicate_text = match.groups() + result_path = run_dir / "result.json" + requests_path = run_dir / "requests.jsonl" + return { + "trial_id": str(result_path.relative_to(run_root)), + "cell": str(result["cell"]), + "tp": int(result["tp"]), + "mns": int(result["mns"]), + "level": level, + "replicate": int(replicate_text), + "offered_rate_per_gpu": float( + result["selection"]["offered_req_s_per_gpu"] + ), + "request_hash": str(result["selection"]["request_id_order_sha256"]), + "request_count": int(result["selection"]["count"]), + "result_sha256": sha256_file(result_path), + "requests_sha256": sha256_file(requests_path), + "full_pass_rate": float(result["pass_rate"]), + "full_feasible": bool(result["feasible"]), + "early_stopped": bool(result["early_stopped"]), + "state": state, + "outcome": outcome, + } + + +def load_interval_trials( + run_root: Path, + intervals_s: tuple[tuple[float, float], ...], +) -> tuple[dict[tuple[float, float], list[dict[str, Any]]], list[dict[str, Any]]]: + by_interval = {interval: [] for interval in intervals_s} + stream_provenance = [] + for cell in sorted((run_root / "cells").iterdir()): + if not cell.is_dir(): + continue + stream_paths = sorted((cell / "opprof").glob("*.jsonl")) + if len(stream_paths) != 1: + raise ValueError(f"{cell}: expected one Layer-1 stream") + stream_path = stream_paths[0] + stream = P1.load_jsonl(stream_path) + stream_provenance.append( + { + "cell": cell.name, + "path": str(stream_path.resolve()), + "sha256": sha256_file(stream_path), + "bytes": stream_path.stat().st_size, + } + ) + for run_dir in sorted(cell.iterdir()): + if not run_dir.is_dir() or P1.RUN_PATTERN.match(run_dir.name) is None: + continue + result_path = run_dir / "result.json" + requests_path = run_dir / "requests.jsonl" + result = json.loads(result_path.read_text(encoding="utf-8")) + requests = P1.load_jsonl(requests_path) + start_ns = int(result["interval"]["start_mono_ns"]) + elapsed_s = float(result["interval"]["elapsed_s"]) + for interval in intervals_s: + start_s, end_s = interval + if start_s < 0.0 or end_s <= start_s: + raise ValueError(f"invalid analysis interval: {interval}") + if elapsed_s + 1e-9 < end_s: + raise ValueError( + f"{result_path}: elapsed {elapsed_s} shorter than {end_s}s" + ) + state = P1.V0.flatten_state( + P1.summarize_engine( + stream, + start_ns=start_ns + int(start_s * 1e9), + end_ns=start_ns + int(end_s * 1e9), + request_count=int(result["selection"]["count"]), + ) + ) + outcome = P1._prefix_outcome(result, requests, end_s) + by_interval[interval].append( + _trial_record( + run_root=run_root, + run_dir=run_dir, + result=result, + state=state, + outcome=outcome, + ) + ) + return by_interval, stream_provenance + + +def coverage(trials: list[dict[str, Any]]) -> dict[str, Any]: + admitted = [float(trial["outcome"]["admitted_fraction"]) for trial in trials] + completed = [ + float(trial["outcome"]["admitted_fraction"]) + * float(trial["outcome"]["completed_over_admitted"]) + for trial in trials + ] + return { + "admitted_fraction_of_total": numeric(admitted), + "completed_fraction_of_total": numeric(completed), + } + + +def slim_window_analysis( + trials: list[dict[str, Any]], *, start_s: float, end_s: float, fraction: float +) -> dict[str, Any]: + analysis = P1.analyze_horizon(trials, end_s) + return { + "start_s": start_s, + "end_s": end_s, + "end_fraction": fraction, + "coverage_at_end": coverage(trials), + "action_pairs": len(analysis["actions"]), + "repeat_pairs": len(analysis["repeats"]), + "response_statistics": analysis["response_statistics"], + "qualifying_response_features": analysis["qualifying_response_features"], + "efficacy": analysis["efficacy"], + "sanity": analysis["sanity"], + } + + +def _pearson(left: list[float], right: list[float]) -> float | None: + if len(left) != len(right) or not left: + raise ValueError("Pearson inputs must be non-empty and have equal length") + left_mean = sum(left) / len(left) + right_mean = sum(right) / len(right) + numerator = sum( + (x - left_mean) * (y - right_mean) + for x, y in zip(left, right, strict=True) + ) + left_ss = sum((x - left_mean) ** 2 for x in left) + right_ss = sum((y - right_mean) ** 2 for y in right) + if left_ss == 0.0 or right_ss == 0.0: + return None + return numerator / math.sqrt(left_ss * right_ss) + + +def trajectory_summary( + block_trials: list[tuple[tuple[float, float], list[dict[str, Any]]]] +) -> dict[str, Any]: + if not block_trials: + raise ValueError("trajectory requires at least one block") + identities = [] + states_by_block = [] + for interval, trials in block_trials: + ordered = sorted( + trials, + key=lambda trial: (trial["cell"], trial["level"], trial["replicate"]), + ) + current_identities = [ + (trial["cell"], trial["level"], trial["replicate"]) for trial in ordered + ] + if identities and current_identities != identities: + raise ValueError("trajectory blocks do not contain identical trials") + identities = current_identities + states_by_block.append((interval, [trial["state"] for trial in ordered])) + + features = {} + for feature in P1.V0.ALL_FEATURES: + block_values = [ + [float(state[feature]) for state in states] + for _interval, states in states_by_block + ] + first = block_values[0] + last = block_values[-1] + delta = [right - left for left, right in zip(first, last, strict=True)] + features[feature] = { + "block_medians": [median(values) for values in block_values], + "first_to_last_delta": numeric(delta), + "first_to_last_abs_delta": numeric(abs(value) for value in delta), + "first_to_last_pearson": _pearson(first, last), + "changed_trials": sum(abs(value) > 1e-12 for value in delta), + } + return { + "trial_count": len(identities), + "blocks": [ + {"start_s": interval[0], "end_s": interval[1]} + for interval, _states in states_by_block + ], + "features": features, + } + + +def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]: + metadata, metadata_streams = load_metadata(run_root) + durations = [float(item["trace_duration_s"]) for item in metadata] + elapsed = [float(item["elapsed_s"]) for item in metadata] + duration = median(durations) + deciles = common_decile_fractions( + trace_duration_s=duration, minimum_elapsed_s=min(elapsed) + ) + if not deciles: + raise ValueError("no complete replay decile is shared by all trials") + cumulative_intervals = tuple( + (0.0, round(duration * fraction, 9)) for fraction in deciles + ) + block_intervals = tuple( + ( + round(duration * (fraction - DECILE_FRACTION), 9), + round(duration * fraction, 9), + ) + for fraction in deciles + ) + all_intervals = tuple(dict.fromkeys([*cumulative_intervals, *block_intervals])) + trials_by_interval, streams = load_interval_trials(run_root, all_intervals) + manifest_validation = P1.validate_manifest( + trials_by_interval[cumulative_intervals[0]], manifest_path + ) + + cumulative = [] + blocks = [] + for fraction, cumulative_interval, block_interval in zip( + deciles, cumulative_intervals, block_intervals, strict=True + ): + cumulative.append( + slim_window_analysis( + trials_by_interval[cumulative_interval], + start_s=cumulative_interval[0], + end_s=cumulative_interval[1], + fraction=fraction, + ) + ) + blocks.append( + slim_window_analysis( + trials_by_interval[block_interval], + start_s=block_interval[0], + end_s=block_interval[1], + fraction=fraction, + ) + ) + + invariants = { + "expected_trial_count": len(metadata) == 36, + "trace_duration_consistent": max(durations) - min(durations) <= 1e-9, + "all_intervals_uncensored": all( + item["elapsed_s"] + 1e-9 >= cumulative_intervals[-1][1] + for item in metadata + ), + "stream_provenance_consistent": metadata_streams == streams, + "manifest_trials_match": ( + manifest_validation["expected_trials"] + == manifest_validation["matched_trials"] + == len(metadata) + ), + "all_window_sanity_pass": all( + not item["sanity"]["red_flags"] for item in [*cumulative, *blocks] + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + complete_full_trajectory = min(elapsed) + 1e-9 >= duration + if red_flags: + decision = "STOP_DATA_INVALID" + elif not complete_full_trajectory: + decision = "REQUIRES_UNCENSORED_PHASE_AWARE_PILOT" + else: + decision = "FULL_TRAJECTORY_AVAILABLE" + + payload = { + "schema": SCHEMA, + "status": "COMPLETE", + "decision": decision, + "claim_boundary": ( + "Post-hoc corrective audit over every common replay decile. It can " + "diagnose horizon sensitivity but cannot establish a held-out tuning claim." + ), + "design": { + "decile_fraction": DECILE_FRACTION, + "available_deciles": list(deciles), + "trace_duration_s": duration, + "maximum_common_end_s": cumulative_intervals[-1][1], + "maximum_common_fraction": deciles[-1], + "select_best_horizon": False, + "cumulative_and_nonoverlapping_blocks": True, + }, + "cumulative": cumulative, + "blocks": blocks, + "trajectory": trajectory_summary( + [(interval, trials_by_interval[interval]) for interval in block_intervals] + ), + "provenance": { + "analysis_script": str(Path(__file__).resolve()), + "analysis_script_sha256": sha256_file(Path(__file__).resolve()), + "p1_analysis_script": str(P1_PATH.resolve()), + "p1_analysis_script_sha256": sha256_file(P1_PATH), + "run_root": str(run_root.resolve()), + "manifest": str(manifest_path.resolve()), + "manifest_sha256": sha256_file(manifest_path), + "manifest_validation": manifest_validation, + "streams": streams, + "trial_inputs": metadata, + }, + "sanity": { + "trials": len(metadata), + "elapsed_s": numeric(elapsed), + "trace_duration_s": numeric(durations), + "early_stopped": sum(bool(item["early_stopped"]) for item in metadata), + "request_count": numeric(item["request_count"] for item in metadata), + "stream_bytes": numeric(item["bytes"] for item in streams), + "invariants": invariants, + "red_flags": red_flags, + }, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = audit( + run_root=args.run_root, + manifest_path=args.manifest, + output_path=args.output, + ) + print( + json.dumps( + { + "decision": payload["decision"], + "design": payload["design"], + "sanity": payload["sanity"], + }, + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v2/analyze_pilot.py b/runs/intervention-response-v2/analyze_pilot.py new file mode 100644 index 0000000..edbfbe1 --- /dev/null +++ b/runs/intervention-response-v2/analyze_pilot.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +"""Analyze the uncensored 300-second phase-aware matched pilot.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +from collections import defaultdict +from pathlib import Path +from typing import Any, Iterable, Mapping + + +HERE = Path(__file__).resolve().parent +P1_PATH = HERE.parent / "intervention-response-v0" / "analyze_p1.py" +SCHEMA = "intervention-response-phase-aware-pilot-analysis-v2" +EXPECTED_ACTION_PAIRS = 9 +EXPECTED_REPEAT_PAIRS = 12 +MIN_EFFICACY_CLASS = 3 + + +def _load_p1(): + spec = importlib.util.spec_from_file_location("intervention_response_p1", P1_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +P1 = _load_p1() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def numeric(values: Iterable[float | int]) -> dict[str, Any]: + return P1.V0.numeric(values) + + +def _trial_record( + *, + run_root: Path, + session: Mapping[str, Any], + level: str, + result: Mapping[str, Any], + result_path: Path, + requests_path: Path, + state: dict[str, float], + outcome: dict[str, float], +) -> dict[str, Any]: + return { + "trial_id": str(result_path.relative_to(run_root)), + "cell": str(result["cell"]), + "tp": int(result["tp"]), + "mns": int(result["mns"]), + "level": level, + "replicate": int(session["replicate"]), + "offered_rate_per_gpu": float( + result["selection"]["offered_req_s_per_gpu"] + ), + "request_hash": str(result["selection"]["request_id_order_sha256"]), + "request_count": int(result["selection"]["count"]), + "result_sha256": sha256_file(result_path), + "requests_sha256": sha256_file(requests_path), + "full_pass_rate": float(result["pass_rate"]), + "full_feasible": bool(result["feasible"]), + "early_stopped": bool(result["early_stopped"]), + "state": state, + "outcome": outcome, + } + + +def validate_result_against_manifest( + *, + result: Mapping[str, Any], + selection: Mapping[str, Any], + session: Mapping[str, Any], + level: str, + expected_duration_s: float, +) -> None: + identity = f"{session['session']}:{level}" + if int(result["mns"]) != int(session["mns"]) or int(result["tp"]) != 4: + raise ValueError(f"config mismatch: {identity}") + if bool(result["early_stopped"]): + raise ValueError(f"early-stopped measured result: {identity}") + if result.get("slo_early_stop_disabled") is not True: + raise ValueError(f"SLO early stop was enabled: {identity}") + if float(result["interval"]["elapsed_s"]) + 1e-9 < expected_duration_s: + raise ValueError(f"result does not cover full arrival window: {identity}") + if int(result["selection"]["count"]) != int(selection["selected_count"]): + raise ValueError(f"selection count mismatch: {identity}") + for result_key, manifest_key in ( + ("request_id_order_sha256", "request_id_order_sha256"), + ("arrival_order_sha256", "arrival_order_sha256"), + ("raw_length_order_sha256", "input_length_order_sha256"), + ): + if result["selection"][result_key] != selection[manifest_key]: + raise ValueError(f"selection hash mismatch {result_key}: {identity}") + if int(result["observed_count"]) != int(selection["selected_count"]): + raise ValueError(f"request accounting mismatch: {identity}") + + +def load_interval_trials( + *, + run_root: Path, + manifest: Mapping[str, Any], + intervals_s: tuple[tuple[float, float], ...], +) -> tuple[dict[tuple[float, float], list[dict[str, Any]]], list[dict[str, Any]]]: + by_interval = {interval: [] for interval in intervals_s} + streams = [] + duration_s = float(manifest["engine"]["duration_s"]) + for session in manifest["sessions"]: + session_root = run_root / "sessions" / str(session["session"]) + stream_paths = sorted((session_root / "opprof").glob("*.jsonl")) + if len(stream_paths) != 1: + raise ValueError(f"{session_root}: expected one Layer-1 stream") + stream_path = stream_paths[0] + stream = P1.load_jsonl(stream_path) + streams.append( + { + "session": str(session["session"]), + "path": str(stream_path.resolve()), + "sha256": sha256_file(stream_path), + "bytes": stream_path.stat().st_size, + } + ) + repetition = manifest["repetitions"][str(session["replicate"])] + for level, selection in repetition["selections"].items(): + result_path = session_root / level / "result.json" + requests_path = session_root / level / "requests.jsonl" + result = json.loads(result_path.read_text(encoding="utf-8")) + requests = P1.load_jsonl(requests_path) + validate_result_against_manifest( + result=result, + selection=selection, + session=session, + level=level, + expected_duration_s=duration_s, + ) + start_ns = int(result["interval"]["start_mono_ns"]) + for interval in intervals_s: + start_s, end_s = interval + state = P1.V0.flatten_state( + P1.summarize_engine( + stream, + start_ns=start_ns + int(start_s * 1e9), + end_ns=start_ns + int(end_s * 1e9), + request_count=int(result["selection"]["count"]), + ) + ) + outcome = P1._prefix_outcome(result, requests, end_s) + by_interval[interval].append( + _trial_record( + run_root=run_root, + session=session, + level=level, + result=result, + result_path=result_path, + requests_path=requests_path, + state=state, + outcome=outcome, + ) + ) + return by_interval, streams + + +def analyze_window( + trials: list[dict[str, Any]], *, start_s: float, end_s: float, fraction: float +) -> dict[str, Any]: + actions, repeats = P1.build_pairs(trials) + response = P1.V0.response_statistics(actions, repeats) + response_qualifying = sorted( + feature for feature, item in response.items() if item["qualifies"] + ) + labels = [int(pair["full_action_efficacy"]) for pair in actions] + cross_validation_possible = all( + set( + int(pair["full_action_efficacy"]) + for pair in actions + if pair["group"]["replicate"] != held_out + ) + == {0, 1} + for held_out in (1, 2, 3) + ) + if cross_validation_possible: + outcome_cv = P1.one_feature_leave_repeat_out( + actions, delta_key="delta_outcome", features=P1.OUTCOME_FEATURES + ) + telemetry_cv = P1.one_feature_leave_repeat_out( + actions, delta_key="delta_state", features=P1.V0.GATE_FEATURES + ) + outcome_best = float(outcome_cv["best_balanced_accuracy"]) + efficacy_qualifying = sorted( + feature + for feature, item in telemetry_cv["features"].items() + if item["balanced_accuracy"] >= P1.MIN_EFFICACY_BALANCED_ACCURACY + and item["balanced_accuracy"] + >= outcome_best + P1.MIN_EFFICACY_DELTA_OVER_OUTCOME + ) + else: + unavailable = { + "status": "UNAVAILABLE", + "reason": "each leave-one-repetition-out train fold needs both classes", + } + outcome_cv = unavailable + telemetry_cv = unavailable + efficacy_qualifying = [] + transitions = defaultdict(int) + for pair in actions: + transitions[pair["full_feasibility_transition"]] += 1 + admitted = [float(trial["outcome"]["admitted_fraction"]) for trial in trials] + completed = [ + float(trial["outcome"]["admitted_fraction"]) + * float(trial["outcome"]["completed_over_admitted"]) + for trial in trials + ] + action_metadata = [ + { + "group": pair["group"], + "source": pair["source"], + "target": pair["target"], + "full_action_efficacy": pair["full_action_efficacy"], + "full_feasibility_transition": pair["full_feasibility_transition"], + "delta_state": pair["delta_state"], + "delta_outcome": pair["delta_outcome"], + } + for pair in actions + ] + invariants = { + "expected_action_pair_count": len(actions) == EXPECTED_ACTION_PAIRS, + "expected_repeat_pair_count": len(repeats) == EXPECTED_REPEAT_PAIRS, + "finite_deltas": all( + math.isfinite(value) + for pair in [*actions, *repeats] + for key in ("delta_state", "delta_outcome") + for value in pair[key].values() + ), + "all_results_uncensored": all(not trial["early_stopped"] for trial in trials), + } + return { + "start_s": start_s, + "end_s": end_s, + "end_fraction": fraction, + "coverage_at_end": { + "admitted_fraction_of_total": numeric(admitted), + "completed_fraction_of_total": numeric(completed), + }, + "action_pairs": len(actions), + "repeat_pairs": len(repeats), + "actions": action_metadata, + "response_statistics": response, + "qualifying_response_features": response_qualifying, + "efficacy": { + "labels": numeric(labels), + "positive": sum(labels), + "negative": len(labels) - sum(labels), + "label_balance_sufficient": ( + sum(labels) >= MIN_EFFICACY_CLASS + and len(labels) - sum(labels) >= MIN_EFFICACY_CLASS + ), + "cross_validation_possible": cross_validation_possible, + "transitions": dict(sorted(transitions.items())), + "outcome_delta": outcome_cv, + "telemetry_delta": telemetry_cv, + "telemetry_qualifying_features": efficacy_qualifying, + }, + "sanity": { + "trials": len(trials), + "invariants": invariants, + "red_flags": [name for name, passed in invariants.items() if not passed], + }, + } + + +def stable_adjacent_features(windows: list[dict[str, Any]]) -> dict[str, list[str]]: + result = {} + for left, right in zip(windows, windows[1:], strict=False): + key = f"{left['end_fraction']:.2f}->{right['end_fraction']:.2f}" + result[key] = sorted( + set(left["qualifying_response_features"]) + & set(right["qualifying_response_features"]) + ) + return result + + +def consistent_load_regimes( + windows: list[dict[str, Any]], stable: dict[str, list[str]] +) -> dict[str, Any]: + by_end = {float(window["end_fraction"]): window for window in windows} + result = {} + for transition, features in stable.items(): + _left_text, right_text = transition.split("->") + window = by_end[float(right_text)] + for feature in features: + deltas_by_level: dict[str, list[float]] = defaultdict(list) + all_deltas = [] + for action in window["actions"]: + value = float(action["delta_state"][feature]) + deltas_by_level[str(action["group"]["level"])].append(value) + all_deltas.append(value) + positive = sum(value > 1e-12 for value in all_deltas) + negative = sum(value < -1e-12 for value in all_deltas) + direction = 1 if positive >= negative else -1 + consistent = [] + for level, values in sorted(deltas_by_level.items()): + matching = sum(direction * value > 1e-12 for value in values) + nonzero = sum(abs(value) > 1e-12 for value in values) + if nonzero and matching / nonzero >= 2.0 / 3.0: + consistent.append(level) + result[f"{transition}:{feature}"] = { + "direction": direction, + "consistent_load_regimes": consistent, + "passes_two_regimes": len(consistent) >= 2, + } + return result + + +def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": + raise ValueError("unexpected phase-aware pilot manifest schema") + fractions = [float(value) for value in manifest["checkpoints"]["fractions"]] + seconds = [float(value) for value in manifest["checkpoints"]["seconds"]] + cumulative_intervals = tuple((0.0, end_s) for end_s in seconds) + quarter_intervals = ((0.0, 75.0), (75.0, 150.0), (150.0, 225.0), (225.0, 300.0)) + intervals = tuple(dict.fromkeys([*cumulative_intervals, *quarter_intervals])) + trials_by_interval, streams = load_interval_trials( + run_root=run_root, manifest=manifest, intervals_s=intervals + ) + cumulative = [ + analyze_window( + trials_by_interval[interval], + start_s=interval[0], + end_s=interval[1], + fraction=fraction, + ) + for fraction, interval in zip(fractions, cumulative_intervals, strict=True) + ] + quarter_blocks = [ + analyze_window( + trials_by_interval[interval], + start_s=interval[0], + end_s=interval[1], + fraction=interval[1] / 300.0, + ) + for interval in quarter_intervals + ] + stable = stable_adjacent_features(cumulative) + load_consistency = consistent_load_regimes(cumulative, stable) + mechanism_features = sorted( + { + key.split(":", 1)[1] + for key, item in load_consistency.items() + if item["passes_two_regimes"] + } + ) + full = cumulative[-1] + efficacy_features = sorted( + set.intersection( + *( + set(window["efficacy"]["telemetry_qualifying_features"]) + for window in cumulative + if window["end_fraction"] >= 0.25 + ) + ) + ) + red_flags = sorted( + { + flag + for window in [*cumulative, *quarter_blocks] + for flag in window["sanity"]["red_flags"] + } + ) + if red_flags: + decision = "STOP_DATA_INVALID" + elif not mechanism_features: + decision = "STOP_NO_PHASE_STABLE_RESPONSE" + elif not full["efficacy"]["label_balance_sufficient"]: + decision = "MECHANISM_ONLY_NO_LABEL_BALANCE" + elif not efficacy_features: + decision = "STOP_NO_INCREMENTAL_TUNING_SIGNAL" + else: + decision = "OPEN_E2E_POLICY_TEST" + payload = { + "schema": SCHEMA, + "status": "COMPLETE", + "decision": decision, + "claim_boundary": "Development mechanism pilot; not a held-out paper claim.", + "mechanism_features": mechanism_features, + "stable_adjacent_features": stable, + "load_consistency": load_consistency, + "stable_incremental_efficacy_features": efficacy_features, + "cumulative": cumulative, + "quarter_blocks": quarter_blocks, + "provenance": { + "analysis_script": str(Path(__file__).resolve()), + "analysis_script_sha256": sha256_file(Path(__file__).resolve()), + "manifest": str(manifest_path.resolve()), + "manifest_sha256": sha256_file(manifest_path), + "run_root": str(run_root.resolve()), + "streams": streams, + }, + "sanity": { + "streams": len(streams), + "stream_bytes": numeric(item["bytes"] for item in streams), + "red_flags": red_flags, + }, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = audit( + run_root=args.run_root, + manifest_path=args.manifest, + output_path=args.output, + ) + print( + json.dumps( + { + "decision": payload["decision"], + "mechanism_features": payload["mechanism_features"], + "stable_incremental_efficacy_features": payload[ + "stable_incremental_efficacy_features" + ], + "sanity": payload["sanity"], + }, + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v2/existing-phase-audit.json b/runs/intervention-response-v2/existing-phase-audit.json new file mode 100644 index 0000000..3884878 --- /dev/null +++ b/runs/intervention-response-v2/existing-phase-audit.json @@ -0,0 +1,9915 @@ +{ + "blocks": [ + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 21, + "max": 0.12295081967213115, + "median": 0.08985507246376812, + "min": 0.04918032786885246, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 30, + "max": 0.09361702127659574, + "median": 0.050437283852088316, + "min": 0.005586592178770949, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.4166666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 0 + ], + "threshold": 0.051724137931034475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.09339080459770116, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.018181818181818188, + "train_balanced_accuracy": 0.625 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.4166666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 0 + ], + "threshold": -0.051724137931034475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.09339080459770113, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.01818181818181816, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.015623848188639142, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0010320465723873684, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": 0.009580791853053022, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0002360795908454838, + "train_balanced_accuracy": 0.625 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.0072977563346891365, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.021249613758603417, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.007412010668000825, + "train_balanced_accuracy": 0.875 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": -0.0008523871757378475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.001080823830574982, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.006279735096096474, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.6666666666666666, + "best_balanced_accuracy": 0.6666666666666666, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.5847120251326046, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.1503494070398912, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.759913509761732, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0003962349313431858, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0006270962078854048, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 0.003423657452078254, + "train_balanced_accuracy": 0.75 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00016329402239600305, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00013516820682281123, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.00013516820682281123, + "train_balanced_accuracy": 0.625 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.3333333333333333, + "balanced_accuracy": 0.3333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.0003659931906908054, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0003153433269442174, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": 0.002846241748737599, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 1.2024471918333335, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.18325025108333315, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 1.2024471918333335, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.6539956596666667, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.009266072583333333, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.6539956596666667, + "train_balanced_accuracy": 0.875 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 1.5833333333333357, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 4.416666666666664, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -19.583333333333332, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.1, + "end_s": 6.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.587816632458785, + "action_delta": { + "distinct_n": 12, + "max": 10.944163150492265, + "min": -0.9368137011239002, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.1464621283214008, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 4.013437734353163, + "repeat_abs_p95": 8.899704585700528, + "repeat_delta": { + "distinct_n": 24, + "max": 9.092698412698415, + "min": -9.933716475095787, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 7.901570082144033, + "action_delta": { + "distinct_n": 12, + "max": 564.520698546648, + "min": -8.58386322115848, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.05293318721992956, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 149.27440604160446, + "repeat_abs_p95": 426.8141407023969, + "repeat_delta": { + "distinct_n": 24, + "max": 449.2116931216931, + "min": -584.6987867177522, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.582158519407685, + "action_delta": { + "distinct_n": 12, + "max": 10.735314111579934, + "min": -0.9288831363334129, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.14646507698937813, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.9747257938484823, + "repeat_abs_p95": 8.770067452024344, + "repeat_delta": { + "distinct_n": 24, + "max": 8.954735449735448, + "min": -9.735983397190292, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.005042559238173261, + "action_delta": { + "distinct_n": 12, + "max": 0.008374384236453203, + "min": -0.10829817158931077, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.12202178454078347, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.04132507369196753, + "repeat_abs_p95": 0.08139543508103343, + "repeat_delta": { + "distinct_n": 24, + "max": 0.12107279693486583, + "min": -0.0822222222222222, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.008375994890060249, + "action_delta": { + "distinct_n": 12, + "max": 0.09274496015002344, + "min": -0.01986863711001642, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 9, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.22398385207837773, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.037395530134597704, + "repeat_abs_p95": 0.07586812708364432, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07631077509373656, + "min": -0.10941890166028097, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0011703901376772582, + "action_delta": { + "distinct_n": 12, + "max": 0.006703036588479568, + "min": -0.0015820428864207082, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.29946486376659237, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.003908271985422232, + "repeat_abs_p95": 0.008187883039017584, + "repeat_delta": { + "distinct_n": 24, + "max": 0.007141591578301508, + "min": -0.009201187757062074, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.006820888350695331, + "action_delta": { + "distinct_n": 12, + "max": 0.140893666867539, + "min": -0.005054735620165451, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 0.6661896437498689, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010238658638254572, + "repeat_abs_p95": 0.1277225497052652, + "repeat_delta": { + "distinct_n": 24, + "max": 0.13402937356379263, + "min": -0.1168947946847837, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.007604372340016818, + "action_delta": { + "distinct_n": 12, + "max": 0.06684057734774362, + "min": 0.00016629145416469093, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 0.3770466591391387, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.020168252802926, + "repeat_abs_p95": 0.11084274153262068, + "repeat_delta": { + "distinct_n": 24, + "max": 0.12803476870816266, + "min": -0.11234888600259774, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0014831270777658032, + "action_delta": { + "distinct_n": 12, + "max": 0.02895016189128262, + "min": -0.00029784868681948656, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.12120157645159224, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.01223686292857884, + "repeat_abs_p95": 0.07958153755957757, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08820147130614941, + "min": -0.08208138515366167, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0028294297587345696, + "action_delta": { + "distinct_n": 12, + "max": 0.013691652669458043, + "min": -0.002949832419021914, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.44362661948582977, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.006377953067861264, + "repeat_abs_p95": 0.031272369907012425, + "repeat_delta": { + "distinct_n": 24, + "max": 0.036461564751038456, + "min": -0.018166035027340932, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.50741128125, + "action_delta": { + "distinct_n": 12, + "max": 15.614046583666667, + "min": -0.30207624649999953, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.12962585804860988, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.9144294887499997, + "repeat_abs_p95": 7.101594725466666, + "repeat_delta": { + "distinct_n": 24, + "max": 6.513803891666667, + "min": -13.5536487565, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.24895167858333334, + "action_delta": { + "distinct_n": 10, + "max": 0.0, + "min": -8.040727011166666, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 9, + "positive": 0, + "zero": 3 + }, + "effect_to_repeat_median": 1.1132189691456902, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.22363226416666668, + "repeat_abs_p95": 2.296808863758333, + "repeat_delta": { + "distinct_n": 16, + "max": 2.3311656403333334, + "min": -5.974843045, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 3.249999999999993, + "action_delta": { + "distinct_n": 12, + "max": 10.5, + "min": -30.999999999999993, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.1065573770491801, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 30.5, + "repeat_abs_p95": 59.92499999999999, + "repeat_delta": { + "distinct_n": 24, + "max": 59.5, + "min": -64.5, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 0.0 + }, + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 22, + "max": 0.20535714285714285, + "median": 0.18235294117647058, + "min": 0.1564245810055866, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 31, + "max": 0.180327868852459, + "median": 0.14411764705882352, + "min": 0.08933333333333335, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.8333333333333334, + "best_balanced_accuracy": 0.8333333333333333, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.012666666666666666, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.008666666666666666, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.008666666666666666, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.03401360544217691, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": 0.006172839506172811, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.04871948779511809, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.11875000000000002, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.07738095238095238, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.07738095238095238, + "train_balanced_accuracy": 0.75 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.03401360544217688, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": -0.006172839506172839, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.04871948779511806, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.028440662518539606, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.02030446551608167, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.008882265987138677, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": -0.0020290170113650557, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.04992359808650447, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.009324194656287296, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.01753702182516767, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.01753702182516767, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.018086205842943556, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.010556909058063094, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.003084423316020539, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.027367186208561683, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.22344269117369397, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.22344269117369397, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.5979430550644496, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.006014036688642874, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.003996367458148475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": 0.004817288488210095, + "train_balanced_accuracy": 0.875 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": 0.0009690245761390937, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.014778075272949286, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00021729561459569828, + "train_balanced_accuracy": 0.75 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.0004249745076460587, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0008306228170546492, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": -0.005838187880452539, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": -0.014251387166666962, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 9.969867595499998, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.36342276333333334, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.37299224008333337, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.09157276941666666, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.08574448658333333, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.75, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.75, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 6.833333333333329, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.2, + "end_s": 12.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.836467208595514, + "action_delta": { + "distinct_n": 12, + "max": 38.01660606060606, + "min": -3.7894183532722323, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.6942297063022367, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 1.2048853585521353, + "repeat_abs_p95": 9.90189234803657, + "repeat_delta": { + "distinct_n": 24, + "max": 6.213858257127487, + "min": -19.9836254925941, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 68.40350288667949, + "action_delta": { + "distinct_n": 12, + "max": 1287.087393939394, + "min": -109.74922437881139, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.6828036858921454, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 40.648534026959354, + "repeat_abs_p95": 252.72760702584105, + "repeat_delta": { + "distinct_n": 24, + "max": 53.09534549057537, + "min": -841.3287810843865, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.79935614604261, + "action_delta": { + "distinct_n": 12, + "max": 37.563515151515155, + "min": -3.7555281535670666, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.6727103099461282, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.1882620709449565, + "repeat_abs_p95": 9.833929419939027, + "repeat_delta": { + "distinct_n": 24, + "max": 6.226902904787521, + "min": -19.76212800652263, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.023223408379118127, + "action_delta": { + "distinct_n": 12, + "max": 0.03622200353327565, + "min": -0.23381818181818181, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 2.5685579123999758, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.009041419026218855, + "repeat_abs_p95": 0.07343409055233649, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08839516238619383, + "min": -0.027874161461947766, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.0196424445317687, + "action_delta": { + "distinct_n": 12, + "max": 0.22624242424242422, + "min": -0.043130466400287726, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 1.9837484419794449, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.009901681138647232, + "repeat_abs_p95": 0.07568864235997094, + "repeat_delta": { + "distinct_n": 24, + "max": 0.02793199167244964, + "min": -0.10324092947411331, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0019741621395090708, + "action_delta": { + "distinct_n": 12, + "max": 0.013535143477830873, + "min": -0.0002595396932579229, + "n": 12 + }, + "action_signs": { + "consistency": 0.9166666666666666, + "negative": 1, + "positive": 11, + "zero": 0 + }, + "effect_to_repeat_median": 0.6824854579006991, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.002892606892433317, + "repeat_abs_p95": 0.013935039231725855, + "repeat_delta": { + "distinct_n": 24, + "max": 0.016710257140637542, + "min": -0.0048630254047383765, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.016292713617491184, + "action_delta": { + "distinct_n": 12, + "max": 0.06869972057736717, + "min": -0.13905774767384216, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 1.2163146605755397, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.013395146951350356, + "repeat_abs_p95": 0.10929110116632139, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07238485363173153, + "min": -0.2557698071735438, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.01447565921273003, + "action_delta": { + "distinct_n": 12, + "max": 0.10932093798799158, + "min": 0.0002683613903166071, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.4049345895249048, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010303439975540174, + "repeat_abs_p95": 0.07329653312019176, + "repeat_delta": { + "distinct_n": 24, + "max": 0.05015486062543706, + "min": -0.12688580277750028, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.005335907884240179, + "action_delta": { + "distinct_n": 12, + "max": 0.06835893039525985, + "min": -0.0020715594873930866, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 1.4198198744789212, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.003758158327089537, + "repeat_abs_p95": 0.054511554710704666, + "repeat_delta": { + "distinct_n": 24, + "max": 0.05530066409194215, + "min": -0.1166293577215588, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00639806071748561, + "action_delta": { + "distinct_n": 12, + "max": 0.01743846124119386, + "min": -0.012883247545976029, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.5860208014218505, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.010917804797990316, + "repeat_abs_p95": 0.038061500221448455, + "repeat_delta": { + "distinct_n": 24, + "max": 0.006259881301210357, + "min": -0.03966667403951463, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.8894945159166667, + "action_delta": { + "distinct_n": 12, + "max": 39.57822979983333, + "min": -2.9770223346666658, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.4702668338949774, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.8914676770833334, + "repeat_abs_p95": 10.096755181974991, + "repeat_delta": { + "distinct_n": 21, + "max": 6.583703286333334, + "min": -20.7125852155, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.79726731775, + "action_delta": { + "distinct_n": 11, + "max": 0.0, + "min": -30.713414816833335, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 10, + "positive": 0, + "zero": 2 + }, + "effect_to_repeat_median": 17.026052234021265, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.04682631691666666, + "repeat_abs_p95": 3.5462989908416667, + "repeat_delta": { + "distinct_n": 17, + "max": 3.561578917166667, + "min": -17.805020274833332, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 13.083333333333334, + "action_delta": { + "distinct_n": 12, + "max": 23.66666666666667, + "min": -40.5, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 1.2661290322580652, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 10.333333333333329, + "repeat_abs_p95": 55.25833333333328, + "repeat_delta": { + "distinct_n": 24, + "max": 71.83333333333333, + "min": -19.333333333333343, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 6.0 + }, + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 22, + "max": 0.30654761904761907, + "median": 0.2795403407264545, + "min": 0.24313725490196078, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 30, + "max": 0.2681564245810056, + "median": 0.22518087736972248, + "min": 0.12849162011173185, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "completed_pass_rate", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.02825325884543762, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.02266666666666667, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.040253258845437616, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.03272660415517559, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.022522522522522515, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.07358436332279389, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.14270516717325227, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.12142857142857144, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.2182462927143778, + "train_balanced_accuracy": 0.75 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.03272660415517557, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.022522522522522515, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0735843633227939, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.37412821213021774, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.009857117327656123, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.008882265987138677, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": -0.004552157689988692, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.007167514027906635, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.006851562902858294, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.017755539496041216, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.007733676747496546, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.5040019049192779, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.007216190772090309, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.007216190772090309, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.24525856553394837, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "graph_padding_fraction", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 1.367157621094477, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.45324800586584313, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.7719319083901095, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.0006822319057190087, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.0006574027141346845, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.0007737011910822953, + "train_balanced_accuracy": 0.75 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.0014242600332016828, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.00020601061133531957, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.0014242600332016828, + "train_balanced_accuracy": 0.75 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 0.0001890219308638419, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": 6.874128372463817e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 0 + ], + "threshold": 0.0012836451412676642, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 2.090120409833334, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.4494322117499996, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 1.383383161750001, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.015561036916666665, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.015561036916666665, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.35703936641666667, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -5.666666666666664, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": 3.0, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -2.6666666666666607, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.3, + "end_s": 18.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 1.4376624963208817, + "action_delta": { + "distinct_n": 12, + "max": 44.66904761904762, + "min": -3.1358226741876534, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.5185498088911062, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.9467338429752983, + "repeat_abs_p95": 8.569947817628815, + "repeat_delta": { + "distinct_n": 24, + "max": 4.8797843665768195, + "min": -24.305454429328734, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 29.54235812446782, + "action_delta": { + "distinct_n": 12, + "max": 981.3419420915683, + "min": -40.25177564153491, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.44186157650879665, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 66.85885285135149, + "repeat_abs_p95": 331.06991793371697, + "repeat_delta": { + "distinct_n": 24, + "max": 347.23329948092976, + "min": -494.75378424985627, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 1.426379390321979, + "action_delta": { + "distinct_n": 12, + "max": 44.29932712215321, + "min": -3.1155318979899898, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.5217762546171796, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.9373121613602792, + "repeat_abs_p95": 8.519190735455247, + "repeat_delta": { + "distinct_n": 24, + "max": 4.8432704402515725, + "min": -24.101136871686787, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.010376892235975155, + "action_delta": { + "distinct_n": 12, + "max": 0.020290776197664284, + "min": -0.19708796988318666, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.5862533874020509, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0177003535654775, + "repeat_abs_p95": 0.05622040303840937, + "repeat_delta": { + "distinct_n": 24, + "max": 0.14549402823018454, + "min": -0.05811329271044907, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.015566450509702876, + "action_delta": { + "distinct_n": 12, + "max": 0.1830537562974035, + "min": -0.028434098673234293, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.9711084270914145, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.016029570000052673, + "repeat_abs_p95": 0.06874268847661695, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07111825772963212, + "min": -0.1536373507057546, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.0019204725324846257, + "action_delta": { + "distinct_n": 12, + "max": 0.011531473650189953, + "min": 1.5710198019351293e-05, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.109916935840515, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0017302849163485332, + "repeat_abs_p95": 0.011201032721421313, + "repeat_delta": { + "distinct_n": 24, + "max": 0.012042080910355725, + "min": -0.011611080181771623, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.012368563885621575, + "action_delta": { + "distinct_n": 12, + "max": 0.1693202894846253, + "min": -0.027238473582144707, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 1.1201833133796462, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.01104155341174029, + "repeat_abs_p95": 0.14212743371007872, + "repeat_delta": { + "distinct_n": 24, + "max": 0.1535118393445899, + "min": -0.2053651713457888, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.005959053027072225, + "action_delta": { + "distinct_n": 12, + "max": 0.22224967428314468, + "min": 3.590298430544703e-05, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 0.49652693165856604, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.012001469904496409, + "repeat_abs_p95": 0.06026674766281165, + "repeat_delta": { + "distinct_n": 24, + "max": 0.06029573383954434, + "min": -0.267359376561095, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.0057476210577813765, + "action_delta": { + "distinct_n": 12, + "max": 0.18534326857338257, + "min": -0.015438462146793161, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.6334301280804308, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.009073804359762887, + "repeat_abs_p95": 0.0458855138590948, + "repeat_delta": { + "distinct_n": 24, + "max": 0.033340684091908934, + "min": -0.24888103806759748, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.003319188436037235, + "action_delta": { + "distinct_n": 12, + "max": 0.013152637562827363, + "min": -0.02492780421592411, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.1361252231813899, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.02438334614602866, + "repeat_abs_p95": 0.07693546674256392, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08157979537248183, + "min": -0.04610629729936333, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 1.8914151395833336, + "action_delta": { + "distinct_n": 12, + "max": 45.162770761333334, + "min": -2.3232337786666672, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 1.438574132498461, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.3147846168333333, + "repeat_abs_p95": 6.971346508883332, + "repeat_delta": { + "distinct_n": 21, + "max": 4.9605220578333356, + "min": -20.271844710166665, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 1.12754089475, + "action_delta": { + "distinct_n": 10, + "max": 0.0, + "min": -47.07583785533333, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 9, + "positive": 0, + "zero": 3 + }, + "effect_to_repeat_median": 16.245384499464624, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.06940684566666666, + "repeat_abs_p95": 5.181811176816667, + "repeat_delta": { + "distinct_n": 18, + "max": 0.6568425336666667, + "min": -7.1538662623333344, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 7.166666666666668, + "action_delta": { + "distinct_n": 12, + "max": 34.0, + "min": -47.33333333333333, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.31617647058823534, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 22.666666666666668, + "repeat_abs_p95": 55.666666666666664, + "repeat_delta": { + "distinct_n": 24, + "max": 85.33333333333333, + "min": -55.66666666666667, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 12.0 + } + ], + "claim_boundary": "Post-hoc corrective audit over every common replay decile. It can diagnose horizon sensitivity but cannot establish a held-out tuning claim.", + "cumulative": [ + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 21, + "max": 0.12295081967213115, + "median": 0.08985507246376812, + "min": 0.04918032786885246, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 30, + "max": 0.09361702127659574, + "median": 0.050437283852088316, + "min": 0.005586592178770949, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.4166666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 0 + ], + "threshold": 0.051724137931034475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.09339080459770116, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.018181818181818188, + "train_balanced_accuracy": 0.625 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.4166666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 0 + ], + "threshold": -0.051724137931034475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.09339080459770113, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.01818181818181816, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.015623848188639142, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0010320465723873684, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": 0.009580791853053022, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0002360795908454838, + "train_balanced_accuracy": 0.625 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.0072977563346891365, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.021249613758603417, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.007412010668000825, + "train_balanced_accuracy": 0.875 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": -0.0008523871757378475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.001080823830574982, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.006279735096096474, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.6666666666666666, + "best_balanced_accuracy": 0.6666666666666666, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.5847120251326046, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.1503494070398912, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.759913509761732, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0003962349313431858, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0006270962078854048, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 0.003423657452078254, + "train_balanced_accuracy": 0.75 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00016329402239600305, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00013516820682281123, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.00013516820682281123, + "train_balanced_accuracy": 0.625 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.3333333333333333, + "balanced_accuracy": 0.3333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.0003659931906908054, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0003153433269442174, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": 0.002846241748737599, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 1.2024471918333335, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.18325025108333315, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 1.2024471918333335, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.6539956596666667, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.009266072583333333, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.6539956596666667, + "train_balanced_accuracy": 0.875 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 1.5833333333333357, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 4.416666666666664, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -19.583333333333332, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.1, + "end_s": 6.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.587816632458785, + "action_delta": { + "distinct_n": 12, + "max": 10.944163150492265, + "min": -0.9368137011239002, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.1464621283214008, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 4.013437734353163, + "repeat_abs_p95": 8.899704585700528, + "repeat_delta": { + "distinct_n": 24, + "max": 9.092698412698415, + "min": -9.933716475095787, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 7.901570082144033, + "action_delta": { + "distinct_n": 12, + "max": 564.520698546648, + "min": -8.58386322115848, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.05293318721992956, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 149.27440604160446, + "repeat_abs_p95": 426.8141407023969, + "repeat_delta": { + "distinct_n": 24, + "max": 449.2116931216931, + "min": -584.6987867177522, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.582158519407685, + "action_delta": { + "distinct_n": 12, + "max": 10.735314111579934, + "min": -0.9288831363334129, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.14646507698937813, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.9747257938484823, + "repeat_abs_p95": 8.770067452024344, + "repeat_delta": { + "distinct_n": 24, + "max": 8.954735449735448, + "min": -9.735983397190292, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.005042559238173261, + "action_delta": { + "distinct_n": 12, + "max": 0.008374384236453203, + "min": -0.10829817158931077, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.12202178454078347, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.04132507369196753, + "repeat_abs_p95": 0.08139543508103343, + "repeat_delta": { + "distinct_n": 24, + "max": 0.12107279693486583, + "min": -0.0822222222222222, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.008375994890060249, + "action_delta": { + "distinct_n": 12, + "max": 0.09274496015002344, + "min": -0.01986863711001642, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 9, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.22398385207837773, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.037395530134597704, + "repeat_abs_p95": 0.07586812708364432, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07631077509373656, + "min": -0.10941890166028097, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0011703901376772582, + "action_delta": { + "distinct_n": 12, + "max": 0.006703036588479568, + "min": -0.0015820428864207082, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.29946486376659237, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.003908271985422232, + "repeat_abs_p95": 0.008187883039017584, + "repeat_delta": { + "distinct_n": 24, + "max": 0.007141591578301508, + "min": -0.009201187757062074, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.006820888350695331, + "action_delta": { + "distinct_n": 12, + "max": 0.140893666867539, + "min": -0.005054735620165451, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 0.6661896437498689, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010238658638254572, + "repeat_abs_p95": 0.1277225497052652, + "repeat_delta": { + "distinct_n": 24, + "max": 0.13402937356379263, + "min": -0.1168947946847837, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.007604372340016818, + "action_delta": { + "distinct_n": 12, + "max": 0.06684057734774362, + "min": 0.00016629145416469093, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 0.3770466591391387, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.020168252802926, + "repeat_abs_p95": 0.11084274153262068, + "repeat_delta": { + "distinct_n": 24, + "max": 0.12803476870816266, + "min": -0.11234888600259774, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0014831270777658032, + "action_delta": { + "distinct_n": 12, + "max": 0.02895016189128262, + "min": -0.00029784868681948656, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.12120157645159224, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.01223686292857884, + "repeat_abs_p95": 0.07958153755957757, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08820147130614941, + "min": -0.08208138515366167, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0028294297587345696, + "action_delta": { + "distinct_n": 12, + "max": 0.013691652669458043, + "min": -0.002949832419021914, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.44362661948582977, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.006377953067861264, + "repeat_abs_p95": 0.031272369907012425, + "repeat_delta": { + "distinct_n": 24, + "max": 0.036461564751038456, + "min": -0.018166035027340932, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.50741128125, + "action_delta": { + "distinct_n": 12, + "max": 15.614046583666667, + "min": -0.30207624649999953, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.12962585804860988, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.9144294887499997, + "repeat_abs_p95": 7.101594725466666, + "repeat_delta": { + "distinct_n": 24, + "max": 6.513803891666667, + "min": -13.5536487565, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.24895167858333334, + "action_delta": { + "distinct_n": 10, + "max": 0.0, + "min": -8.040727011166666, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 9, + "positive": 0, + "zero": 3 + }, + "effect_to_repeat_median": 1.1132189691456902, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.22363226416666668, + "repeat_abs_p95": 2.296808863758333, + "repeat_delta": { + "distinct_n": 16, + "max": 2.3311656403333334, + "min": -5.974843045, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 3.249999999999993, + "action_delta": { + "distinct_n": 12, + "max": 10.5, + "min": -30.999999999999993, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.1065573770491801, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 30.5, + "repeat_abs_p95": 59.92499999999999, + "repeat_delta": { + "distinct_n": 24, + "max": 59.5, + "min": -64.5, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 0.0 + }, + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 22, + "max": 0.20535714285714285, + "median": 0.18235294117647058, + "min": 0.1564245810055866, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 31, + "max": 0.180327868852459, + "median": 0.14411764705882352, + "min": 0.08933333333333335, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.8333333333333334, + "best_balanced_accuracy": 0.8333333333333333, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.012666666666666666, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.008666666666666666, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.008666666666666666, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.03401360544217691, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": 0.006172839506172811, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.04871948779511809, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.11875000000000002, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.07738095238095238, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.07738095238095238, + "train_balanced_accuracy": 0.75 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.03401360544217688, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": -0.006172839506172839, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.04871948779511806, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.028440662518539606, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.02030446551608167, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.008882265987138677, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": -0.0020290170113650557, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.04992359808650447, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.009324194656287296, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.01753702182516767, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.01753702182516767, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.018086205842943556, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.010556909058063094, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.003084423316020539, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.027367186208561683, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.6666666666666666, + "best_balanced_accuracy": 0.6666666666666666, + "best_feature": "scheduler_steps_per_s", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": -0.17161348298791945, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.30619421565755833, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.6654132739143297, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.005113976430465315, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -5.156781096974641e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": 0.0022397228564162543, + "train_balanced_accuracy": 0.75 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -1.7899149433659497e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -2.0776132762541973e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00013983916927168966, + "train_balanced_accuracy": 0.75 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 3.837762108915843e-05, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": -0.00021675516061736566, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.837762108915843e-05, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": -0.0830968510000003, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.12225680983333342, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.2734091407916659, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.2763032289583333, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.08217400995833332, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.0689820661666667, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.2916666666666643, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.5833333333333286, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 6.291666666666657, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.2, + "end_s": 12.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.6805455173736323, + "action_delta": { + "distinct_n": 12, + "max": 21.022824302134648, + "min": -1.6199557480463245, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.2914327913831549, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 2.335171392840622, + "repeat_abs_p95": 10.342426400114645, + "repeat_delta": { + "distinct_n": 24, + "max": 10.582610461920808, + "min": -14.50489117143556, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 30.73978867286374, + "action_delta": { + "distinct_n": 12, + "max": 837.5693088520675, + "min": -38.259099896431565, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.5085540691379722, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 60.44546792235762, + "repeat_abs_p95": 370.0653796793133, + "repeat_delta": { + "distinct_n": 24, + "max": 385.95091643367505, + "min": -701.2529783740349, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.6733471921553615, + "action_delta": { + "distinct_n": 12, + "max": 20.720637408568443, + "min": -1.606656623670089, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.2892646285380025, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 2.327789593766041, + "repeat_abs_p95": 10.226231015894982, + "repeat_delta": { + "distinct_n": 24, + "max": 10.460062847993882, + "min": -14.289496209342136, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.008498282584807226, + "action_delta": { + "distinct_n": 12, + "max": 0.024842293569343754, + "min": -0.15582922824302126, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.517404626039514, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.01642482915133081, + "repeat_abs_p95": 0.0799421671518786, + "repeat_delta": { + "distinct_n": 24, + "max": 0.11509974496034647, + "min": -0.08092575333954644, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.01137367331008028, + "action_delta": { + "distinct_n": 12, + "max": 0.14248395282878046, + "min": -0.030463233217211186, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 8, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.6710383267094245, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.016949364674672823, + "repeat_abs_p95": 0.07611571753021439, + "repeat_delta": { + "distinct_n": 24, + "max": 0.0761583865032141, + "min": -0.11305593403905953, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.0022211647342633674, + "action_delta": { + "distinct_n": 12, + "max": 0.007755344475118115, + "min": -0.0006496983398026016, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 1.0130418206479903, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0021925696343341517, + "repeat_abs_p95": 0.006616310818721587, + "repeat_delta": { + "distinct_n": 24, + "max": 0.007943398472261458, + "min": -0.005259691066565701, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.0021058619896323627, + "action_delta": { + "distinct_n": 12, + "max": 0.06906439560383515, + "min": -0.0026342182586551743, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.17774345954986717, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.011847760783802841, + "repeat_abs_p95": 0.0656650541454329, + "repeat_delta": { + "distinct_n": 24, + "max": 0.020281746428214564, + "min": -0.10385652912378862, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.011216828476763308, + "action_delta": { + "distinct_n": 12, + "max": 0.08922043547542868, + "min": 0.0002683613903166071, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.0844557892569866, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010343278709820436, + "repeat_abs_p95": 0.07465434288403082, + "repeat_delta": { + "distinct_n": 24, + "max": 0.05015486062543706, + "min": -0.12688580277750028, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0041456974144474384, + "action_delta": { + "distinct_n": 12, + "max": 0.047957700246711504, + "min": -0.00037567596362376825, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.8378282569881378, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.004948147045494235, + "repeat_abs_p95": 0.06813512586166338, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07105816994418354, + "min": -0.09865864218773521, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0003854997463232923, + "action_delta": { + "distinct_n": 12, + "max": 0.012211161699714035, + "min": -0.0029429188195312372, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.04836213965959768, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.007971106097386826, + "repeat_abs_p95": 0.020160934134987728, + "repeat_delta": { + "distinct_n": 24, + "max": 0.011632967170489006, + "min": -0.023705586156158254, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.5689573993333328, + "action_delta": { + "distinct_n": 12, + "max": 27.59238343825, + "min": -0.6469868719999994, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.26512988706199775, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 2.1459572349166667, + "repeat_abs_p95": 7.856052630799997, + "repeat_delta": { + "distinct_n": 24, + "max": 6.557133349166667, + "min": -17.1293622325, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.40246922141666663, + "action_delta": { + "distinct_n": 11, + "max": 0.0, + "min": -19.377070914, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 10, + "positive": 0, + "zero": 2 + }, + "effect_to_repeat_median": 4.226054832868273, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.09523521045833332, + "repeat_abs_p95": 2.510103133283333, + "repeat_delta": { + "distinct_n": 18, + "max": 2.268429387666668, + "min": -11.883906298916667, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 7.958333333333329, + "action_delta": { + "distinct_n": 12, + "max": 11.75, + "min": -35.16666666666667, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.38047808764940205, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 20.91666666666667, + "repeat_abs_p95": 51.31666666666662, + "repeat_delta": { + "distinct_n": 24, + "max": 65.66666666666667, + "min": -34.58333333333333, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 0.0 + }, + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 22, + "max": 0.30654761904761907, + "median": 0.2795403407264545, + "min": 0.24313725490196078, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 30, + "max": 0.2681564245810056, + "median": 0.22518087736972248, + "min": 0.12849162011173185, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "completed_pass_rate", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.02825325884543762, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.02266666666666667, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.040253258845437616, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.03272660415517559, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.022522522522522515, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.07358436332279389, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.14270516717325227, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.12142857142857144, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.2182462927143778, + "train_balanced_accuracy": 0.75 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.03272660415517557, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.022522522522522515, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0735843633227939, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.37412821213021774, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.009857117327656123, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.008882265987138677, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": -0.004552157689988692, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.007167514027906635, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.006851562902858294, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.017755539496041216, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.007733676747496546, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.5040019049192779, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.007216190772090309, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.007216190772090309, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.24525856553394837, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "scheduler_steps_per_s", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.44507332514357945, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": -0.24667983170250252, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.5943046676188155, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.004225031845271454, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0004360066659713927, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": 0.0034270719814960106, + "train_balanced_accuracy": 0.875 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.00013357357159209972, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00013492200088597965, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.00018506887864057613, + "train_balanced_accuracy": 0.625 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": 0.0008649236257326498, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.0017369622931825424, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": 0.0005824031404205732, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 11.957967036527776, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.23078758088888907, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 18.57988346463889, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.18938916494444447, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.12479013208333334, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.23008102869444447, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 4.277777777777775, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.888888888888893, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 4.277777777777775, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.3, + "end_s": 18.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.6066330085464959, + "action_delta": { + "distinct_n": 12, + "max": 28.73606563075747, + "min": -1.1485291194593508, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.4934987464973646, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 1.2292493402508278, + "repeat_abs_p95": 8.067582719584564, + "repeat_delta": { + "distinct_n": 24, + "max": 8.686808262759335, + "min": -14.918642465494425, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 10.317307738579501, + "action_delta": { + "distinct_n": 12, + "max": 801.7642402753155, + "min": -13.613525901316592, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.17112975179373358, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 60.28938644763055, + "repeat_abs_p95": 340.3503302119946, + "repeat_delta": { + "distinct_n": 24, + "max": 374.64907917730886, + "min": -561.6272830402723, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.6035604856985604, + "action_delta": { + "distinct_n": 12, + "max": 28.411725240727222, + "min": -1.1421295113155576, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.4962266942146829, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.2162999144045272, + "repeat_abs_p95": 7.9623825938693935, + "repeat_delta": { + "distinct_n": 24, + "max": 8.570025092978447, + "min": -14.737105312913595, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.006690006870484877, + "action_delta": { + "distinct_n": 12, + "max": 0.013223015997172571, + "min": -0.1490562102409011, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.6364709072202164, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010511096099747042, + "repeat_abs_p95": 0.06858287350501925, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08329551900170162, + "min": -0.0738002419680065, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.01055037975803224, + "action_delta": { + "distinct_n": 12, + "max": 0.13907776271422118, + "min": -0.018167596600093858, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 8, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.9080980815382388, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.01161810598714273, + "repeat_abs_p95": 0.06969738634332051, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07468521754716137, + "min": -0.08558328606541879, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.0024714849986972203, + "action_delta": { + "distinct_n": 12, + "max": 0.006494393749041705, + "min": 0.000227090832439927, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.373420385875363, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0017995109320603211, + "repeat_abs_p95": 0.005547351274885883, + "repeat_delta": { + "distinct_n": 24, + "max": 0.005694038006766439, + "min": -0.0035699364880986415, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.010270226283016048, + "action_delta": { + "distinct_n": 12, + "max": 0.23522057920397632, + "min": -0.003022494939389775, + "n": 12 + }, + "action_signs": { + "consistency": 0.9166666666666666, + "negative": 1, + "positive": 11, + "zero": 0 + }, + "effect_to_repeat_median": 1.07628737944467, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.009542271403679525, + "repeat_abs_p95": 0.08564791687481275, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08662204016385266, + "min": -0.31416724947547214, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.015846588112988946, + "action_delta": { + "distinct_n": 12, + "max": 0.22224967428314468, + "min": 0.0002683613903166071, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.3242319261149558, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.011966625936500275, + "repeat_abs_p95": 0.10502297931861322, + "repeat_delta": { + "distinct_n": 24, + "max": 0.05015486062543706, + "min": -0.14511939254670791, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.004416664987538066, + "action_delta": { + "distinct_n": 12, + "max": 0.06671902028938567, + "min": -0.00038309683389285525, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 0.8499259269415633, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.005196529306302399, + "repeat_abs_p95": 0.03531051450496546, + "repeat_delta": { + "distinct_n": 24, + "max": 0.028898615902990022, + "min": -0.05744492686343953, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.001539018865967745, + "action_delta": { + "distinct_n": 12, + "max": 0.003050797976300035, + "min": -0.0015887887661509836, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.15591584638446954, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.009870830333516656, + "repeat_abs_p95": 0.021489430916148855, + "repeat_delta": { + "distinct_n": 24, + "max": 0.021795755533912664, + "min": -0.02021322592113628, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.5940786813055561, + "action_delta": { + "distinct_n": 12, + "max": 33.446711043166665, + "min": -0.6256704651111118, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.30447578852990603, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.9511524518055556, + "repeat_abs_p95": 4.689088691175, + "repeat_delta": { + "distinct_n": 24, + "max": 3.8720077572777782, + "min": -11.837926215166668, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.4166666666666667, + "action_abs_median": 0.9777240376944445, + "action_delta": { + "distinct_n": 11, + "max": 0.0, + "min": -28.609993227777778, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 10, + "positive": 0, + "zero": 2 + }, + "effect_to_repeat_median": 8.318675069172114, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.11753362519444445, + "repeat_abs_p95": 2.0560520443777768, + "repeat_delta": { + "distinct_n": 20, + "max": 1.6669221998333335, + "min": -9.630657473833335, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 5.527777777777779, + "action_delta": { + "distinct_n": 12, + "max": 9.166666666666671, + "min": -33.27777777777777, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.3338926174496646, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 16.55555555555555, + "repeat_abs_p95": 37.27222222222221, + "repeat_delta": { + "distinct_n": 24, + "max": 31.888888888888886, + "min": -41.611111111111114, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 0.0 + } + ], + "decision": "REQUIRES_UNCENSORED_PHASE_AWARE_PILOT", + "design": { + "available_deciles": [ + 0.1, + 0.2, + 0.3 + ], + "cumulative_and_nonoverlapping_blocks": true, + "decile_fraction": 0.1, + "maximum_common_end_s": 18.0, + "maximum_common_fraction": 0.3, + "select_best_horizon": false, + "trace_duration_s": 60.0 + }, + "provenance": { + "analysis_script": "/home/gahow/phd/aituner/runs/intervention-response-v2/analyze_existing.py", + "analysis_script_sha256": "c7eb2cd2d00f2b3afa62e22e7b2f1cce6b87ec51ebb7921ef7ffb9d13d28c50a", + "manifest": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/pilot-manifest.json", + "manifest_sha256": "86b249a087ab1bcd51cd8de3d1da6bf34afd41af977eb58c93229b941dbf5c8a", + "manifest_validation": { + "expected_trials": 36, + "matched_trials": 36, + "schema": "fidelity-prefix-pilot-manifest-v1" + }, + "p1_analysis_script": "/home/gahow/phd/aituner/runs/intervention-response-v0/analyze_p1.py", + "p1_analysis_script_sha256": "902adcba58bcb73f97c455fdea3710f6cb28fa8d8f4ea5bc4915143ed106b17a", + "run_root": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b", + "streams": [ + { + "bytes": 24501686, + "cell": "tp1_mns64", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp1_mns64/opprof/opprof-v1-dp0-pid234511-1784005631788937266.jsonl", + "sha256": "a813076afe677e5c13010b2ec44f46998794ca44cdf30ee81be836e3623f0068" + }, + { + "bytes": 25524027, + "cell": "tp1_mns8", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp1_mns8/opprof/opprof-v1-dp0-pid227734-1784005177936445360.jsonl", + "sha256": "1708759d2e77c42bff1940d43214fb2d927c3eebeda62d53a3fb7d04817e9b22" + }, + { + "bytes": 29431988, + "cell": "tp2_mns64", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp2_mns64/opprof/opprof-v1-dp0-pid247791-1784006495467408663.jsonl", + "sha256": "65355de883f2eb5d31be161d82b26dddc53c41a0a35a0fdb807777bc5b21c5ec" + }, + { + "bytes": 26036993, + "cell": "tp2_mns8", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp2_mns8/opprof/opprof-v1-dp0-pid241594-1784006096655191299.jsonl", + "sha256": "485e203d37d45fdd3c4de768b95da2050478f111f4d739b129d4f4ebd4737b70" + }, + { + "bytes": 17449143, + "cell": "tp4_mns16", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp4_mns16/opprof/opprof-v1-dp0-pid256090-1784007028237607411.jsonl", + "sha256": "59e98014d01bfc68d9f97112ecd782cb03571ad437382ed67416c7b85ae48c56" + }, + { + "bytes": 22674678, + "cell": "tp4_mns64", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp4_mns64/opprof/opprof-v1-dp0-pid262293-1784007376383569515.jsonl", + "sha256": "03e4f3cca3023a78e859b663703b9468dedf89e58320f0292e2973070e7616c8" + } + ], + "trial_inputs": [ + { + "cell": "tp1_mns64", + "early_stopped": true, + "elapsed_s": 19.448401608, + "level": "high", + "mns": 64, + "replicate": 1, + "request_count": 179, + "requests_sha256": "e970dcac20144cb1e13a35997377f6695a9c57d788a88bad7f18cf6c64e35cec", + "result_sha256": "3233686b60b920dcd6cc2f45193a81f3af2d46f781ef42c4d117630612084feb", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/high-rep1/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": true, + "elapsed_s": 56.362951594, + "level": "high", + "mns": 64, + "replicate": 2, + "request_count": 179, + "requests_sha256": "ec06878e2aefdc4853d972197cedc29ab94df7147abb2701e0d88b5dabf29728", + "result_sha256": "8215477a014f061c55e94d879a25bbf1676ee2ef1c5e81efadd23d20d337e07a", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/high-rep2/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": true, + "elapsed_s": 59.320339642, + "level": "high", + "mns": 64, + "replicate": 3, + "request_count": 179, + "requests_sha256": "18d72d1fe16136359e28a1909e395ffd553dbe3edd2c0fd93a9cc270a5ec6d60", + "result_sha256": "480a3b50acaf9b9c1902e2669cf3aa7723a6703a00778da269b5b9757990ceef", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/high-rep3/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": false, + "elapsed_s": 60.653813767, + "level": "low", + "mns": 64, + "replicate": 1, + "request_count": 122, + "requests_sha256": "44d729c1ace9b09fe2ea4c4e110bd0f95fee69f83edc78ee0f65256922dcd2c2", + "result_sha256": "d2c08e71c5d4e0dd99662bab8457bd1ad0a4aba6bda57794dbfce62e36720f62", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/low-rep1/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": false, + "elapsed_s": 61.821918535, + "level": "low", + "mns": 64, + "replicate": 2, + "request_count": 122, + "requests_sha256": "97de3cced64fd88a5c2dcd087ca6c29d8b09a0a43d7bf0e4e844ef246462465b", + "result_sha256": "f194db2f7abb995171cd24626e4e515c000481f81283c929b9a5b63f110f1506", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/low-rep2/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": false, + "elapsed_s": 59.372640014, + "level": "low", + "mns": 64, + "replicate": 3, + "request_count": 122, + "requests_sha256": "30f4623100c58580cfa4efe9a1998c63581c8e44511c556ef39cf5bd777883bf", + "result_sha256": "0be457aa2fc9ce95bdb92a6eb25544f5f61a9739932ee5afa40535d26456e348", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/low-rep3/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": true, + "elapsed_s": 24.256301395, + "level": "high", + "mns": 8, + "replicate": 1, + "request_count": 179, + "requests_sha256": "5f5bfa4a16846190f488d3369eed89ae895804145cea59197e6b544b703223fc", + "result_sha256": "f65e3dc91d17f5caab1f4aecf97d2c464b99f107231af2cfd96f5484880136ee", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/high-rep1/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": true, + "elapsed_s": 57.560632316, + "level": "high", + "mns": 8, + "replicate": 2, + "request_count": 179, + "requests_sha256": "e802d09057de349e5164c18314fcad09dc88c9673834fd6e5b100a57dd12fbf4", + "result_sha256": "4835a985dd6bb8dc50b53737baa980ac95a49e22f0b8a7326e1fb416c9b4e798", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/high-rep2/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": true, + "elapsed_s": 46.401123902, + "level": "high", + "mns": 8, + "replicate": 3, + "request_count": 179, + "requests_sha256": "cf1226185845cd8aa4e7ead0cb5c95c77eda62b84851a2ab579f984bbb88e8ac", + "result_sha256": "95d5103f2678eeb99c258870324566932e2f2853c3b6e0ee94bc6c538f6ed865", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/high-rep3/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": false, + "elapsed_s": 60.653530104, + "level": "low", + "mns": 8, + "replicate": 1, + "request_count": 122, + "requests_sha256": "0edc6095b16e5361a88b2765cc5d0f267ed16ff1d883e02fc3c58f57e59c1018", + "result_sha256": "8bc433109d148c21b7d158df95ec5d1fa4fa619243114563068e1e98c4241a29", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/low-rep1/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": false, + "elapsed_s": 62.156958699, + "level": "low", + "mns": 8, + "replicate": 2, + "request_count": 122, + "requests_sha256": "ca58a878401ac44cca0a9a595eab41659dafec2e923f7f2474e4f82ea30b2f3e", + "result_sha256": "42a56af136d1d3bdfc16d9cea092c2db0eaa4db2c85dceb21958c5e418122900", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/low-rep2/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": false, + "elapsed_s": 59.499655201, + "level": "low", + "mns": 8, + "replicate": 3, + "request_count": 122, + "requests_sha256": "8a72240f84bbeff9eabd63e7d1f7a4c19910660fc5e4335013a1deaf89408127", + "result_sha256": "0214f32cabbcdc1bad6c331c3945c505cad136fcc9a8336aeb6314b27474e279", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/low-rep3/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 60.842664607, + "level": "high", + "mns": 64, + "replicate": 1, + "request_count": 345, + "requests_sha256": "19537ec05f1e40543d850964db12234561954c209d55cd8cd0a1da716ab105fc", + "result_sha256": "49523b3269db649aa70072058666da3309ecabb93242cd504b6413f5de0f5d54", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/high-rep1/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 61.33597184, + "level": "high", + "mns": 64, + "replicate": 2, + "request_count": 345, + "requests_sha256": "d95e0dcc3d3f25af11b4d3da0ca133f647fd602753b804337a7131cc56e03059", + "result_sha256": "f37a42d9ea734adcb24892f86cbb19e2c51cee0baf8a6a63c5d51c4820bd28f2", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/high-rep2/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 60.925907661, + "level": "high", + "mns": 64, + "replicate": 3, + "request_count": 345, + "requests_sha256": "ccba3d0a18a7db8501b040855bda473ee55c519619d4b9fa2a105bda06300829", + "result_sha256": "55447419925f30397d19863e149890241c58863d0adaafd511a67757905cb58e", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/high-rep3/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 60.73903317, + "level": "low", + "mns": 64, + "replicate": 1, + "request_count": 235, + "requests_sha256": "8685f475baf247f7ce52a6a2df1183cd77a6f580cd57a3a87c407b4e96c0895f", + "result_sha256": "7a8f4e462d496bd56b73b0a686653cbe499a238df62a8c9a50db687d051f1c68", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/low-rep1/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 61.049384964, + "level": "low", + "mns": 64, + "replicate": 2, + "request_count": 235, + "requests_sha256": "016a76ab3dc015700f9ecb9f60a20fcc3b456c45270472fdd1d04e84ebd7021a", + "result_sha256": "368869326f44a0032985eff9d3abfa2306b93c25370caa05cee255a8c0208c0e", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/low-rep2/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 60.611333104, + "level": "low", + "mns": 64, + "replicate": 3, + "request_count": 235, + "requests_sha256": "5455d1cdeabd6e2937130ed8fc8d86c1aaa6f04cbf26c3cb69ad064be5f67d73", + "result_sha256": "2de5c4524f57b487b94b4fdba35defc2a566236fc3e31f7260b7a9f85bb7007c", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/low-rep3/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": true, + "elapsed_s": 25.452006333, + "level": "high", + "mns": 8, + "replicate": 1, + "request_count": 336, + "requests_sha256": "5d41a316f329e2efcc0f728cdd078377eb72fb9258738636db704bcfcca290aa", + "result_sha256": "466b3991190a232c984ba1b54fb6793cac1893dce196ee96f7238d70d49fcb74", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/high-rep1/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": true, + "elapsed_s": 24.770382696, + "level": "high", + "mns": 8, + "replicate": 2, + "request_count": 336, + "requests_sha256": "1b7d2cdcd639c00c443ae1b98287a7cc57323eb80c7f0e1bfc72f1ab5bf48760", + "result_sha256": "e9156d6871cc8cb3a20dce629cc2443df7473f3836b46eb0944d491ea5c4d44c", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/high-rep2/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": true, + "elapsed_s": 25.69871505, + "level": "high", + "mns": 8, + "replicate": 3, + "request_count": 336, + "requests_sha256": "5a68fda7c29477e5b3e66d70ad2b05b77be6c200cc1beea89d663a5729f31a2b", + "result_sha256": "f2116797b35057949acbf1f57898befac9ff18b94c808462f0969bdcf2446dd6", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/high-rep3/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": false, + "elapsed_s": 60.605998599, + "level": "low", + "mns": 8, + "replicate": 1, + "request_count": 229, + "requests_sha256": "455a1c21388700f0e3234c49d15ba3b0426b0d3b551c096b297793afa0d4602f", + "result_sha256": "13de25a73cc8c752245f169cfca1d64220f697c15b835a646ba969a1005047fb", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/low-rep1/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": false, + "elapsed_s": 61.925251312, + "level": "low", + "mns": 8, + "replicate": 2, + "request_count": 229, + "requests_sha256": "cae4b764ebb8714a6f5d2009bb79497420b388986a992c51410f1681810c14c6", + "result_sha256": "92f92a576e990946c4767a2107d2ce0e6ba1b066337911399360c199fae3f5e4", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/low-rep2/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": false, + "elapsed_s": 60.612475588, + "level": "low", + "mns": 8, + "replicate": 3, + "request_count": 229, + "requests_sha256": "cc8c01e4560411cd5b84cdc9bebdda9a86b88615ee1c39a996affe669d26c167", + "result_sha256": "c9a7d026bd89cc027c45b8f1899d56e21d53d4234e95db365a9492f670a78d36", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/low-rep3/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 25.468414615, + "level": "high", + "mns": 16, + "replicate": 1, + "request_count": 750, + "requests_sha256": "8695422a0f1e0e1c097a08a7f4c082f1ba2e3bb26b58caadae0afc7438aeca90", + "result_sha256": "c54a2bb5c1b78bdef3fa09c39f5974215370d6fc7c2b6d403a3d376d53a1d9eb", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/high-rep1/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 24.508063881, + "level": "high", + "mns": 16, + "replicate": 2, + "request_count": 750, + "requests_sha256": "aa0482b10414d911af227e2df172c19ecca0add2e0024dc88f85ffbd063854bb", + "result_sha256": "91e2491de4ca0868b667e7d126b755f03bdd78378681b8070a2bbc0c38fb9055", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/high-rep2/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 27.456202024, + "level": "high", + "mns": 16, + "replicate": 3, + "request_count": 750, + "requests_sha256": "3a23ffbe0e724648226152ef5433cc5825ad640c37cf47997c2f091210eceeec", + "result_sha256": "136dde0451f806841ff3996b081fe0bffac0a76f60c3d78b7516f2689e106008", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/high-rep3/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 48.952560958, + "level": "low", + "mns": 16, + "replicate": 1, + "request_count": 510, + "requests_sha256": "d7eaaefd35cc906f759dcaa7ac3c8543249a1a02f28572bec953e41e4379a29f", + "result_sha256": "11d9ec9d2a3358aed988f7b4b1014af087d61c4b5a72eb9076ae19bb5cc481cd", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/low-rep1/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 49.912113702, + "level": "low", + "mns": 16, + "replicate": 2, + "request_count": 510, + "requests_sha256": "7fdb90f679bba1747fbacc92fac2045c820e09965c755b20f6dafd4fd9e26a7a", + "result_sha256": "800e2aa6e9f9efef4be12ee778be6817364914d63ae0624775d02729089768d6", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/low-rep2/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 48.923086251, + "level": "low", + "mns": 16, + "replicate": 3, + "request_count": 510, + "requests_sha256": "b592fe254476957265fc9139aefdea77be4b5540f3c1b8a429c10ea6ccab9a6a", + "result_sha256": "cc1b1f7f5280c2892d0c6deac9df9f34ce126b0284e6ad893ffbf0ddbf27969f", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/low-rep3/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 61.434823743, + "level": "high", + "mns": 64, + "replicate": 1, + "request_count": 750, + "requests_sha256": "b4aeaf5b3175bd85fdac08a2ef657fb2278c6c4b36ea32ea417275ab6d55812d", + "result_sha256": "111a1042b614d05772888665a7cfbc86925b2fbeb52ffe19edb62a5bcfe2f513", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/high-rep1/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 61.130504059, + "level": "high", + "mns": 64, + "replicate": 2, + "request_count": 750, + "requests_sha256": "5d82b6885f5c654ae1ae126696df4bafe04f5be1c1109db24c7856b0456d9443", + "result_sha256": "6ed574c18723e265ab1327c5e43d3c76d37b3fc5711d06c77a51718e4ba16601", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/high-rep2/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 61.22613867, + "level": "high", + "mns": 64, + "replicate": 3, + "request_count": 750, + "requests_sha256": "5c5861d716181f06dc3f951ae0d6ee479b1b61fc21ebf24541c5b45053fcc167", + "result_sha256": "35b8f0465f6b0c197f71b5ecaf4e771e6613f86998ae44f8701ab9e4e1e440b4", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/high-rep3/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 61.123849552, + "level": "low", + "mns": 64, + "replicate": 1, + "request_count": 510, + "requests_sha256": "931d372ded1e3aaa9c67f06653fac8f79b4ee4f369028c1148163580386681ce", + "result_sha256": "325ce57f33133670a6761b5dffb831ed3b2f464755ff743b56976b4a974c1215", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/low-rep1/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 60.86041954, + "level": "low", + "mns": 64, + "replicate": 2, + "request_count": 510, + "requests_sha256": "72c8f70a240f79edcaa4eb0a2fe9bade8ac07093339c5b2d09c49ddd1cf27d16", + "result_sha256": "ff8b2283d02d5967893ca6225b561ae11011da6149d1f6cff7b4275c00170e1d", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/low-rep2/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 60.47213624, + "level": "low", + "mns": 64, + "replicate": 3, + "request_count": 510, + "requests_sha256": "8b2c779d2097c5e4038e94e1ac9b986e29f55d9acfd67eeb366c4e0e9270f6e0", + "result_sha256": "f026b69356701894fab184e0f2f7504a30b537982e823373899c9ab123a33891", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/low-rep3/result.json" + } + ] + }, + "sanity": { + "early_stopped": 15, + "elapsed_s": { + "distinct_n": 36, + "max": 62.156958699, + "median": 60.5390674195, + "min": 19.448401608, + "n": 36 + }, + "invariants": { + "all_intervals_uncensored": true, + "all_window_sanity_pass": true, + "expected_trial_count": true, + "manifest_trials_match": true, + "stream_provenance_consistent": true, + "trace_duration_consistent": true + }, + "red_flags": [], + "request_count": { + "distinct_n": 8, + "max": 750.0, + "median": 285.5, + "min": 122.0, + "n": 36 + }, + "stream_bytes": { + "distinct_n": 6, + "max": 29431988.0, + "median": 25012856.5, + "min": 17449143.0, + "n": 6 + }, + "trace_duration_s": { + "distinct_n": 1, + "max": 60.0, + "median": 60.0, + "min": 60.0, + "n": 36 + }, + "trials": 36 + }, + "schema": "intervention-response-phase-aware-existing-v2", + "status": "COMPLETE", + "trajectory": { + "blocks": [ + { + "end_s": 6.0, + "start_s": 0.0 + }, + { + "end_s": 12.0, + "start_s": 6.0 + }, + { + "end_s": 18.0, + "start_s": 12.0 + } + ], + "features": { + "batch_size.mean": { + "block_medians": [ + 6.414992146729492, + 7.937718404518684, + 7.938679599969923 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 44.717667092379735, + "median": 2.4047228794539084, + "min": 0.12661717921526972, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 44.717667092379735, + "median": 1.715194194802789, + "min": -6.818300947246247, + "n": 36 + }, + "first_to_last_pearson": 0.7466021228318502 + }, + "batch_tokens.mean": { + "block_medians": [ + 182.10234474017744, + 172.7298136645963, + 202.28603044732077 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 908.0537249893571, + "median": 75.02808983931541, + "min": 7.692264317468613, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 908.0537249893571, + "median": 21.47667280168706, + "min": -316.4167321876521, + "n": 36 + }, + "first_to_last_pearson": 0.7254256632404311 + }, + "decode_batch_size.mean": { + "block_medians": [ + 6.354735422986181, + 7.873722548380083, + 7.875311697892343 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 44.34555129842486, + "median": 2.4157231102242664, + "min": 0.15471898197242862, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 44.34555129842486, + "median": 1.7131997250467412, + "min": -6.730278183383879, + "n": 36 + }, + "first_to_last_pearson": 0.7458594110784589 + }, + "graph_full_share": { + "block_medians": [ + 0.9440933791043857, + 0.9375353988798221, + 0.9361517143943923 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.20131971051511277, + "median": 0.027226282812290892, + "min": 0.00023443249531129862, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.08227563742558686, + "median": -0.0017697738257967388, + "min": -0.20131971051511277, + "n": 36 + }, + "first_to_last_pearson": 0.6826036350375567 + }, + "graph_none_share": { + "block_medians": [ + 0.05080178541907753, + 0.05845283871165627, + 0.06118872118872119 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.19429544487015749, + "median": 0.01965769475771631, + "min": 0.0028422907600285444, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.19429544487015749, + "median": 0.0040749575702726006, + "min": -0.0846718334643753, + "n": 36 + }, + "first_to_last_pearson": 0.65716602522967 + }, + "graph_padding_fraction": { + "block_medians": [ + 0.005497687293445758, + 0.0075248839288535735, + 0.006051201250183472 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.014587933688096199, + "median": 0.002744262881772697, + "min": 4.7975018144524526e-05, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.014587933688096199, + "median": -0.0005530753213689365, + "min": -0.008320005718079814, + "n": 36 + }, + "first_to_last_pearson": 0.3098612628425477 + }, + "kv_usage_end_minus_start": { + "block_medians": [ + 0.023398338975546773, + -0.001777370408002521, + 0.0038338466257342163 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.262813467878909, + "median": 0.017891096563429187, + "min": 0.0015150201237521532, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.07218503346987704, + "median": -0.01752170592552832, + "min": -0.262813467878909, + "n": 36 + }, + "first_to_last_pearson": -0.012340699297270389 + }, + "kv_usage_max": { + "block_medians": [ + 0.03336296364308622, + 0.04022944403944284, + 0.030052448221737305 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.29553401938255563, + "median": 0.011325832339947661, + "min": 0.0002397506593143639, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.29553401938255563, + "median": 0.00083657676867116, + "min": -0.09986012588670201, + "n": 36 + }, + "first_to_last_pearson": 0.4134682913789121 + }, + "kv_usage_mean": { + "block_medians": [ + 0.016731908775651918, + 0.02129250131291721, + 0.01914443130373699 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.271100737119201, + "median": 0.009396606492664827, + "min": 0.0007061033361193698, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.271100737119201, + "median": 0.002361521286958815, + "min": -0.06598177225454588, + "n": 36 + }, + "first_to_last_pearson": 0.10554346287202872 + }, + "preemptions": { + "block_medians": [ + 0.0, + 0.0, + 0.0 + ], + "changed_trials": 0, + "first_to_last_abs_delta": { + "distinct_n": 1, + "max": 0.0, + "median": 0.0, + "min": 0.0, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 1, + "max": 0.0, + "median": 0.0, + "min": 0.0, + "n": 36 + }, + "first_to_last_pearson": null + }, + "prefill_token_fraction": { + "block_medians": [ + 0.9617744444596128, + 0.955926466887596, + 0.9488462859261657 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.0574568246654229, + "median": 0.013436705979377572, + "min": 0.00024055263508171443, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.008129115874074722, + "median": -0.013436705979377572, + "min": -0.0574568246654229, + "n": 36 + }, + "first_to_last_pearson": 0.23303776424770067 + }, + "queue_running_mean": { + "block_medians": [ + 6.85322045575, + 8.0, + 8.0 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 44.04634992016667, + "median": 2.4438409930833327, + "min": 0.5856487165000002, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 44.04634992016667, + "median": 1.8893361798333332, + "min": -4.201522480666667, + "n": 36 + }, + "first_to_last_pearson": 0.8338308544566243 + }, + "queue_waiting_mean": { + "block_medians": [ + 0.0, + 0.020209624833333332, + 0.008993971666666666 + ], + "changed_trials": 23, + "first_to_last_abs_delta": { + "distinct_n": 24, + "max": 39.884282579499995, + "median": 0.008993971666666666, + "min": 0.0, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 24, + "max": 39.884282579499995, + "median": 0.00026988233333333, + "min": -0.7860144426666666, + "n": 36 + }, + "first_to_last_pearson": 0.8749679224022264 + }, + "scheduler_steps_per_s": { + "block_medians": [ + 75.83333333333334, + 78.16666666666667, + 73.58333333333334 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 76.5, + "median": 28.250000000000004, + "min": 2.1666666666666714, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 73.33333333333333, + "median": -3.1666666666666714, + "min": -76.5, + "n": 36 + }, + "first_to_last_pearson": 0.40161538799446195 + } + }, + "trial_count": 36 + } +} diff --git a/runs/intervention-response-v2/pilot_controller.py b/runs/intervention-response-v2/pilot_controller.py new file mode 100644 index 0000000..98b5903 --- /dev/null +++ b/runs/intervention-response-v2/pilot_controller.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Serialized, resumable controller for the 300-second phase-aware pilot.""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +HERE = Path(__file__).resolve().parent +PHASE6 = HERE.parent / "opprof-phase6" +sys.path.insert(0, str(PHASE6)) + +import opprof_phase6_controller as base # noqa: E402 + + +SCHEMA = "intervention-response-phase-aware-pilot-state-v2" +SESSION_ESTIMATE_H20_HOURS = 1.25 +SAFETY_H20_HOURS = 0.20 +CLIENT_TIMEOUT_S = 450.0 + + +def atomic_json(path: Path, payload: Any) -> None: + base.atomic_json(path, payload) + + +def wait_all_idle(timeout_s: float = 30.0) -> None: + deadline = time.monotonic() + timeout_s + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + base.assert_all_idle() + return + except RuntimeError as error: + last_error = error + time.sleep(1.0) + raise last_error or RuntimeError("GPU idle timeout") + + +def configure(args: argparse.Namespace, manifest: dict[str, Any]) -> None: + base.WORKDIR = args.run_root.parent + base.RUN_ROOT = args.run_root + base.STATE = args.run_root / "controller-state.json" + base.SOURCE = args.vllm_source + base.VENV = args.venv + base.AITUNER = args.aituner_root + base.MODEL = args.model + base.CLIENT = args.client + base.GPU_LIMIT = float(manifest["budget"]["hard_cap_h20_hours"]) + base.MARKER = "intervention-response-phase-aware-v2" + base.CELLS = { + f"tp4_mns{mns}": {"tp": 4, "mns": int(mns)} + for mns in manifest["engine"]["mns_endpoints"] + } + + +def load_state(path: Path, hard_cap: float) -> dict[str, Any]: + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return { + "schema": SCHEMA, + "status": "initialized", + "hard_cap_h20_hours": hard_cap, + "gpu_hours_total": 0.0, + "completed_sessions": 0, + "sessions": {}, + "failures": [], + "started_at": time.time(), + } + + +def save_state(path: Path, state: dict[str, Any]) -> None: + atomic_json(path, state) + + +def append_echo(run_root: Path, line: str) -> None: + run_root.mkdir(parents=True, exist_ok=True) + with (run_root / "launch-echo.log").open("a", encoding="utf-8") as target: + target.write(line + "\n") + print(line, flush=True) + + +def remaining_projection(session_count: int, index: int) -> float: + return (session_count - index) * SESSION_ESTIMATE_H20_HOURS + SAFETY_H20_HOURS + + +def start_server( + *, session: dict[str, Any], index: int, run_root: Path +) -> dict[str, Any]: + cell = f"tp4_mns{int(session['mns'])}" + gpus = (0, 1, 2, 3) + session_root = run_root / "sessions" / str(session["session"]) + session_root.mkdir(parents=True, exist_ok=True) + port = 8950 + index + command = base.server_command(cell, gpus, port) + with (session_root / "commands.log").open("a", encoding="utf-8") as log: + log.write(f"SERVER {shlex.join(command)}\n") + server_log = (session_root / "server.log").open("ab", buffering=0) + environment = os.environ.copy() + environment.update( + { + "CUDA_VISIBLE_DEVICES": "0,1,2,3", + "VLLM_OPPROF_DIR": str(session_root / "opprof"), + "OPPROF_PHASE6_MARKER": base.MARKER, + "AITUNER_ROOT": str(base.AITUNER), + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "PYTHONUNBUFFERED": "1", + } + ) + server = subprocess.Popen( + command, + cwd=base.SOURCE, + env=environment, + stdout=server_log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + base.OWNED_PGIDS.add(server.pid) + return { + "cell": cell, + "gpus": gpus, + "port": port, + "dir": session_root, + "server": server, + "server_handle": server_log, + "spawned_at": time.time(), + "results": [], + } + + +def client_command( + entry: dict[str, Any], + *, + study: str, + anchor: float, + output: Path, + warmup: bool, +) -> list[str]: + config = base.CELLS[entry["cell"]] + command = [ + "taskset", + "-c", + base.cpu_mask(entry["gpus"]), + str(base.VENV / "bin/python"), + str(base.CLIENT), + "warmup" if warmup else "run-anchor", + "--study", + study, + "--cell", + entry["cell"], + "--anchor", + str(anchor), + "--tp", + str(config["tp"]), + "--mns", + str(config["mns"]), + "--base-url", + f"http://127.0.0.1:{entry['port']}", + "--result-dir", + str(output), + "--disable-slo-early-stop", + ] + return command + + +def run_client( + *, + entry: dict[str, Any], + role: str, + study: str, + selection: dict[str, Any], + output: Path, + state: dict[str, Any], + warmup: bool = False, +) -> dict[str, Any]: + command = client_command( + entry, + study=study, + anchor=float(selection["anchor"]), + output=output, + warmup=warmup, + ) + with (entry["dir"] / "commands.log").open("a", encoding="utf-8") as log: + log.write(f"CLIENT role={role} {shlex.join(command)}\n") + handle = (output.parent / f"{output.name}.log").open("ab", buffering=0) + environment = os.environ.copy() + environment.update({"AITUNER_ROOT": str(base.AITUNER), "PYTHONUNBUFFERED": "1"}) + process = subprocess.Popen( + command, + cwd=base.WORKDIR, + env=environment, + stdout=handle, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + deadline = time.monotonic() + (180.0 if warmup else CLIENT_TIMEOUT_S) + try: + while process.poll() is None: + if time.monotonic() > deadline: + raise TimeoutError(f"client timeout: {entry['cell']} {role}") + if entry["server"].poll() is not None: + raise RuntimeError(f"server exited during {entry['cell']} {role}") + base.assert_no_other_compute() + if state["gpu_hours_total"] + base.live_gpu_hours([entry]) >= base.GPU_LIMIT: + raise RuntimeError("phase-aware pilot H20-hour hard cap reached") + time.sleep(1.0) + except Exception: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=10.0) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=10.0) + raise + finally: + handle.close() + if process.returncode: + raise RuntimeError( + f"client failed: cell={entry['cell']} role={role} rc={process.returncode}" + ) + result = json.loads((output / "result.json").read_text(encoding="utf-8")) + validate_result( + result=result, + selection=selection, + role=role, + warmup=warmup, + ) + entry["results"].append( + { + "anchor": float(selection["anchor"]), + "dir": str(output), + "kind": result["kind"], + } + ) + return result + + +def validate_result( + *, result: dict[str, Any], selection: dict[str, Any], role: str, warmup: bool +) -> None: + if result.get("slo_early_stop_disabled") is not True: + raise RuntimeError(f"SLO early stop was not disabled: {role}") + if warmup: + if result["kind"] != "warmup" or int(result["selection"]["count"]) != 16: + raise RuntimeError(f"invalid warmup result: {role}") + if not all( + result["invariants"].get(key, False) + for key in ("warmup_16", "warmup_exact_16", "warmup_long") + ): + raise RuntimeError(f"warmup invariant failed: {role}") + return + if bool(result["early_stopped"]): + raise RuntimeError(f"uncensored run early-stopped: {role}") + if int(result["selection"]["count"]) != int(selection["selected_count"]): + raise RuntimeError(f"selection count mismatch: {role}") + for key, manifest_key in ( + ("request_id_order_sha256", "request_id_order_sha256"), + ("arrival_order_sha256", "arrival_order_sha256"), + ("raw_length_order_sha256", "input_length_order_sha256"), + ): + if result["selection"][key] != selection[manifest_key]: + raise RuntimeError(f"selection hash mismatch {key}: {role}") + if int(result["observed_count"]) != int(selection["selected_count"]): + raise RuntimeError(f"request accounting mismatch: {role}") + + +def execute_session( + *, + index: int, + session: dict[str, Any], + manifest: dict[str, Any], + run_root: Path, + state_path: Path, + state: dict[str, Any], +) -> None: + name = str(session["session"]) + if state["sessions"].get(name, {}).get("status") == "complete": + return + projection = remaining_projection(len(manifest["sessions"]), index) + if state["gpu_hours_total"] + projection > base.GPU_LIMIT: + state["status"] = "budget_projection_stop" + state["budget_stop"] = { + "before_session": name, + "spent_h20_hours": state["gpu_hours_total"], + "remaining_projection_h20_hours": projection, + "hard_cap_h20_hours": base.GPU_LIMIT, + } + save_state(state_path, state) + raise RuntimeError(f"projected pilot cost exceeds hard cap before {name}") + + replicate = str(session["replicate"]) + repetition = manifest["repetitions"][replicate] + echo = ( + f"PHASE_PILOT_SESSION_ECHO session={name} tp=4 mns={session['mns']} " + f"gpus=0-3 workload={manifest['source']['window_id']} duration_s=300 " + f"loads={','.join(repetition['load_order'])} disable_slo_early_stop=true " + f"spent_h20h={state['gpu_hours_total']:.6f} " + f"remaining_projection_h20h={projection:.3f} cap_h20h={base.GPU_LIMIT:.1f} " + f"manifest={run_root / 'pilot-manifest.json'}" + ) + append_echo(run_root, echo) + wait_all_idle() + session_state = { + "status": "starting", + "replicate": int(replicate), + "mns": int(session["mns"]), + "started_at": time.time(), + "runs": [], + } + state["status"] = "running" + state["sessions"][name] = session_state + save_state(state_path, state) + entry = start_server(session=session, index=index, run_root=run_root) + failure: Exception | None = None + try: + base.wait_ready(entry) + high = repetition["selections"]["high"] + session_state["status"] = "warmup" + save_state(state_path, state) + run_client( + entry=entry, + role="warmup", + study=repetition["study"], + selection=high, + output=entry["dir"] / "warmup", + state=state, + warmup=True, + ) + session_state["status"] = "burnin" + save_state(state_path, state) + burnin = manifest["burnin"] + burnin_result = run_client( + entry=entry, + role="burnin", + study=burnin["study"], + selection=burnin, + output=entry["dir"] / "burnin", + state=state, + ) + session_state["burnin"] = { + "pass_rate": burnin_result["pass_rate"], + "feasible": burnin_result["feasible"], + "elapsed_s": burnin_result["interval"]["elapsed_s"], + } + session_state["status"] = "measured" + save_state(state_path, state) + for level in repetition["load_order"]: + selection = repetition["selections"][level] + result = run_client( + entry=entry, + role=level, + study=repetition["study"], + selection=selection, + output=entry["dir"] / level, + state=state, + ) + session_state["runs"].append( + { + "level": level, + "selected_count": selection["selected_count"], + "offered_req_s_per_gpu": selection["offered_req_s_per_gpu"], + "pass_rate": result["pass_rate"], + "feasible": result["feasible"], + "elapsed_s": result["interval"]["elapsed_s"], + "early_stopped": result["early_stopped"], + } + ) + save_state(state_path, state) + session_state["status"] = "stopping" + save_state(state_path, state) + except Exception as error: # noqa: BLE001 + failure = error + finally: + try: + base.stop_entry(entry) + except Exception as error: # noqa: BLE001 + failure = failure or error + time.sleep(2.0) + try: + wait_all_idle() + except Exception as error: # noqa: BLE001 + failure = failure or error + + session_hours = base.live_gpu_hours([entry]) + state["gpu_hours_total"] += session_hours + session_state["gpu_hours"] = session_hours + if failure is not None: + session_state["status"] = "failed" + session_state["failure"] = repr(failure) + state["status"] = "failed" + state["failures"].append({"session": name, "failure": repr(failure)}) + save_state(state_path, state) + raise failure + validation = base.validate_cell(entry) + session_state["validation"] = validation + session_state["status"] = "complete" + session_state["completed_at"] = time.time() + state["completed_sessions"] += 1 + save_state(state_path, state) + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--manifest", type=Path, required=True) + result.add_argument("--run-root", type=Path, required=True) + result.add_argument("--aituner-root", type=Path, required=True) + result.add_argument("--vllm-source", type=Path, required=True) + result.add_argument("--venv", type=Path, required=True) + result.add_argument("--model", type=Path, required=True) + result.add_argument("--client", type=Path, required=True) + return result + + +def main() -> None: + args = parser().parse_args() + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": + raise RuntimeError("unexpected phase-aware pilot manifest schema") + if manifest["status"] != "PASS": + raise RuntimeError("phase-aware pilot manifest did not pass preflight") + args.run_root.mkdir(parents=True, exist_ok=True) + copied_manifest = args.run_root / "pilot-manifest.json" + if not copied_manifest.exists(): + atomic_json(copied_manifest, manifest) + configure(args, manifest) + state_path = args.run_root / "controller-state.json" + state = load_state(state_path, base.GPU_LIMIT) + state["status"] = "running" + save_state(state_path, state) + for index, session in enumerate(manifest["sessions"]): + execute_session( + index=index, + session=session, + manifest=manifest, + run_root=args.run_root, + state_path=state_path, + state=state, + ) + state["status"] = "complete" + state["completed_at"] = time.time() + save_state(state_path, state) + wait_all_idle() + print( + json.dumps( + { + "status": state["status"], + "completed_sessions": state["completed_sessions"], + "gpu_hours_total": state["gpu_hours_total"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v2/prepare_pilot.py b/runs/intervention-response-v2/prepare_pilot.py new file mode 100644 index 0000000..2b735d5 --- /dev/null +++ b/runs/intervention-response-v2/prepare_pilot.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Prepare the uncensored 300-second TP4 matched pilot on dash0.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import sys +from pathlib import Path +from typing import Any + + +AITUNER_ROOT = Path(os.environ.get("AITUNER_ROOT", Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(AITUNER_ROOT / "src")) + +from aituner.spec import load_study_spec # noqa: E402 +from aituner.trace import load_trace_requests, select_requests_for_threshold # noqa: E402 + + +SCHEMA = "intervention-response-phase-aware-pilot-manifest-v2" +TP = 4 +MNS_ENDPOINTS = (16, 64) +REPLAY_TIME_SCALE = 0.5 +EXPECTED_DURATION_S = 300.0 +SAFETY_DEADLINE_S = 360.0 +LOADS_REQ_S_GPU = {"low": 1.5, "mid": 2.125, "high": 3.125} +REPLICATE_ROLES = ("high1", "high2", "high3") +LOAD_ORDERS = { + 1: ("low", "mid", "high"), + 2: ("high", "low", "mid"), + 3: ("mid", "high", "low"), +} +SESSION_ORDER = ( + (1, 16), + (1, 64), + (2, 64), + (2, 16), + (3, 16), + (3, 64), +) + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def order_hash(values: list[str]) -> str: + return hashlib.sha256("\n".join(values).encode()).hexdigest() + + +def attainable_anchor(requests: list[Any], target_count: int) -> tuple[float, list[Any]]: + ordered = sorted(float(request.sampling_u) for request in requests) + if not ordered: + raise ValueError("no requests remain after study filtering") + if target_count <= 0 or target_count > len(ordered): + raise ValueError( + f"target count {target_count} is outside available range 1..{len(ordered)}" + ) + indices = sorted( + { + max(0, min(len(ordered) - 1, target_count - 1)), + max(0, min(len(ordered) - 1, target_count)), + } + ) + candidates = [] + for index in indices: + anchor = ordered[index] + selected = select_requests_for_threshold(requests, threshold=anchor) + candidates.append((abs(len(selected) - target_count), len(selected), anchor, selected)) + _error, _count, anchor, selected = min( + candidates, key=lambda item: (item[0], item[1], item[2]) + ) + return anchor, selected + + +def selection_record(selected: list[Any], *, duration_s: float) -> dict[str, Any]: + return { + "anchor": max(float(request.sampling_u) for request in selected), + "selected_count": len(selected), + "offered_req_s": len(selected) / duration_s, + "offered_req_s_per_gpu": len(selected) / duration_s / TP, + "request_id_order_sha256": order_hash([request.row_id for request in selected]), + "arrival_order_sha256": order_hash( + [f"{request.arrival_s:.12f}" for request in selected] + ), + "input_length_order_sha256": order_hash( + [str(request.prompt_tokens_hint) for request in selected] + ), + } + + +def materialize_study(source: Path, target: Path, *, replicate: int) -> Path: + payload = json.loads(source.read_text(encoding="utf-8")) + payload["study_id"] = f"phase-aware-telemetry-v2-rep{replicate}" + payload["hardware"]["host_candidates"] = ["dash0"] + payload["engine"]["engine_version"] = "0.24.1.dev3+opprof" + trace = payload["trace"] + trace["replay_time_scale"] = REPLAY_TIME_SCALE + trace["early_stop_max_lag_s"] = None + trace["early_stop_max_elapsed_s"] = SAFETY_DEADLINE_S + trace["restart_engine_after_early_stop"] = False + trace["adaptive_stop"] = {"enabled": False} + atomic_json(target, payload) + return target + + +def build_manifest( + *, base_manifest_path: Path, private_root: Path +) -> dict[str, Any]: + base = json.loads(base_manifest_path.read_text(encoding="utf-8")) + if base.get("schema") != "fidelity-prefix-pilot-manifest-v1": + raise ValueError("unexpected base P1 manifest schema") + + repetitions = {} + selection_hashes = [] + selected_ids_by_replicate: dict[int, set[str]] = {} + for replicate, role in enumerate(REPLICATE_ROLES, start=1): + source_study = Path(base["private"]["studies"][role]["tp4"]) + target_study = private_root / "studies" / f"rep{replicate}-tp4.json" + materialize_study(source_study, target_study, replicate=replicate) + study = load_study_spec(target_study) + window, requests = load_trace_requests(study, study_spec_path=target_study) + duration_s = float(window.window_end - window.window_start) + if not math.isclose( + duration_s, EXPECTED_DURATION_S, rel_tol=0.0, abs_tol=1e-9 + ): + raise ValueError( + f"rep{replicate}: replay duration {duration_s} != {EXPECTED_DURATION_S}" + ) + selections = {} + selected_ids_by_level: dict[str, set[str]] = {} + for level, target_rate in LOADS_REQ_S_GPU.items(): + target_count = round(target_rate * duration_s * TP) + anchor, selected = attainable_anchor(requests, target_count) + record = selection_record(selected, duration_s=duration_s) + record.update( + { + "anchor": anchor, + "target_count": target_count, + "target_req_s_per_gpu": target_rate, + } + ) + selections[level] = record + selected_ids_by_level[level] = {request.row_id for request in selected} + selection_hashes.append(record["request_id_order_sha256"]) + selected_ids_by_replicate[replicate] = selected_ids_by_level["high"] + repetitions[str(replicate)] = { + "source_role": role, + "study": str(target_study), + "study_sha256": sha256_file(target_study), + "duration_s": duration_s, + "load_order": list(LOAD_ORDERS[replicate]), + "selections": selections, + } + + burnin = base["cells"]["tp4_mns16"]["targets"]["low"]["selections"][ + "burnin" + ] + burnin = dict(burnin) + burnin["study_sha256"] = sha256_file(Path(burnin["study"])) + sessions = [ + { + "session": f"rep{replicate}-mns{mns}", + "replicate": replicate, + "mns": mns, + } + for replicate, mns in SESSION_ORDER + ] + invariants = { + "three_repetitions": len(repetitions) == 3, + "six_sessions": len(sessions) == 6, + "load_levels_three": all( + len(item["selections"]) == 3 for item in repetitions.values() + ), + "selection_hashes_unique": len(selection_hashes) == len(set(selection_hashes)), + "selection_sets_disjoint_across_repetitions": all( + not selected_ids_by_replicate[left] & selected_ids_by_replicate[right] + for left in selected_ids_by_replicate + for right in selected_ids_by_replicate + if left < right + ), + "all_counts_positive": all( + selection["selected_count"] > 0 + for item in repetitions.values() + for selection in item["selections"].values() + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + return { + "schema": SCHEMA, + "status": "PASS" if not red_flags else "STOP", + "source": { + "base_manifest": str(base_manifest_path), + "base_manifest_sha256": sha256_file(base_manifest_path), + "window_id": base["source"]["window_id"], + "source_trace": base["source"]["trace"], + "source_trace_sha256": base["source"]["trace_sha256"], + }, + "engine": { + "tp": TP, + "mns_endpoints": list(MNS_ENDPOINTS), + "replay_time_scale": REPLAY_TIME_SCALE, + "duration_s": EXPECTED_DURATION_S, + "safety_deadline_s": SAFETY_DEADLINE_S, + "disable_slo_early_stop": True, + }, + "burnin": burnin, + "repetitions": repetitions, + "sessions": sessions, + "checkpoints": { + "fractions": [0.1, 0.25, 0.5, 0.75, 1.0], + "seconds": [30.0, 75.0, 150.0, 225.0, 300.0], + }, + "budget": { + "hard_cap_h20_hours": 8.0, + "expected_wall_minutes": [95, 120], + "expected_h20_hours": [6.3, 8.0], + }, + "sanity": { + "red_flags": red_flags, + "invariants": invariants, + "selected_sets": len(selection_hashes), + "distinct_selected_sets": len(set(selection_hashes)), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-manifest", type=Path, required=True) + parser.add_argument("--private-root", type=Path, required=True) + parser.add_argument("--public-manifest", type=Path, required=True) + args = parser.parse_args() + manifest = build_manifest( + base_manifest_path=args.base_manifest, private_root=args.private_root + ) + atomic_json(args.public_manifest, manifest) + print( + json.dumps( + { + "status": manifest["status"], + "manifest": str(args.public_manifest), + "sanity": manifest["sanity"], + }, + sort_keys=True, + ) + ) + if manifest["status"] != "PASS": + raise RuntimeError(f"phase-aware pilot preflight failed: {manifest['sanity']}") + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v2/test_analysis.py b/runs/intervention-response-v2/test_analysis.py new file mode 100644 index 0000000..db13f56 --- /dev/null +++ b/runs/intervention-response-v2/test_analysis.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import math +from pathlib import Path +from types import SimpleNamespace + + +HERE = Path(__file__).resolve().parent + + +def load_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_phase_aware_v2", HERE / "analyze_existing.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def load_prepare_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_phase_aware_prepare", HERE / "prepare_pilot.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def load_controller_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_phase_aware_controller", HERE / "pilot_controller.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def load_pilot_analysis_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_phase_aware_pilot_analysis", HERE / "analyze_pilot.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def main() -> None: + module = load_module() + assert module.common_decile_fractions( + trace_duration_s=60.0, minimum_elapsed_s=19.448 + ) == (0.1, 0.2, 0.3) + assert module.common_decile_fractions( + trace_duration_s=60.0, minimum_elapsed_s=60.0 + )[-1] == 1.0 + stats = module.numeric([0.0, 1.0, 2.0]) + assert stats == { + "n": 3, + "min": 0.0, + "max": 2.0, + "distinct_n": 3, + "median": 1.0, + } + assert math.isclose(module._pearson([1.0, 2.0], [2.0, 4.0]), 1.0) + assert module._pearson([1.0, 1.0], [2.0, 3.0]) is None + prepare = load_prepare_module() + requests = [ + SimpleNamespace( + sampling_u=index / 10.0, + row_id=f"r{index}", + arrival_s=float(index), + prompt_tokens_hint=100 + index, + ) + for index in range(1, 6) + ] + _anchor, selected = prepare.attainable_anchor(requests, 3) + assert len(selected) == 3 + record = prepare.selection_record(selected, duration_s=3.0) + assert record["selected_count"] == 3 + assert record["offered_req_s_per_gpu"] == 0.25 + assert len(prepare.SESSION_ORDER) == 6 + assert {mns for _replicate, mns in prepare.SESSION_ORDER} == {16, 64} + controller = load_controller_module() + assert math.isclose(controller.remaining_projection(6, 0), 7.7) + assert math.isclose(controller.remaining_projection(6, 5), 1.45) + pilot_analysis = load_pilot_analysis_module() + stable = pilot_analysis.stable_adjacent_features( + [ + {"end_fraction": 0.1, "qualifying_response_features": ["queue"]}, + { + "end_fraction": 0.25, + "qualifying_response_features": ["kv", "queue"], + }, + {"end_fraction": 0.5, "qualifying_response_features": ["queue"]}, + ] + ) + assert stable == {"0.10->0.25": ["queue"], "0.25->0.50": ["queue"]} + print("phase-aware intervention response v2 analysis: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/opprof-phase6/opprof_phase6_client.py b/runs/opprof-phase6/opprof_phase6_client.py index 1dde17b..600fb3a 100644 --- a/runs/opprof-phase6/opprof_phase6_client.py +++ b/runs/opprof-phase6/opprof_phase6_client.py @@ -95,7 +95,11 @@ def run_replay(args: argparse.Namespace, *, warmup: bool) -> dict[str, Any]: base_url=args.base_url, timeout_s=study.engine.request_timeout_s, max_concurrency=study.trace.max_concurrency, - target_pass_rate=(0.0 if warmup else study.slo.target_pass_rate), + target_pass_rate=( + 0.0 + if warmup or args.disable_slo_early_stop + else study.slo.target_pass_rate + ), max_lag_s=study.trace.early_stop_max_lag_s, max_elapsed_s=( 120.0 if warmup else _probe_drain_deadline( @@ -162,6 +166,7 @@ def run_replay(args: argparse.Namespace, *, warmup: bool) -> dict[str, Any]: "feasible": bool(slo_summary["feasible"]), "early_stopped": early_stopped, "early_stop_reason": early_stop_reason, + "slo_early_stop_disabled": bool(args.disable_slo_early_stop), "ttft_ms": numeric([item.ttft_ms for item in outcomes]), "tpot_ms": numeric([item.tpot_ms for item in outcomes]), "invariants": { @@ -240,6 +245,7 @@ def parser() -> argparse.ArgumentParser: q.add_argument("--mns", type=int, required=True) q.add_argument("--base-url", required=True) q.add_argument("--result-dir", required=True) + q.add_argument("--disable-slo-early-stop", action="store_true") return p diff --git a/runs/opprof-phase6/test_phase6_tools.py b/runs/opprof-phase6/test_phase6_tools.py index 0cc338d..6c71c47 100644 --- a/runs/opprof-phase6/test_phase6_tools.py +++ b/runs/opprof-phase6/test_phase6_tools.py @@ -20,6 +20,7 @@ def main() -> None: controller = load("p6controller", HERE / "opprof_phase6_controller.py") solo = load("p6solo", HERE / "opprof_phase6_solo_controller.py") analysis = load("p6analysis", HERE / "analyze_phase6.py") + client = load("p6client", HERE / "opprof_phase6_client.py") assert len(controller.CELLS) == 12 primary = sum(3 if item.get("trap") else 2 for item in controller.CELLS.values()) assert primary == 25 @@ -36,6 +37,18 @@ def main() -> None: assert sum(solo.CELL_ESTIMATE.values()) + solo.SAFETY_HOURS == 3.44 assert solo.next_below([.1, .2, .3], {.2, .3}) == .1 assert solo.next_above([.1, .2, .3], {.1, .2}) == .3 + parsed = client.parser().parse_args([ + "run-anchor", + "--study", "study.json", + "--cell", "tp4_mns16", + "--anchor", "0.5", + "--tp", "4", + "--mns", "16", + "--base-url", "http://127.0.0.1:8000", + "--result-dir", "out", + "--disable-slo-early-stop", + ]) + assert parsed.disable_slo_early_stop is True print("phase6 tools: PASS")