diff --git a/runs/frontier-attn-structured-v0/.gitignore b/runs/frontier-attn-structured-v0/.gitignore new file mode 100644 index 0000000..e840b79 --- /dev/null +++ b/runs/frontier-attn-structured-v0/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +cache/ +replay/ diff --git a/runs/frontier-attn-structured-v0/0001-Experiment-with-structured-attention-prefill-predict.patch b/runs/frontier-attn-structured-v0/0001-Experiment-with-structured-attention-prefill-predict.patch new file mode 100644 index 0000000..ba761a4 --- /dev/null +++ b/runs/frontier-attn-structured-v0/0001-Experiment-with-structured-attention-prefill-predict.patch @@ -0,0 +1,249 @@ +From 1f8900a4ac64e45754b03d0aa7c1dddab65785cf Mon Sep 17 00:00:00 2001 +From: Gahow Wang +Date: Thu, 23 Jul 2026 15:27:40 +0800 +Subject: [PATCH] Experiment with structured attention prefill predictor + +--- + .../shared_prediction_model_manager.py | 16 +++- + .../sklearn_execution_time_predictor.py | 16 +++- + .../structured_attention_prefill.py | 79 +++++++++++++++++++ + .../unit/test_structured_attention_prefill.py | 59 ++++++++++++++ + 4 files changed, 165 insertions(+), 5 deletions(-) + create mode 100644 frontier/execution_time_predictor/structured_attention_prefill.py + create mode 100644 tests/unit/test_structured_attention_prefill.py + +diff --git a/frontier/execution_time_predictor/shared_prediction_model_manager.py b/frontier/execution_time_predictor/shared_prediction_model_manager.py +index 8a65a49..4a21165 100644 +--- a/frontier/execution_time_predictor/shared_prediction_model_manager.py ++++ b/frontier/execution_time_predictor/shared_prediction_model_manager.py +@@ -19,6 +19,9 @@ from frontier.execution_time_predictor.attention_tp_policy import ( + from frontier.execution_time_predictor.attention_dataset_contract import ( + enforce_mixed_attention_input_contract, + ) ++from frontier.execution_time_predictor.structured_attention_prefill import ( ++ StructuredAttentionPrefillRegressor, ++) + from frontier.logger import init_logger + from frontier.moe_gating_runtime import ( + DEFAULT_MOE_GATING_RUNTIME_CONTEXT, +@@ -1254,7 +1257,10 @@ class ExecutionTimePredictionModelManager: + raise ValueError( + "Missing required column 'prefill_chunk_size' in attention profiling data." + ) +- standard_prefill_df = prefill_df[prefill_df["prefill_chunk_size"] > 0].copy() ++ standard_prefill_df = prefill_df[ ++ (prefill_df["prefill_chunk_size"] > 0) ++ & (prefill_df["batch_size"] == 1) ++ ].copy() + + prefill_model_signature = f"attn_prefill_{attention_signature}" + if prefill_model_signature not in trained_model_signatures: +@@ -1742,7 +1748,13 @@ class ExecutionTimePredictionModelManager: + # initialization to generate missing cache files. + # ============================================================ + +- estimator, grid_search_params = self._create_estimator_and_params(execution_time_predictor_config) ++ if model_name == "attn_prefill": ++ estimator = StructuredAttentionPrefillRegressor() ++ grid_search_params = {} ++ else: ++ estimator, grid_search_params = self._create_estimator_and_params( ++ execution_time_predictor_config ++ ) + + cv = min(execution_time_predictor_config.k_fold_cv_splits, len(df)) if len(df) >= 2 else 2 + +diff --git a/frontier/execution_time_predictor/sklearn_execution_time_predictor.py b/frontier/execution_time_predictor/sklearn_execution_time_predictor.py +index 27b62bf..b7f5350 100644 +--- a/frontier/execution_time_predictor/sklearn_execution_time_predictor.py ++++ b/frontier/execution_time_predictor/sklearn_execution_time_predictor.py +@@ -45,6 +45,9 @@ from frontier.execution_time_predictor.attention_tp_policy import ( + from frontier.execution_time_predictor.attention_dataset_contract import ( + enforce_mixed_attention_input_contract, + ) ++from frontier.execution_time_predictor.structured_attention_prefill import ( ++ StructuredAttentionPrefillRegressor, ++) + from frontier.logger import init_logger + from frontier.moe_gating_runtime import get_moe_gating_base_model_name + from frontier.profiling.cpu_overhead.schema import ( +@@ -2573,8 +2576,12 @@ class SklearnExecutionTimePredictor(BaseExecutionTimePredictor): + if cached_model: + return cached_model + +- model = self._get_estimator() +- grid_search_params = self._get_grid_search_params() ++ if model_name == "attn_prefill": ++ model = StructuredAttentionPrefillRegressor() ++ grid_search_params = {} ++ else: ++ model = self._get_estimator() ++ grid_search_params = self._get_grid_search_params() + + if len(df) < self._config.k_fold_cv_splits: + cv = 2 +@@ -2869,7 +2876,10 @@ class SklearnExecutionTimePredictor(BaseExecutionTimePredictor): + raise ValueError( + "Missing required column 'prefill_chunk_size' in attention profiling data." + ) +- standard_prefill_df = prefill_df[prefill_df["prefill_chunk_size"] > 0].copy() ++ standard_prefill_df = prefill_df[ ++ (prefill_df["prefill_chunk_size"] > 0) ++ & (prefill_df["batch_size"] == 1) ++ ].copy() + if len(standard_prefill_df) == 0: + raise ValueError( + "No standard prefill rows (prefill_chunk_size > 0) found in eager attention profiling data." +diff --git a/frontier/execution_time_predictor/structured_attention_prefill.py b/frontier/execution_time_predictor/structured_attention_prefill.py +new file mode 100644 +index 0000000..1829047 +--- /dev/null ++++ b/frontier/execution_time_predictor/structured_attention_prefill.py +@@ -0,0 +1,79 @@ ++"""Structured latency model for single-request chunked prefill attention.""" ++ ++from typing import Any ++ ++import numpy as np ++from sklearn.base import BaseEstimator, RegressorMixin ++from sklearn.isotonic import IsotonicRegression ++from sklearn.linear_model import LinearRegression ++ ++ ++class StructuredAttentionPrefillRegressor(RegressorMixin, BaseEstimator): ++ """Model attention as a monotone base curve plus continuous KV growth. ++ ++ Input columns retain the existing Frontier contract: ++ ``[kv_cache_size, prefill_chunk_size_squared]``. ++ """ ++ ++ def fit(self, X: Any, y: Any) -> "StructuredAttentionPrefillRegressor": ++ values = self._as_feature_array(X) ++ target = np.asarray(y, dtype=float) ++ kv_cache_size = values[:, 0] ++ prefill_chunk_size = np.sqrt(np.maximum(values[:, 1], 0.0)) ++ ++ base_mask = np.isclose(kv_cache_size, 0.0) ++ growth_mask = kv_cache_size > 0.0 ++ if not np.any(base_mask) or not np.any(growth_mask): ++ raise ValueError( ++ "structured attn_prefill training requires both KV=0 base rows " ++ "and KV>0 growth rows" ++ ) ++ ++ base_q = prefill_chunk_size[base_mask] ++ base_y = target[base_mask] ++ unique_q = np.unique(base_q) ++ grouped_y = np.asarray( ++ [np.mean(base_y[np.isclose(base_q, q)]) for q in unique_q], ++ dtype=float, ++ ) ++ self._base_model = IsotonicRegression( ++ increasing=True, ++ out_of_bounds="clip", ++ ).fit(unique_q, grouped_y) ++ ++ growth_q = prefill_chunk_size[growth_mask] ++ growth_kv = kv_cache_size[growth_mask] ++ growth_base = self._base_model.predict(growth_q) ++ growth_features = np.column_stack( ++ (growth_kv, growth_q * growth_kv) ++ ) ++ self._growth_model = LinearRegression( ++ fit_intercept=False, ++ positive=True, ++ ).fit(growth_features, target[growth_mask] - growth_base) ++ ++ self.n_features_in_ = 2 ++ self._frontier_base_q_min = float(unique_q.min()) ++ self._frontier_base_q_max = float(unique_q.max()) ++ self._frontier_growth_kv_max = float(growth_kv.max()) ++ return self ++ ++ def predict(self, X: Any) -> np.ndarray: ++ values = self._as_feature_array(X) ++ kv_cache_size = values[:, 0] ++ prefill_chunk_size = np.sqrt(np.maximum(values[:, 1], 0.0)) ++ base = self._base_model.predict(prefill_chunk_size) ++ growth_features = np.column_stack( ++ (kv_cache_size, prefill_chunk_size * kv_cache_size) ++ ) ++ return np.maximum(base + self._growth_model.predict(growth_features), 0.0) ++ ++ @staticmethod ++ def _as_feature_array(X: Any) -> np.ndarray: ++ values = np.asarray(X, dtype=float) ++ if values.ndim != 2 or values.shape[1] != 2: ++ raise ValueError( ++ "structured attn_prefill expects exactly two features: " ++ "kv_cache_size and prefill_chunk_size_squared" ++ ) ++ return values +diff --git a/tests/unit/test_structured_attention_prefill.py b/tests/unit/test_structured_attention_prefill.py +new file mode 100644 +index 0000000..12c4247 +--- /dev/null ++++ b/tests/unit/test_structured_attention_prefill.py +@@ -0,0 +1,59 @@ ++import pickle ++import unittest ++ ++import numpy as np ++ ++from frontier.execution_time_predictor.structured_attention_prefill import ( ++ StructuredAttentionPrefillRegressor, ++) ++ ++ ++class StructuredAttentionPrefillRegressorTest(unittest.TestCase): ++ def setUp(self) -> None: ++ q = np.asarray([64, 128, 256, 512, 1024, 2048, 4096, 8192], dtype=float) ++ base = 0.05 + 1e-4 * q + 4e-8 * q**2 ++ context_q = np.asarray([2048, 4096, 8192] * 3, dtype=float) ++ context_kv = np.repeat([8192, 16384, 24576], 3).astype(float) ++ context_y = ( ++ np.interp(context_q, q, base) ++ + 1.5e-5 * context_kv ++ + 3e-8 * context_q * context_kv ++ ) ++ self.X = np.column_stack( ++ ( ++ np.concatenate((np.zeros_like(q), context_kv)), ++ np.concatenate((q**2, context_q**2)), ++ ) ++ ) ++ self.y = np.concatenate((base, context_y)) ++ ++ def test_recovers_structured_curve(self) -> None: ++ model = StructuredAttentionPrefillRegressor().fit(self.X, self.y) ++ np.testing.assert_allclose(model.predict(self.X), self.y, rtol=1e-6) ++ ++ def test_prediction_is_nonnegative_and_monotone(self) -> None: ++ model = StructuredAttentionPrefillRegressor().fit(self.X, self.y) ++ q = np.arange(1, 8193, dtype=float) ++ for kv in (0, 8192, 32768, 40912): ++ X = np.column_stack((np.full_like(q, kv), q**2)) ++ prediction = model.predict(X) ++ self.assertTrue(np.all(prediction >= 0)) ++ self.assertTrue(np.all(np.diff(prediction) >= -1e-12)) ++ ++ kv = np.arange(0, 40913, 64, dtype=float) ++ for q_value in (64, 2048, 8192): ++ X = np.column_stack((kv, np.full_like(kv, q_value**2))) ++ self.assertTrue(np.all(np.diff(model.predict(X)) >= -1e-12)) ++ ++ def test_pickle_round_trip(self) -> None: ++ model = StructuredAttentionPrefillRegressor().fit(self.X, self.y) ++ restored = pickle.loads(pickle.dumps(model)) ++ np.testing.assert_allclose(restored.predict(self.X), self.y, rtol=1e-6) ++ ++ def test_requires_base_and_growth_rows(self) -> None: ++ with self.assertRaisesRegex(ValueError, "KV=0 base rows"): ++ StructuredAttentionPrefillRegressor().fit(self.X[:8], self.y[:8]) ++ ++ ++if __name__ == "__main__": ++ unittest.main() +-- +2.43.0 + diff --git a/runs/frontier-attn-structured-v0/analyze_predictor_ablation.py b/runs/frontier-attn-structured-v0/analyze_predictor_ablation.py new file mode 100644 index 0000000..4eb79ce --- /dev/null +++ b/runs/frontier-attn-structured-v0/analyze_predictor_ablation.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Offline predictor ablation for EXP-ATTN-STRUCTURED. + +This is deliberately profile-only: it decides whether the structured model is +good enough to justify the expensive 7-cell trace replay. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.ensemble import RandomForestRegressor + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--profile", + type=Path, + default=REPO + / "runs/frontier-prefill-kvgrowth-fix-v0/profiles/" + "profile-v5-kvgrowth/attention.csv", + ) + parser.add_argument( + "--frontier-checkout", + type=Path, + default=Path("/tmp/frontier-attn-structured-v0"), + ) + parser.add_argument("--output-root", type=Path, default=ROOT / "results") + return parser.parse_args() + + +def normalize_bool(series: pd.Series) -> pd.Series: + return series.astype(str).str.strip().str.lower().isin( + {"1", "true", "t", "yes", "y"} + ) + + +def load_profile(path: Path) -> pd.DataFrame: + df = pd.read_csv(path).drop_duplicates() + for column in ("is_prefill", "is_true_mixed_batch"): + df[column] = normalize_bool(df[column]) + df = df[ + (df["n_embd"] == 2048) + & (df["n_q_head"] == 32) + & (df["n_kv_head"] == 4) + & (df["block_size"] == 16) + & df["is_prefill"] + & ~df["is_true_mixed_batch"] + & (df["prefill_chunk_size"] > 0) + ].copy() + df["prefill_chunk_size_squared"] = df["prefill_chunk_size"] ** 2 + return df + + +def mape(actual: np.ndarray, predicted: np.ndarray) -> float: + return float(np.mean(np.abs((predicted - actual) / actual)) * 100) + + +def make_rf() -> RandomForestRegressor: + # Exact best parameters selected by the current profile-v5 GridSearchCV. + return RandomForestRegressor( + random_state=0, + n_estimators=250, + max_depth=8, + min_samples_split=2, + ) + + +def features(df: pd.DataFrame) -> pd.DataFrame: + return df[["kv_cache_size", "prefill_chunk_size_squared"]] + + +def score_model( + name: str, + estimator: Any, + train: pd.DataFrame, + single: pd.DataFrame, + grid: pd.DataFrame, +) -> dict[str, Any]: + target = "time_stats.attn_prefill.median" + estimator.fit(features(train), train[target]) + grid_prediction = estimator.predict(features(grid)) + single_prediction = estimator.predict(features(single)) + + heldout_actual: list[float] = [] + heldout_prediction: list[float] = [] + for context in sorted(grid["kv_cache_size"].unique()): + test = grid[grid["kv_cache_size"] == context] + fold_train = train.drop(index=test.index, errors="ignore") + fold_model = ( + make_rf() + if name.startswith("rf") + else estimator.__class__() + ) + fold_model.fit(features(fold_train), fold_train[target]) + heldout_actual.extend(test[target].astype(float)) + heldout_prediction.extend(fold_model.predict(features(test))) + + q = np.arange(1, 8193, dtype=float) + q_deltas: list[float] = [] + prediction_min: list[float] = [] + for context in (0, 8192, 16384, 24576, 32768, 40912): + X = pd.DataFrame( + { + "kv_cache_size": np.full_like(q, context), + "prefill_chunk_size_squared": q**2, + } + ) + prediction = estimator.predict(X) + prediction_min.append(float(prediction.min())) + q_deltas.append(float(np.diff(prediction).min())) + + kv = np.arange(0, 40913, 64, dtype=float) + kv_deltas: list[float] = [] + for query in (64, 512, 2048, 4096, 8192): + X = pd.DataFrame( + { + "kv_cache_size": kv, + "prefill_chunk_size_squared": np.full_like(kv, query**2), + } + ) + kv_deltas.append(float(np.diff(estimator.predict(X)).min())) + + heldout_actual_array = np.asarray(heldout_actual) + heldout_prediction_array = np.asarray(heldout_prediction) + return { + "candidate": name, + "training_rows": len(train), + "grid_fit_mape_pct": mape( + grid[target].to_numpy(), np.asarray(grid_prediction) + ), + "single_fit_mape_pct": mape( + single[target].to_numpy(), np.asarray(single_prediction) + ), + "heldout_context_mape_pct": mape( + heldout_actual_array, heldout_prediction_array + ), + "heldout_context_max_abs_error_pct": float( + np.max( + np.abs( + (heldout_prediction_array - heldout_actual_array) + / heldout_actual_array + ) + ) + * 100 + ), + "prediction_min_ms": min(prediction_min), + "q_min_delta_ms": min(q_deltas), + "kv_min_delta_ms": min(kv_deltas), + "monotone_and_nonnegative": ( + min(prediction_min) >= 0 + and min(q_deltas) >= -1e-12 + and min(kv_deltas) >= -1e-12 + ), + } + + +def main() -> None: + args = parse_args() + sys.path.insert(0, str(args.frontier_checkout)) + from frontier.execution_time_predictor.structured_attention_prefill import ( + StructuredAttentionPrefillRegressor, + ) + + df = load_profile(args.profile) + records: list[dict[str, Any]] = [] + data_audit: dict[str, Any] = {} + for tp in (1, 2, 4): + tp_df = df[df["num_tensor_parallel_workers"] == tp].copy() + single = tp_df[tp_df["batch_size"] == 1].copy() + grid = single[ + single["prefill_chunk_size"].isin((2048, 4096, 8192)) + & (single["kv_cache_size"] > 0) + ].copy() + duplicate_groups = ( + tp_df.groupby( + ["kv_cache_size", "prefill_chunk_size_squared"] + ) + .size() + .gt(1) + .sum() + ) + data_audit[f"tp{tp}"] = { + "standard_rows": len(tp_df), + "single_request_rows": len(single), + "target_grid_rows": len(grid), + "duplicate_feature_groups": int(duplicate_groups), + } + + candidates = ( + ("rf_all", make_rf(), tp_df), + ("rf_single", make_rf(), single), + ( + "structured_single", + StructuredAttentionPrefillRegressor(), + single, + ), + ) + for name, model, train in candidates: + result = score_model(name, model, train, single, grid) + result["tp"] = tp + records.append(result) + + structured = [r for r in records if r["candidate"] == "structured_single"] + checks = { + "heldout_context_mape_le_5pct": all( + r["heldout_context_mape_pct"] <= 5 for r in structured + ), + "monotone_and_nonnegative": all( + r["monotone_and_nonnegative"] for r in structured + ), + } + checks["profile_gate"] = all(checks.values()) + payload = { + "schema": "frontier-attn-structured-ablation-v1", + "profile": str(args.profile.resolve()), + "frontier_checkout": str(args.frontier_checkout.resolve()), + "data_audit": data_audit, + "results": records, + "checks": checks, + } + + args.output_root.mkdir(parents=True, exist_ok=True) + (args.output_root / "predictor-ablation.json").write_text( + json.dumps(payload, indent=2) + ) + with (args.output_root / "predictor-ablation.csv").open( + "w", newline="" + ) as stream: + writer = csv.DictWriter(stream, fieldnames=list(records[0])) + writer.writeheader() + writer.writerows(records) + print(json.dumps(checks, indent=2)) + for row in records: + print( + f"TP{row['tp']} {row['candidate']:18s} " + f"grid={row['grid_fit_mape_pct']:.2f}% " + f"heldout={row['heldout_context_mape_pct']:.2f}% " + f"max={row['heldout_context_max_abs_error_pct']:.2f}% " + f"monotone={row['monotone_and_nonnegative']}" + ) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-attn-structured-v0/analyze_trace_verdict.py b/runs/frontier-attn-structured-v0/analyze_trace_verdict.py new file mode 100644 index 0000000..d8def59 --- /dev/null +++ b/runs/frontier-attn-structured-v0/analyze_trace_verdict.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +"""Trial-aware verdict for the seven structured-attention trace replays.""" + +from __future__ import annotations + +import csv +import json +import math +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] +S3_REAL = REPO / "runs/frontier-s3-real-v0" +V5 = REPO / "runs/frontier-prefill-kvgrowth-fix-v0" + +CELLS = { + "tp1_rho0p00125": { + "real": "frontier-tp1-real-r0p00125-t*", + "old": V5 / "sim-replay-tp1/v5/tp1_rho0p00125", + }, + "tp1_rho0p0025": { + "real": "frontier-tp1-real-r0p0025-t*", + "old": V5 / "sim-replay-tp1/v5/tp1_rho0p0025", + }, + "tp2_rho0p0025": { + "real": "frontier-s3-real-full-r0p0025-tp2-t*", + "old": V5 / "sim-replay/tp2_rho0p0025", + }, + "tp2_rho0p005": { + "real": "frontier-s3-real-full-r0p005-tp2-t*", + "old": V5 / "sim-replay/tp2_rho0p005", + }, + "tp4_rho0p0025": { + "real": "frontier-s3-real-full-r0p0025-tp4-t*", + "old": V5 / "sim-replay/tp4_rho0p0025", + }, + "tp4_rho0p005": { + "real": "frontier-s3-real-full-r0p005-tp4-t*", + "old": V5 / "sim-replay/tp4_rho0p005", + }, + "tp4_rho0p01": { + "real": "frontier-s3-real-full-r0p01-tp4-t*", + "old": V5 / "sim-replay/tp4_rho0p01", + }, +} +METRICS = { + "ttft": ("ttft_ms", "ttft"), + "tpot": ("tpot_ms", "tpot"), + "e2e": ("e2e_ms", "request_e2e_time"), +} +QUANTILES = {"mean": None, "p50": 0.5, "p90": 0.9, "p99": 0.99} + + +def percentile(values: list[float], quantile: float) -> float: + ordered = sorted(values) + position = (len(ordered) - 1) * quantile + lower, upper = math.floor(position), math.ceil(position) + if lower == upper: + return ordered[lower] + return ( + ordered[lower] * (upper - position) + + ordered[upper] * (position - lower) + ) + + +def summarize(values: list[float]) -> dict[str, float]: + return { + name: ( + sum(values) / len(values) + if quantile is None + else percentile(values, quantile) + ) + for name, quantile in QUANTILES.items() + } + + +def load_real_trials(pattern: str) -> list[list[dict[str, Any]]]: + trials = [] + for run_root in sorted((S3_REAL / "fleet-artifacts").glob(pattern)): + results = list( + run_root.glob( + "artifacts/outputs/full-real/*/*/trial-*/results/result.json" + ) + ) + if len(results) != 1: + raise ValueError(f"expected one result in {run_root}, got {results}") + trials.append(json.loads(results[0].read_text())["requests"]) + if len(trials) != 2: + raise ValueError(f"expected two real trials for {pattern}, got {len(trials)}") + return trials + + +def load_sim(root: Path) -> list[dict[str, str]]: + matches = list((root / "metrics").rglob("request_metrics.csv")) + if len(matches) != 1: + raise ValueError(f"expected one request_metrics.csv below {root}: {matches}") + rows = list(csv.DictReader(matches[0].open())) + rows.sort(key=lambda row: int(float(row["Request Id"]))) + return rows + + +def distribution_bias( + real_rows: list[dict[str, Any]], + sim_rows: list[dict[str, str]], +) -> dict[str, dict[str, float]]: + output: dict[str, dict[str, float]] = {} + for metric, (real_key, sim_key) in METRICS.items(): + pairs = [ + (float(real[real_key]), float(sim[sim_key])) + for real, sim in zip(real_rows, sim_rows) + if real.get("success") + ] + real_summary = summarize([pair[0] for pair in pairs]) + sim_summary = summarize([pair[1] for pair in pairs]) + output[metric] = { + name: (sim_summary[name] - real_summary[name]) / real_summary[name] + for name in QUANTILES + } + return output + + +def paired_relative_error( + real_rows: list[dict[str, Any]], + sim_rows: list[dict[str, str]], +) -> dict[str, dict[str, float]]: + output: dict[str, dict[str, float]] = {} + for metric, (real_key, sim_key) in METRICS.items(): + errors = [ + (float(sim[sim_key]) - float(real[real_key])) / float(real[real_key]) + for real, sim in zip(real_rows, sim_rows) + if real.get("success") and float(real[real_key]) != 0 + ] + output[metric] = summarize(errors) + return output + + +def aggregate_trial_bias( + trial_biases: list[dict[str, dict[str, float]]], +) -> dict[str, dict[str, dict[str, float]]]: + return { + metric: { + quantile: { + "mean": sum(values) / len(values), + "min": min(values), + "max": max(values), + } + for quantile in QUANTILES + for values in [ + [trial[metric][quantile] for trial in trial_biases] + ] + } + for metric in METRICS + } + + +def legacy_pooled_bias( + real_trials: list[list[dict[str, Any]]], + sim_rows: list[dict[str, str]], +) -> dict[str, dict[str, float]]: + output: dict[str, dict[str, float]] = {} + for metric, (real_key, sim_key) in METRICS.items(): + real_values = [ + float(row[real_key]) + for trial in real_trials + for row in trial[: len(sim_rows)] + if row.get("success") + ] + sim_values = [float(row[sim_key]) for row in sim_rows] + real_summary = summarize(real_values) + sim_summary = summarize(sim_values) + output[metric] = { + name: (sim_summary[name] - real_summary[name]) / real_summary[name] + for name in QUANTILES + } + return output + + +def waiting_p99(sim_rows: list[dict[str, str]]) -> float: + return percentile( + [float(row["request_waiting_time_total"]) for row in sim_rows], 0.99 + ) + + +def main() -> None: + results: dict[str, Any] = {} + flat_rows: list[dict[str, Any]] = [] + for label, paths in CELLS.items(): + real_trials = load_real_trials(paths["real"]) + old_sim = load_sim(paths["old"]) + new_sim = load_sim(ROOT / "replay" / label) + old_trial_bias = [ + distribution_bias(trial, old_sim) for trial in real_trials + ] + new_trial_bias = [ + distribution_bias(trial, new_sim) for trial in real_trials + ] + old_legacy = legacy_pooled_bias(real_trials, old_sim) + new_legacy = legacy_pooled_bias(real_trials, new_sim) + wait_p99 = waiting_p99(new_sim) + results[label] = { + "old": { + "trialwise_distribution_bias": old_trial_bias, + "trialwise_distribution_bias_summary": aggregate_trial_bias( + old_trial_bias + ), + "legacy_pooled_distribution_bias": old_legacy, + }, + "new": { + "trialwise_distribution_bias": new_trial_bias, + "trialwise_distribution_bias_summary": aggregate_trial_bias( + new_trial_bias + ), + "paired_relative_error": [ + paired_relative_error(trial, new_sim) + for trial in real_trials + ], + "legacy_pooled_distribution_bias": new_legacy, + "waiting_p99_ms": wait_p99, + "validity": ( + "PASS_SUBCRITICAL" + if wait_p99 < 1000 + else "GATE_FAIL_DIAGNOSTIC" + ), + }, + } + for metric in METRICS: + for quantile in QUANTILES: + flat_rows.append( + { + "cell": label, + "metric": metric, + "quantile": quantile, + "old_bias": old_legacy[metric][quantile], + "new_bias": new_legacy[metric][quantile], + "abs_bias_delta_pp": 100 + * ( + abs(new_legacy[metric][quantile]) + - abs(old_legacy[metric][quantile]) + ), + "validity": results[label]["new"]["validity"], + } + ) + + tp1_checks = [] + for cell in ("tp1_rho0p00125", "tp1_rho0p0025"): + for quantile in ("mean", "p99"): + old = results[cell]["old"]["legacy_pooled_distribution_bias"]["ttft"][ + quantile + ] + new = results[cell]["new"]["legacy_pooled_distribution_bias"]["ttft"][ + quantile + ] + tp1_checks.append(abs(old) - abs(new) >= 0.05) + + regressions = [ + row + for row in flat_rows + if row["cell"].startswith(("tp2", "tp4")) + and row["metric"] in ("ttft", "e2e") + and row["abs_bias_delta_pp"] > 5 + ] + checks = { + "tp1_ttft_mean_p99_improve_ge_5pp": all(tp1_checks), + "tp2_tp4_ttft_e2e_no_abs_regression_gt_5pp": not regressions, + "regressions": regressions, + } + checks["trace_gate"] = ( + checks["tp1_ttft_mean_p99_improve_ge_5pp"] + and checks["tp2_tp4_ttft_e2e_no_abs_regression_gt_5pp"] + ) + + payload = { + "schema": "frontier-attn-structured-trial-aware-verdict-v1", + "metric_note": ( + "Primary values are per-real-trial distribution biases with request " + "alignment by index. legacy_pooled reproduces the old milestone " + "quantile convention only for direct comparison." + ), + "cells": results, + "checks": checks, + } + output = ROOT / "results" + output.mkdir(parents=True, exist_ok=True) + (output / "trace-verdict.json").write_text(json.dumps(payload, indent=2)) + with (output / "trace-verdict.csv").open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(flat_rows[0])) + writer.writeheader() + writer.writerows(flat_rows) + + print(json.dumps(checks, indent=2)) + for label, result in results.items(): + old = result["old"]["legacy_pooled_distribution_bias"]["ttft"] + new = result["new"]["legacy_pooled_distribution_bias"]["ttft"] + print( + f"{label}: TTFT mean {old['mean']:+.1%}->{new['mean']:+.1%}, " + f"p99 {old['p99']:+.1%}->{new['p99']:+.1%}, " + f"waiting_p99={result['new']['waiting_p99_ms']:.0f}ms " + f"{result['new']['validity']}" + ) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-attn-structured-v0/experiment-card.md b/runs/frontier-attn-structured-v0/experiment-card.md new file mode 100644 index 0000000..cb4f6a7 --- /dev/null +++ b/runs/frontier-attn-structured-v0/experiment-card.md @@ -0,0 +1,82 @@ +# 实验 EXP-ATTN-STRUCTURED：结构化 predictor 能否关闭大 KV 端的 RF 欠拟合 + +> **状态：** 已完成（profile gate PASS；global merge gate FAIL） +> +> Parent campaign：[`../frontier-simulator-gap-campaign-v0/README.md`](../frontier-simulator-gap-campaign-v0/README.md) + +## Claim 与决策 + +- **Parent claim：** profile-v5 已补齐 chunked-prefill KV-context 测量，但当前 RF 仍在 TP1/2/4 的新网格上产生约 12%--14% self-fit MAPE，并在 TP1 真实 trace 中留下 −13% 到 −22% TTFT 偏差。 +- **目的：** 检查该 residual 是否来自可工程修复的 predictor representation，而不是 profile 数据或 serving path。 +- **Competing hypotheses：** + - H1：standard prefill 模型错误混入 pure multi-request rows，且 RF 对连续 attention scaling 作阶梯平滑；使用单请求数据和结构化 `base(q)+KV×(a+bq)` 可关闭残余。 + - H2：残余主要来自未建模的 serving-path 组件；替换 predictor 不会改善 7-cell trace fidelity。 +- **事前预测：** + - H1：held-out context MAPE ≤5%，TP1 TTFT mean/p99 绝对偏差至少改善 5 pp。 + - H2：profile gate 失败，或 profile gate 通过但 trace TTFT 几乎不动。 +- **判定规则：** + - profile gate：TP1/2/4 held-out context MAPE 均 ≤5%；q/KV 单调且预测非负。 + - trace gate：两个 TP1 cell 的 TTFT mean/p99 |bias| 各改善 ≥5 pp；TP2/TP4 任一 TTFT/E2E quantile 不恶化 >5 pp。 + - profile gate 失败即停止；trace gate 失败则回退 patch，不进入 EXP-2。 + +## Setup + +- **自变量：** + - A：现有 RF，standard prefill 全部非 true-mixed rows。 + - B：现有 RF，但仅 `batch_size=1`。 + - C：仅 `batch_size=1` 的 structured predictor： + - `base(q)`：KV=0 profile 的单调分段线性插值； + - growth：非负 least-squares `KV×(a+bq)`。 +- **控制变量：** attention/linear/MoE/collective profile、trace、prefix cache、scheduler、graph mode、KV blocks、MNS、全部 argv。 +- **System context：** Qwen3-30B-A3B BF16；H20；Frontier `deadc4a3`；TP1/2/4；MNS16；chunk 8192；prefix caching。 +- **Workload 或 trace：** 现有 7-cell 60-min production chat trace matrix；real 侧每 cell 两个 trial。 +- **Baselines：** `docs/assets/frontier-fidelity/full-matrix.csv` 的 sim-v5。 +- **Metrics：** + - profile：grid fit MAPE、leave-one-context MAPE/max error、q/KV monotonicity； + - trace：request-ID paired bias；每个 real trial 单独计算后报告 mean 与 trial interval； + - queue validity：waiting p99，TP1 超过 1 s 的 cell 标为 diagnostic。 + +## 预期产物与 review + +- **预期数据：** `results/predictor-ablation.{json,csv}`、`replay//`、`results/paired-verdict.json`。 +- **Figure prototype：** `figure-prototype.png`；左图为 q8k 随 KV 增长的 actual/RF/structured，右图为 7-cell TTFT bias 的事前期望。 +- **人工 review：** 已按 campaign 顺序批准执行。 +- **Review 意见：** 只改 standard single-request predictor；不得改 mixed predictor 或任何 profile row。 + +## 复现信息 + +- **Code：** Frontier base `deadc4a321f0baaa534c6ebd17f974123733cdc2`；实验 patch 将保存为 `frontier-structured-attn.patch` 并记录 SHA256。 +- **Environment：** 本地 CPU replay；Python dependency roots 复用 `runs/frontier-collective-joint-v0/counterfactual/joint-r2/manifest.json`。 +- **产物路径：** 本目录。 +- **已知 deviation：** milestone 文档将 7-cell 口径称为“逐 request paired”，但旧脚本实际 pool 两个 real trial 后比较 quantile；本实验会修正分析口径，不改旧结果文件。 + +## 预分析事实 + +- 现有训练代码使用 `["kv_cache_size", "prefill_chunk_size_squared"]` 与 RF grid search。 +- runtime cache 注释明确 standard model 是 per-request；多请求 prefill 在模型存在时走 `attn_prefill_mixed`。 +- profile-v5 的 standard 训练集每 TP 有 29 行，其中单请求 23 行；有 4 组相同 `(KV,q²)` feature 对应多个 pure-batch 标签。 +- 初步 structured candidate 的 leave-one-context MAPE：TP1 0.84%、TP2 1.61%、TP4 3.01%；max error 分别 2.04%、3.47%、5.49%。这些是实现前的临时计算，须由版本化脚本复现后才进入结果。 + +## 结果 + +- **观察事实：** + - structured held-out-context MAPE 为 TP1/2/4=`0.84%/1.60%/3.01%`； + 当前 RF 为 `44.40%/44.20%/43.54%`。单调/非负 gate 通过。 + - TP1 两点 TTFT mean bias `−13.5/−17.7% → −6.3/−9.5%`，p99 + `−16.8/−22.1% → −8.6/−14.4%`。 + - TP2 两点 TTFT mean bias `−11.2/−14.1% → −4.5/−7.1%`，p99 + `−17.3/−19.4% → −7.7/−9.0%`。 + - TP4 三点 TTFT mean bias `+2.5/+2.9/−0.1% → +7.8/+8.4/+6.0%`； + 三点均使绝对误差恶化 `5.3--5.8 pp`，触发预设回归 gate。 + - validity 重新审计：TP1 两点 waiting p99=`1.34/1.89 s`；TP2 + ρ=.005=`1.17 s`，均标为 `GATE_FAIL_DIAGNOSTIC`。其余四点通过。 +- **异常：** TP4 ρ=.005 的 TPOT p99 从 `+31.2%` 变为 `+36.6%`， + 表明该 tail 对 prefill/mixed-decode 相位敏感，不是本 patch 能关闭的稳定 + decode predictor 偏差。 +- **含义：** H1 的 representation 机制得到支持，但“全局替换 RF 可直接提升 + 7-cell fidelity”被反驳。TP4 原先接近零的 mean TTFT 含有 predictor + 欠拟合与其它正向 residual 的误差抵消；单独修正 attention 会揭开后者。 +- **Claim update：** structured predictor 是明确的工程候选，但必须与 TP4 + residual 联合收敛后才可 merge；当前 patch 只保留为 ablation。 +- **下一步：** EXP-2 先重算 structured 分支的 TP2 chunk-level residual； + 仅 residual ≥10% 才运行 GPU serving-path 三臂 profile。 diff --git a/runs/frontier-attn-structured-v0/figure-prototype.png b/runs/frontier-attn-structured-v0/figure-prototype.png new file mode 100644 index 0000000..149b544 Binary files /dev/null and b/runs/frontier-attn-structured-v0/figure-prototype.png differ diff --git a/runs/frontier-attn-structured-v0/frontier-reference.json b/runs/frontier-attn-structured-v0/frontier-reference.json new file mode 100644 index 0000000..bf87302 --- /dev/null +++ b/runs/frontier-attn-structured-v0/frontier-reference.json @@ -0,0 +1,525 @@ +{ + "cc_cache": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/cc-cache", + "cells": { + "tp1_mns16": { + "argv": [ + "/usr/bin/python3", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/run_frontier_with_curves.py", + "--simulation_mode", + "online", + "--sys_arch", + "co-location", + "--cc_backend_config_type", + "vidur", + "--cluster_config_num_replicas", + "1", + "--cluster_scheduler_config_type", + "sticky_round_robin", + "--replica_config_model_name", + "qwen3-a3b-30b-moe", + "--replica_config_device", + "h20", + "--replica_config_network_device", + "h20_dgx", + "--replica_config_attn_tensor_parallel_size", + "1", + "--replica_config_attn_data_parallel_size", + "1", + "--replica_config_moe_tensor_parallel_size", + "1", + "--replica_config_moe_expert_parallel_size", + "1", + "--replica_config_num_pipeline_stages", + "1", + "--replica_scheduler_config_type", + "vllm_v1", + "--decode_cuda_graph_mode", + "piecewise", + "--vllm_v1_scheduler_config_batch_size_cap", + "16", + "--vllm_v1_scheduler_config_max_tokens_in_batch", + "8192", + "--vllm_v1_scheduler_config_long_prefill_token_threshold", + "0", + "--vllm_v1_scheduler_config_block_size", + "16", + "--vllm_v1_scheduler_config_num_blocks_mode", + "explicit", + "--vllm_v1_scheduler_config_gpu_memory_utilization", + "0.92", + "--vllm_v1_scheduler_config_non_kv_cache_overhead_bytes", + "0", + "--request_generator_config_type", + "trace_replay", + "--trace_request_generator_config_trace_file", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/inputs/tp1-frontier.csv", + "--trace_request_generator_config_max_tokens", + "40960", + "--metrics_config_output_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/sim/tp1_mns16/metrics", + "--metrics_config_run_id", + "joint_tp1_mns16", + "--metrics_config_write_metrics", + "--metrics_config_store_request_metrics", + "--metrics_config_store_batch_metrics", + "--metrics_config_store_token_completion_metrics", + "--metrics_config_store_utilization_metrics", + "--no-metrics_config_store_plots", + "--no-metrics_config_enable_chrome_trace", + "--no-metrics_config_write_json_trace", + "--metrics_config_store_frontier_stage_batch_ledger", + "--no-random_forrest_execution_time_predictor_config_enable_dummy_mode", + "--random_forrest_execution_time_predictor_config_linear_op_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/linear_op.csv", + "--random_forrest_execution_time_predictor_config_atten_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/attention.csv", + "--random_forrest_execution_time_predictor_config_moe_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/moe.csv", + "--random_forrest_execution_time_predictor_config_linear_op_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/linear_op.csv", + "--random_forrest_execution_time_predictor_config_atten_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/attention.csv", + "--random_forrest_execution_time_predictor_config_moe_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/moe.csv", + "--random_forrest_execution_time_predictor_config_prediction_max_prefill_chunk_size", + "8192", + "--random_forrest_execution_time_predictor_config_prediction_max_batch_size", + "32", + "--random_forrest_execution_time_predictor_config_prediction_max_tokens_per_request", + "40960", + "--random_forrest_execution_time_predictor_config_no_cache", + "--random_forrest_execution_time_predictor_config_skip_cpu_overhead_modeling", + "--vllm_v1_scheduler_config_num_blocks", + "20128", + "--vllm_v1_scheduler_config_enable_chunked_prefill", + "--random_forrest_execution_time_predictor_config_num_training_job_threads", + "4", + "--cudagraph_capture_sizes", + "1", + "2", + "4", + "8", + "16", + "24", + "32", + "--vidur_cc_backend_config_all_reduce_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/measured-allreduce.csv", + "--vidur_cc_backend_config_cache_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/cc-cache", + "--vidur_cc_backend_config_k_fold_cv_splits", + "6", + "--vidur_cc_backend_config_num_training_job_threads", + "1", + "--metrics_config_cache_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/model-cache" + ], + "log": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/logs/tp1_mns16.log", + "source_command": "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-lo-fixed-pd-cells/sim/fixed-pd/runs/tp1_mns16/tp1/command.json", + "source_command_sha256": "a9815797b1601bf6f6cdf0269e84acb376a84945609e338868dc8347aab650e6", + "usage": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/usage/tp1_mns16.json" + }, + "tp2_mns16": { + "argv": [ + "/usr/bin/python3", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/run_frontier_with_curves.py", + "--simulation_mode", + "online", + "--sys_arch", + "co-location", + "--cc_backend_config_type", + "vidur", + "--cluster_config_num_replicas", + "1", + "--cluster_scheduler_config_type", + "sticky_round_robin", + "--replica_config_model_name", + "qwen3-a3b-30b-moe", + "--replica_config_device", + "h20", + "--replica_config_network_device", + "h20_dgx", + "--replica_config_attn_tensor_parallel_size", + "2", + "--replica_config_attn_data_parallel_size", + "1", + "--replica_config_moe_tensor_parallel_size", + "2", + "--replica_config_moe_expert_parallel_size", + "1", + "--replica_config_num_pipeline_stages", + "1", + "--replica_scheduler_config_type", + "vllm_v1", + "--decode_cuda_graph_mode", + "piecewise", + "--vllm_v1_scheduler_config_batch_size_cap", + "16", + "--vllm_v1_scheduler_config_max_tokens_in_batch", + "8192", + "--vllm_v1_scheduler_config_long_prefill_token_threshold", + "0", + "--vllm_v1_scheduler_config_block_size", + "16", + "--vllm_v1_scheduler_config_num_blocks_mode", + "explicit", + "--vllm_v1_scheduler_config_gpu_memory_utilization", + "0.92", + "--vllm_v1_scheduler_config_non_kv_cache_overhead_bytes", + "0", + "--request_generator_config_type", + "trace_replay", + "--trace_request_generator_config_trace_file", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/inputs/tp2-frontier.csv", + "--trace_request_generator_config_max_tokens", + "40960", + "--metrics_config_output_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/sim/tp2_mns16/metrics", + "--metrics_config_run_id", + "joint_tp2_mns16", + "--metrics_config_write_metrics", + "--metrics_config_store_request_metrics", + "--metrics_config_store_batch_metrics", + "--metrics_config_store_token_completion_metrics", + "--metrics_config_store_utilization_metrics", + "--no-metrics_config_store_plots", + "--no-metrics_config_enable_chrome_trace", + "--no-metrics_config_write_json_trace", + "--metrics_config_store_frontier_stage_batch_ledger", + "--no-random_forrest_execution_time_predictor_config_enable_dummy_mode", + "--random_forrest_execution_time_predictor_config_linear_op_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/linear_op.csv", + "--random_forrest_execution_time_predictor_config_atten_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/attention.csv", + "--random_forrest_execution_time_predictor_config_moe_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/moe.csv", + "--random_forrest_execution_time_predictor_config_linear_op_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/linear_op.csv", + "--random_forrest_execution_time_predictor_config_atten_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/attention.csv", + "--random_forrest_execution_time_predictor_config_moe_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/moe.csv", + "--random_forrest_execution_time_predictor_config_prediction_max_prefill_chunk_size", + "8192", + "--random_forrest_execution_time_predictor_config_prediction_max_batch_size", + "32", + "--random_forrest_execution_time_predictor_config_prediction_max_tokens_per_request", + "40960", + "--random_forrest_execution_time_predictor_config_no_cache", + "--random_forrest_execution_time_predictor_config_skip_cpu_overhead_modeling", + "--vllm_v1_scheduler_config_num_blocks", + "76620", + "--vllm_v1_scheduler_config_enable_chunked_prefill", + "--random_forrest_execution_time_predictor_config_num_training_job_threads", + "4", + "--cudagraph_capture_sizes", + "1", + "2", + "4", + "8", + "16", + "24", + "32", + "--vidur_cc_backend_config_all_reduce_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/measured-allreduce.csv", + "--vidur_cc_backend_config_cache_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/cc-cache", + "--vidur_cc_backend_config_k_fold_cv_splits", + "6", + "--vidur_cc_backend_config_num_training_job_threads", + "1", + "--metrics_config_cache_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/model-cache" + ], + "log": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/logs/tp2_mns16.log", + "source_command": "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-lo-fixed-pd-cells/sim/fixed-pd/runs/tp2_mns16/tp2/command.json", + "source_command_sha256": "61788a8810be301c9dbc006624aa19b6a932bc44d341b836861087833cffc3df", + "usage": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/usage/tp2_mns16.json" + }, + "tp4_mns16": { + "argv": [ + "/usr/bin/python3", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/run_frontier_with_curves.py", + "--simulation_mode", + "online", + "--sys_arch", + "co-location", + "--cc_backend_config_type", + "vidur", + "--cluster_config_num_replicas", + "1", + "--cluster_scheduler_config_type", + "sticky_round_robin", + "--replica_config_model_name", + "qwen3-a3b-30b-moe", + "--replica_config_device", + "h20", + "--replica_config_network_device", + "h20_dgx", + "--replica_config_attn_tensor_parallel_size", + "4", + "--replica_config_attn_data_parallel_size", + "1", + "--replica_config_moe_tensor_parallel_size", + "4", + "--replica_config_moe_expert_parallel_size", + "1", + "--replica_config_num_pipeline_stages", + "1", + "--replica_scheduler_config_type", + "vllm_v1", + "--decode_cuda_graph_mode", + "piecewise", + "--vllm_v1_scheduler_config_batch_size_cap", + "16", + "--vllm_v1_scheduler_config_max_tokens_in_batch", + "8192", + "--vllm_v1_scheduler_config_long_prefill_token_threshold", + "0", + "--vllm_v1_scheduler_config_block_size", + "16", + "--vllm_v1_scheduler_config_num_blocks_mode", + "explicit", + "--vllm_v1_scheduler_config_gpu_memory_utilization", + "0.92", + "--vllm_v1_scheduler_config_non_kv_cache_overhead_bytes", + "0", + "--request_generator_config_type", + "trace_replay", + "--trace_request_generator_config_trace_file", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/inputs/tp4-frontier.csv", + "--trace_request_generator_config_max_tokens", + "40960", + "--metrics_config_output_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/sim/tp4_mns16/metrics", + "--metrics_config_run_id", + "joint_tp4_mns16", + "--metrics_config_write_metrics", + "--metrics_config_store_request_metrics", + "--metrics_config_store_batch_metrics", + "--metrics_config_store_token_completion_metrics", + "--metrics_config_store_utilization_metrics", + "--no-metrics_config_store_plots", + "--no-metrics_config_enable_chrome_trace", + "--no-metrics_config_write_json_trace", + "--metrics_config_store_frontier_stage_batch_ledger", + "--no-random_forrest_execution_time_predictor_config_enable_dummy_mode", + "--random_forrest_execution_time_predictor_config_linear_op_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/linear_op.csv", + "--random_forrest_execution_time_predictor_config_atten_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/attention.csv", + "--random_forrest_execution_time_predictor_config_moe_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/moe.csv", + "--random_forrest_execution_time_predictor_config_linear_op_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/linear_op.csv", + "--random_forrest_execution_time_predictor_config_atten_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/attention.csv", + "--random_forrest_execution_time_predictor_config_moe_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/moe.csv", + "--random_forrest_execution_time_predictor_config_prediction_max_prefill_chunk_size", + "8192", + "--random_forrest_execution_time_predictor_config_prediction_max_batch_size", + "32", + "--random_forrest_execution_time_predictor_config_prediction_max_tokens_per_request", + "40960", + "--random_forrest_execution_time_predictor_config_no_cache", + "--random_forrest_execution_time_predictor_config_skip_cpu_overhead_modeling", + "--vllm_v1_scheduler_config_num_blocks", + "191882", + "--vllm_v1_scheduler_config_enable_chunked_prefill", + "--random_forrest_execution_time_predictor_config_num_training_job_threads", + "4", + "--cudagraph_capture_sizes", + "1", + "2", + "4", + "8", + "16", + "24", + "32", + "--vidur_cc_backend_config_all_reduce_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/measured-allreduce.csv", + "--vidur_cc_backend_config_cache_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/cc-cache", + "--vidur_cc_backend_config_k_fold_cv_splits", + "6", + "--vidur_cc_backend_config_num_training_job_threads", + "1", + "--metrics_config_cache_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/model-cache" + ], + "log": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/logs/tp4_mns16.log", + "source_command": "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-lo-fixed-pd-cells/sim/fixed-pd/runs/tp4_mns16/tp4/command.json", + "source_command_sha256": "9bbcf10446336ba5885193f391dd628cd18ff64d91a51ebcd463ffd24be95532", + "usage": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/usage/tp4_mns16.json" + }, + "tp4_mns32": { + "argv": [ + "/usr/bin/python3", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/run_frontier_with_curves.py", + "--simulation_mode", + "online", + "--sys_arch", + "co-location", + "--cc_backend_config_type", + "vidur", + "--cluster_config_num_replicas", + "1", + "--cluster_scheduler_config_type", + "sticky_round_robin", + "--replica_config_model_name", + "qwen3-a3b-30b-moe", + "--replica_config_device", + "h20", + "--replica_config_network_device", + "h20_dgx", + "--replica_config_attn_tensor_parallel_size", + "4", + "--replica_config_attn_data_parallel_size", + "1", + "--replica_config_moe_tensor_parallel_size", + "4", + "--replica_config_moe_expert_parallel_size", + "1", + "--replica_config_num_pipeline_stages", + "1", + "--replica_scheduler_config_type", + "vllm_v1", + "--decode_cuda_graph_mode", + "piecewise", + "--vllm_v1_scheduler_config_batch_size_cap", + "32", + "--vllm_v1_scheduler_config_max_tokens_in_batch", + "8192", + "--vllm_v1_scheduler_config_long_prefill_token_threshold", + "0", + "--vllm_v1_scheduler_config_block_size", + "16", + "--vllm_v1_scheduler_config_num_blocks_mode", + "explicit", + "--vllm_v1_scheduler_config_gpu_memory_utilization", + "0.92", + "--vllm_v1_scheduler_config_non_kv_cache_overhead_bytes", + "0", + "--request_generator_config_type", + "trace_replay", + "--trace_request_generator_config_trace_file", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/inputs/tp4-frontier.csv", + "--trace_request_generator_config_max_tokens", + "40960", + "--metrics_config_output_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/sim/tp4_mns32/metrics", + "--metrics_config_run_id", + "joint_tp4_mns32", + "--metrics_config_write_metrics", + "--metrics_config_store_request_metrics", + "--metrics_config_store_batch_metrics", + "--metrics_config_store_token_completion_metrics", + "--metrics_config_store_utilization_metrics", + "--no-metrics_config_store_plots", + "--no-metrics_config_enable_chrome_trace", + "--no-metrics_config_write_json_trace", + "--metrics_config_store_frontier_stage_batch_ledger", + "--no-random_forrest_execution_time_predictor_config_enable_dummy_mode", + "--random_forrest_execution_time_predictor_config_linear_op_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/linear_op.csv", + "--random_forrest_execution_time_predictor_config_atten_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/attention.csv", + "--random_forrest_execution_time_predictor_config_moe_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/profile-v4-trace-final/moe.csv", + "--random_forrest_execution_time_predictor_config_linear_op_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/linear_op.csv", + "--random_forrest_execution_time_predictor_config_atten_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/attention.csv", + "--random_forrest_execution_time_predictor_config_moe_kernel_only_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/frozen-kernel-only/moe.csv", + "--random_forrest_execution_time_predictor_config_prediction_max_prefill_chunk_size", + "8192", + "--random_forrest_execution_time_predictor_config_prediction_max_batch_size", + "64", + "--random_forrest_execution_time_predictor_config_prediction_max_tokens_per_request", + "40960", + "--random_forrest_execution_time_predictor_config_no_cache", + "--random_forrest_execution_time_predictor_config_skip_cpu_overhead_modeling", + "--vllm_v1_scheduler_config_num_blocks", + "191786", + "--vllm_v1_scheduler_config_enable_chunked_prefill", + "--random_forrest_execution_time_predictor_config_num_training_job_threads", + "4", + "--cudagraph_capture_sizes", + "1", + "2", + "4", + "8", + "16", + "24", + "32", + "40", + "48", + "56", + "64", + "--vidur_cc_backend_config_all_reduce_input_file", + "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-profiles/measured-allreduce.csv", + "--vidur_cc_backend_config_cache_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/cc-cache", + "--vidur_cc_backend_config_k_fold_cv_splits", + "6", + "--vidur_cc_backend_config_num_training_job_threads", + "1", + "--metrics_config_cache_dir", + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/model-cache" + ], + "log": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/logs/tp4_mns32.log", + "source_command": "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-lo-fixed-pd-cells/sim/fixed-pd/runs/tp4_mns32/tp4/command.json", + "source_command_sha256": "fbc7dee55590b415ed1cde8072de835ed155a0c20ba0eb305c3cb22aa8065a51", + "usage": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/usage/tp4_mns32.json" + } + }, + "collective_curve": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/results/collective-curve.json", + "collective_curve_sha256": "f9543649d4ea78f08240bf1284ab74083aa5cf5671ed47e386047f1453300b36", + "collective_curve_variant": "drop_mean", + "frontier_checkout": "/tmp/frontier-attn-structured-v0", + "frontier_commit": "1f8900a4ac64e45754b03d0aa7c1dddab65785cf", + "mode": "joint", + "model_cache": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/model-cache", + "moe_curve": "/home/gahow/phd/aituner/runs/frontier-fused-moe-profile-v0/results/fused-moe-curve.json", + "moe_curve_sha256": "b94d65d9d581adefcc6c14ed4920cce6a1136f74f1014737dc3e4249bc8250d2", + "python": "/usr/bin/python3", + "python_dependency_roots": [ + "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/python-deps", + "/home/gahow/.cache/uv/archive-v0/-_kzErLcPO5nASZFX8b9k", + "/home/gahow/.cache/uv/archive-v0/FbaBs_QJ9QKEbQ9V_4aIR", + "/home/gahow/.cache/uv/archive-v0/fuHsGXD0Lv_UjFC8yI4-7", + "/home/gahow/.cache/uv/archive-v0/jFGdqQLpB1eopfm9VxT3j", + "/home/gahow/.cache/uv/archive-v0/YWW6ExSJuPVvv4-qYQTin", + "/home/gahow/.cache/uv/archive-v0/3_qxZ5Ll-EpVAGZfbksfe" + ], + "traces": { + "1": { + "first_arrival_s": 0.0, + "last_arrival_s": 595.348837209302, + "requests": 129, + "source_request_metrics": "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-lo-fixed-pd-cells/sim/fixed-pd/runs/tp1_mns16/tp1/metrics/qwen3_a3b_30b_moe/online_serving/qwen30_trace_tp1_mns16_tp1/request_metrics.csv", + "source_sha256": "0b82e09644a5884fcd10d894b68495daefdabb32b770146c2f9ece37b8469f4f", + "trace": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/inputs/tp1-frontier.csv", + "trace_sha256": "59dd8996ff879ef94330004104dfdf515b791bce4036576eccc93290e9206dad" + }, + "2": { + "first_arrival_s": 0.0, + "last_arrival_s": 297.674418604651, + "requests": 129, + "source_request_metrics": "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-lo-fixed-pd-cells/sim/fixed-pd/runs/tp2_mns16/tp2/metrics/qwen3_a3b_30b_moe/online_serving/qwen30_trace_tp2_mns16_tp2/request_metrics.csv", + "source_sha256": "33983081bb20dd5e2053e9e3d13def8732e958150c9b47a8609ba345123f2316", + "trace": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/inputs/tp2-frontier.csv", + "trace_sha256": "64fc077b38274a76a8279884ac4115836cd1157c95119c64fabac50d81124f69" + }, + "4": { + "first_arrival_s": 0.0, + "last_arrival_s": 148.837209302326, + "requests": 129, + "source_request_metrics": "/home/gahow/phd/aituner/runs/frontier-split-rootcause-v0/frozen-inputs/q30-lo-fixed-pd-cells/sim/fixed-pd/runs/tp4_mns16/tp4/metrics/qwen3_a3b_30b_moe/online_serving/qwen30_trace_tp4_mns16_tp4/request_metrics.csv", + "source_sha256": "b36cd383c07b546d2c1f2fac754d5dbb92efd6880316b4233a7aef9fa1115a36", + "trace": "/home/gahow/phd/aituner/runs/frontier-collective-joint-v0/counterfactual/joint-r2/inputs/tp4-frontier.csv", + "trace_sha256": "adb3d6f3932a44c86c3d9e7cf1e57739594e8c48d19a26b5aa54b37dce0e0c19" + } + } +} \ No newline at end of file diff --git a/runs/frontier-attn-structured-v0/plot_figure_prototype.py b/runs/frontier-attn-structured-v0/plot_figure_prototype.py new file mode 100644 index 0000000..a3368f9 --- /dev/null +++ b/runs/frontier-attn-structured-v0/plot_figure_prototype.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Schematic figure frozen before EXP-ATTN-STRUCTURED execution.""" + +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +ROOT = Path(__file__).resolve().parent +SURFACE = "#fcfcfb" +INK = "#111111" +MUTED = "#77736c" +GRID = "#dedbd2" +RF = "#d95f02" +STRUCTURED = "#1b75bc" + +fig, axes = plt.subplots(1, 2, figsize=(10.8, 4.2), dpi=160) +fig.patch.set_facecolor(SURFACE) + +for ax in axes: + ax.set_facecolor(SURFACE) + ax.grid(axis="y", color=GRID, linewidth=0.8) + ax.set_axisbelow(True) + ax.spines[["top", "right"]].set_visible(False) + ax.tick_params(colors=MUTED, labelsize=8) + +kv = np.array([8, 16, 24], dtype=float) +actual = np.array([11.97, 19.92, 27.84]) +rf = np.array([9.46, 16.40, 24.52]) +structured_expected = np.array([12.0, 19.9, 27.9]) +axes[0].plot(kv, actual, "o-", color=INK, label="profile actual") +axes[0].plot(kv, rf, "s--", color=RF, label="current RF") +axes[0].plot( + kv, + structured_expected, + "^:", + color=STRUCTURED, + label="structured (expected)", +) +axes[0].set_xlabel("KV context (ktok)") +axes[0].set_ylabel("TP1 q8k attention time (ms)") +axes[0].set_title("(a) Continuous KV growth", loc="left", fontsize=10) +axes[0].legend(frameon=False, fontsize=8) + +labels = ["TP1\n.00125", "TP1\n.0025", "TP2\n.0025", "TP2\n.005", + "TP4\n.0025", "TP4\n.005", "TP4\n.01"] +x = np.arange(len(labels)) +v5_mean = np.array([-13.5, -17.7, -11.2, -14.1, 2.5, 2.9, -0.2]) +expected = np.array([-5, -8, -9, -11, 3, 3, 0]) +axes[1].axhspan(-15, 15, color=GRID, alpha=0.5) +axes[1].axhline(0, color=MUTED, linewidth=0.8) +axes[1].plot(x, v5_mean, "o-", color=RF, label="sim-v5 measured") +axes[1].plot(x, expected, "s--", color=STRUCTURED, label="H1 expected") +axes[1].set_xticks(x, labels) +axes[1].set_ylabel("TTFT mean bias (%)") +axes[1].set_title("(b) 7-cell trace gate", loc="left", fontsize=10) +axes[1].legend(frameon=False, fontsize=8) + +fig.suptitle( + "MOCK / schematic — EXP-ATTN-STRUCTURED (not measured results)", + x=0.01, + ha="left", + color=RF, + fontsize=9, +) +fig.tight_layout(rect=(0, 0, 1, 0.95)) +fig.savefig(ROOT / "figure-prototype.png", facecolor=SURFACE) +fig.savefig(ROOT / "figure-prototype.svg", facecolor=SURFACE) +print(ROOT / "figure-prototype.png") diff --git a/runs/frontier-attn-structured-v0/results/predictor-ablation.csv b/runs/frontier-attn-structured-v0/results/predictor-ablation.csv new file mode 100644 index 0000000..3f5e290 --- /dev/null +++ b/runs/frontier-attn-structured-v0/results/predictor-ablation.csv @@ -0,0 +1,10 @@ +candidate,training_rows,grid_fit_mape_pct,single_fit_mape_pct,heldout_context_mape_pct,heldout_context_max_abs_error_pct,prediction_min_ms,q_min_delta_ms,kv_min_delta_ms,monotone_and_nonnegative,tp +rf_all,29,15.074275515235467,19.842994757611535,44.40139318281943,82.78279487156401,0.06029164119272453,0.0,-4.2841601371801374e-05,False,1 +rf_single,23,11.834992526698676,22.893412972496023,34.45356428541224,62.49334437588834,0.059327708247725125,-0.00022153525203457564,-1.4336001873005433e-05,False,1 +structured_single,23,0.8432511364168856,2.227110646811972,0.841926169535806,2.0408978739639134,0.05679146709541477,0.0,0.0016745062683911627,True,1 +rf_all,29,13.812689689573157,17.879870012606048,44.20143320278334,82.0656368501208,0.06000113548192927,-0.004361070463210395,-0.009949388915300408,False,2 +rf_single,23,10.738962684891058,20.818820413545826,34.84360402100271,66.67253880294443,0.059971319361210015,-0.003811210796127021,-0.009949388915300408,False,2 +structured_single,23,1.518412258922364,6.226292654776418,1.6047958673086566,3.464671475193195,0.05767893331746252,0.0,0.0013929374121726124,True,2 +rf_all,29,14.117015331381916,15.49366794487052,43.53650868837558,81.33508178007524,0.05866772018640992,-0.0009967416035880083,-5.5955198407176e-05,False,4 +rf_single,23,12.07517666989496,17.672692652721008,34.14562332866605,62.31478818862995,0.058213693721655094,-0.0012244979345549661,-8.259841203689389e-05,False,4 +structured_single,23,3.088740104976349,5.969109476280061,3.0103379904473164,5.490598706238697,0.05747733327249683,0.0,0.0012048051417407057,True,4 diff --git a/runs/frontier-attn-structured-v0/results/predictor-ablation.json b/runs/frontier-attn-structured-v0/results/predictor-ablation.json new file mode 100644 index 0000000..ca894c5 --- /dev/null +++ b/runs/frontier-attn-structured-v0/results/predictor-ablation.json @@ -0,0 +1,149 @@ +{ + "schema": "frontier-attn-structured-ablation-v1", + "profile": "/home/gahow/phd/aituner/runs/frontier-prefill-kvgrowth-fix-v0/profiles/profile-v5-kvgrowth/attention.csv", + "frontier_checkout": "/tmp/frontier-attn-structured-v0", + "data_audit": { + "tp1": { + "standard_rows": 29, + "single_request_rows": 23, + "target_grid_rows": 10, + "duplicate_feature_groups": 4 + }, + "tp2": { + "standard_rows": 29, + "single_request_rows": 23, + "target_grid_rows": 10, + "duplicate_feature_groups": 4 + }, + "tp4": { + "standard_rows": 29, + "single_request_rows": 23, + "target_grid_rows": 10, + "duplicate_feature_groups": 4 + } + }, + "results": [ + { + "candidate": "rf_all", + "training_rows": 29, + "grid_fit_mape_pct": 15.074275515235467, + "single_fit_mape_pct": 19.842994757611535, + "heldout_context_mape_pct": 44.40139318281943, + "heldout_context_max_abs_error_pct": 82.78279487156401, + "prediction_min_ms": 0.06029164119272453, + "q_min_delta_ms": 0.0, + "kv_min_delta_ms": -4.2841601371801374e-05, + "monotone_and_nonnegative": false, + "tp": 1 + }, + { + "candidate": "rf_single", + "training_rows": 23, + "grid_fit_mape_pct": 11.834992526698676, + "single_fit_mape_pct": 22.893412972496023, + "heldout_context_mape_pct": 34.45356428541224, + "heldout_context_max_abs_error_pct": 62.49334437588834, + "prediction_min_ms": 0.059327708247725125, + "q_min_delta_ms": -0.00022153525203457564, + "kv_min_delta_ms": -1.4336001873005433e-05, + "monotone_and_nonnegative": false, + "tp": 1 + }, + { + "candidate": "structured_single", + "training_rows": 23, + "grid_fit_mape_pct": 0.8432511364168856, + "single_fit_mape_pct": 2.227110646811972, + "heldout_context_mape_pct": 0.841926169535806, + "heldout_context_max_abs_error_pct": 2.0408978739639134, + "prediction_min_ms": 0.05679146709541477, + "q_min_delta_ms": 0.0, + "kv_min_delta_ms": 0.0016745062683911627, + "monotone_and_nonnegative": true, + "tp": 1 + }, + { + "candidate": "rf_all", + "training_rows": 29, + "grid_fit_mape_pct": 13.812689689573157, + "single_fit_mape_pct": 17.879870012606048, + "heldout_context_mape_pct": 44.20143320278334, + "heldout_context_max_abs_error_pct": 82.0656368501208, + "prediction_min_ms": 0.06000113548192927, + "q_min_delta_ms": -0.004361070463210395, + "kv_min_delta_ms": -0.009949388915300408, + "monotone_and_nonnegative": false, + "tp": 2 + }, + { + "candidate": "rf_single", + "training_rows": 23, + "grid_fit_mape_pct": 10.738962684891058, + "single_fit_mape_pct": 20.818820413545826, + "heldout_context_mape_pct": 34.84360402100271, + "heldout_context_max_abs_error_pct": 66.67253880294443, + "prediction_min_ms": 0.059971319361210015, + "q_min_delta_ms": -0.003811210796127021, + "kv_min_delta_ms": -0.009949388915300408, + "monotone_and_nonnegative": false, + "tp": 2 + }, + { + "candidate": "structured_single", + "training_rows": 23, + "grid_fit_mape_pct": 1.518412258922364, + "single_fit_mape_pct": 6.226292654776418, + "heldout_context_mape_pct": 1.6047958673086566, + "heldout_context_max_abs_error_pct": 3.464671475193195, + "prediction_min_ms": 0.05767893331746252, + "q_min_delta_ms": 0.0, + "kv_min_delta_ms": 0.0013929374121726124, + "monotone_and_nonnegative": true, + "tp": 2 + }, + { + "candidate": "rf_all", + "training_rows": 29, + "grid_fit_mape_pct": 14.117015331381916, + "single_fit_mape_pct": 15.49366794487052, + "heldout_context_mape_pct": 43.53650868837558, + "heldout_context_max_abs_error_pct": 81.33508178007524, + "prediction_min_ms": 0.05866772018640992, + "q_min_delta_ms": -0.0009967416035880083, + "kv_min_delta_ms": -5.5955198407176e-05, + "monotone_and_nonnegative": false, + "tp": 4 + }, + { + "candidate": "rf_single", + "training_rows": 23, + "grid_fit_mape_pct": 12.07517666989496, + "single_fit_mape_pct": 17.672692652721008, + "heldout_context_mape_pct": 34.14562332866605, + "heldout_context_max_abs_error_pct": 62.31478818862995, + "prediction_min_ms": 0.058213693721655094, + "q_min_delta_ms": -0.0012244979345549661, + "kv_min_delta_ms": -8.259841203689389e-05, + "monotone_and_nonnegative": false, + "tp": 4 + }, + { + "candidate": "structured_single", + "training_rows": 23, + "grid_fit_mape_pct": 3.088740104976349, + "single_fit_mape_pct": 5.969109476280061, + "heldout_context_mape_pct": 3.0103379904473164, + "heldout_context_max_abs_error_pct": 5.490598706238697, + "prediction_min_ms": 0.05747733327249683, + "q_min_delta_ms": 0.0, + "kv_min_delta_ms": 0.0012048051417407057, + "monotone_and_nonnegative": true, + "tp": 4 + } + ], + "checks": { + "heldout_context_mape_le_5pct": true, + "monotone_and_nonnegative": true, + "profile_gate": true + } +} \ No newline at end of file diff --git a/runs/frontier-attn-structured-v0/results/trace-verdict.csv b/runs/frontier-attn-structured-v0/results/trace-verdict.csv new file mode 100644 index 0000000..be7687d --- /dev/null +++ b/runs/frontier-attn-structured-v0/results/trace-verdict.csv @@ -0,0 +1,85 @@ +cell,metric,quantile,old_bias,new_bias,abs_bias_delta_pp,validity +tp1_rho0p00125,ttft,mean,-0.13461915993830945,-0.06328385143753473,-7.133530850077471,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,ttft,p50,-0.1888032883165517,-0.1117698463137391,-7.7033442002812595,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,ttft,p90,-0.23223319000679374,-0.0884727580604037,-14.376043194639005,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,ttft,p99,-0.16824856840168442,-0.08585570369326061,-8.239286470842382,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,tpot,mean,0.1305653876178269,0.14303898259628217,1.2473594978455265,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,tpot,p50,0.16837991778063455,0.1688780144991309,0.04980967184963492,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,tpot,p90,0.012855564422932954,0.02293841255516459,1.0082848132231637,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,tpot,p99,-0.08151080694091946,-0.06480973978913974,-1.6701067151779714,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,e2e,mean,0.06539157436639341,0.0838094906426525,1.8417916276259092,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,e2e,p50,0.10867934171755954,0.11142188099086506,0.2742539273305519,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,e2e,p90,0.10946179160934073,0.11872607178176105,0.9264280172420314,GATE_FAIL_DIAGNOSTIC +tp1_rho0p00125,e2e,p99,-0.0657146472433028,-0.04585643943465822,-1.9858207808644577,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,ttft,mean,-0.17677087458166194,-0.09487906467830536,-8.189180990335657,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,ttft,p50,0.008311436147272566,0.015508635458377175,0.7197199311104608,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,ttft,p90,-0.2608553178054708,-0.129865645852306,-13.098967195316478,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,ttft,p99,-0.22116442296103195,-0.14410151018893788,-7.706291277209407,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,tpot,mean,0.014380733682430142,0.052302860177306544,3.7922126494876403,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,tpot,p50,0.13351145963877706,0.14244154194999165,0.8930082311214588,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,tpot,p90,-0.05456950130438105,0.018069551387063856,-3.649994991731719,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,tpot,p99,-0.23635406197836167,-0.1790206784227079,-5.733338355565376,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,e2e,mean,-0.020164189983441452,0.01563919326555167,-0.4524996717889782,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,e2e,p50,0.07632815851795742,0.10505951594320918,2.8731357425251765,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,e2e,p90,0.049809009042946335,0.08165607475145953,3.1847065708513194,GATE_FAIL_DIAGNOSTIC +tp1_rho0p0025,e2e,p99,-0.18303183751478602,-0.15333409130015813,-2.96977462146279,GATE_FAIL_DIAGNOSTIC +tp2_rho0p0025,ttft,mean,-0.11161154024124531,-0.045433491316312524,-6.617804892493279,PASS_SUBCRITICAL +tp2_rho0p0025,ttft,p50,-0.18420934047220774,-0.0875990583320057,-9.661028214020204,PASS_SUBCRITICAL +tp2_rho0p0025,ttft,p90,-0.21836264490202395,-0.1142200855839675,-10.414255931805645,PASS_SUBCRITICAL +tp2_rho0p0025,ttft,p99,-0.17283197594971011,-0.07674087497700505,-9.609110097270507,PASS_SUBCRITICAL +tp2_rho0p0025,tpot,mean,0.13711499081487563,0.15073709978689778,1.362210897202215,PASS_SUBCRITICAL +tp2_rho0p0025,tpot,p50,0.17555321305308488,0.18527285925405948,0.9719646200974597,PASS_SUBCRITICAL +tp2_rho0p0025,tpot,p90,0.0591579975137338,0.07863996768588354,1.9481970172149734,PASS_SUBCRITICAL +tp2_rho0p0025,tpot,p99,0.1300483675091633,0.1661995397125918,3.6151172203428503,PASS_SUBCRITICAL +tp2_rho0p0025,e2e,mean,0.10925353865257875,0.12696146154030977,1.7707922887731016,PASS_SUBCRITICAL +tp2_rho0p0025,e2e,p50,0.1339165600755408,0.14796251184179712,1.404595176625631,PASS_SUBCRITICAL +tp2_rho0p0025,e2e,p90,0.10666828724664539,0.12767235986604622,2.1004072619400835,PASS_SUBCRITICAL +tp2_rho0p0025,e2e,p99,-0.07437942975605877,-0.03965040123988098,-3.472902851617779,PASS_SUBCRITICAL +tp2_rho0p005,ttft,mean,-0.14129871969878843,-0.07082500585057615,-7.047371384821228,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,ttft,p50,-0.2042045530944649,-0.1284088888361358,-7.579566425832909,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,ttft,p90,-0.19003454588767263,-0.111136573344055,-7.889797254361763,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,ttft,p99,-0.19373351009539902,-0.09012556161973535,-10.360794847566366,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,tpot,mean,0.03117337438562004,0.05457336599027876,2.3399991604658723,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,tpot,p50,0.07668249597302182,0.0872512146277895,1.0568718654767675,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,tpot,p90,-0.03724827658429621,0.01132352325010614,-2.5924753334190074,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,tpot,p99,-0.10050436157089844,-0.05948329733667248,-4.102106423422596,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,e2e,mean,0.023867807624916495,0.050164236759908075,2.629642913499158,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,e2e,p50,0.07536668131278851,0.0912493994395978,1.588271812680929,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,e2e,p90,-0.03100470462321266,-0.0004979191794830456,-3.0506785443729614,GATE_FAIL_DIAGNOSTIC +tp2_rho0p005,e2e,p99,-0.04909581632382212,-0.0014369075488634014,-4.765890877495872,GATE_FAIL_DIAGNOSTIC +tp4_rho0p0025,ttft,mean,0.025022574738277282,0.07766646565061346,5.2643890912336175,PASS_SUBCRITICAL +tp4_rho0p0025,ttft,p50,-0.040042171846277425,0.029153379043297147,-1.0888792802980278,PASS_SUBCRITICAL +tp4_rho0p0025,ttft,p90,-0.056230097634382616,0.025222989748299444,-3.100710788608317,PASS_SUBCRITICAL +tp4_rho0p0025,ttft,p99,-0.07535478089127973,0.0205549324689376,-5.479984842234213,PASS_SUBCRITICAL +tp4_rho0p0025,tpot,mean,0.2170232618103144,0.2295170016795841,1.2493739869269715,PASS_SUBCRITICAL +tp4_rho0p0025,tpot,p50,0.22406751004936917,0.22406940610958842,0.00018960602192474862,PASS_SUBCRITICAL +tp4_rho0p0025,tpot,p90,0.1725668492041552,0.1795965483385546,0.7029699134399409,PASS_SUBCRITICAL +tp4_rho0p0025,tpot,p99,0.1603764334794579,0.254199450927048,9.382301744759008,PASS_SUBCRITICAL +tp4_rho0p0025,e2e,mean,0.18328079455378776,0.19190660476104554,0.8625810207257778,PASS_SUBCRITICAL +tp4_rho0p0025,e2e,p50,0.2057109615696404,0.21253242300694286,0.6821461437302473,PASS_SUBCRITICAL +tp4_rho0p0025,e2e,p90,0.1869703879211648,0.19279855916666536,0.5828171245500557,PASS_SUBCRITICAL +tp4_rho0p0025,e2e,p99,0.14838304065885655,0.15160588346499027,0.3222842806133719,PASS_SUBCRITICAL +tp4_rho0p005,ttft,mean,0.028648879997638963,0.08356973519379125,5.492085519615229,PASS_SUBCRITICAL +tp4_rho0p005,ttft,p50,0.028237979190582876,0.09021324737193111,6.197526818134823,PASS_SUBCRITICAL +tp4_rho0p005,ttft,p90,-0.055961238578361966,-0.0006012940489499138,-5.535994452941205,PASS_SUBCRITICAL +tp4_rho0p005,ttft,p99,-0.045491187615110146,0.05253708684194205,0.7045899226831902,PASS_SUBCRITICAL +tp4_rho0p005,tpot,mean,0.1707193936623511,0.1813506819061229,1.0631288243771824,PASS_SUBCRITICAL +tp4_rho0p005,tpot,p50,0.16851374859025317,0.1743324539248605,0.5818705334607321,PASS_SUBCRITICAL +tp4_rho0p005,tpot,p90,0.10492621353626864,0.11816822907155744,1.3242015535288796,PASS_SUBCRITICAL +tp4_rho0p005,tpot,p99,0.31231888769471766,0.3662267201704473,5.390783247572961,PASS_SUBCRITICAL +tp4_rho0p005,e2e,mean,0.151102823467785,0.1630793421767109,1.197651870892591,PASS_SUBCRITICAL +tp4_rho0p005,e2e,p50,0.1594213531394918,0.17360527090575292,1.418391776626113,PASS_SUBCRITICAL +tp4_rho0p005,e2e,p90,0.1266352410718406,0.13652104764058856,0.9885806568747962,PASS_SUBCRITICAL +tp4_rho0p005,e2e,p99,0.15576537699445703,0.17677450343779522,2.100912644333819,PASS_SUBCRITICAL +tp4_rho0p01,ttft,mean,-0.0014652143973501086,0.059650620031540064,5.8185405634189955,PASS_SUBCRITICAL +tp4_rho0p01,ttft,p50,0.22936601881498542,0.24159106387124998,1.2225045056264565,PASS_SUBCRITICAL +tp4_rho0p01,ttft,p90,-0.0646093465409875,-0.011548419893895705,-5.30609266470918,PASS_SUBCRITICAL +tp4_rho0p01,ttft,p99,-0.09621678853313553,-0.015475807392170575,-8.074098114096495,PASS_SUBCRITICAL +tp4_rho0p01,tpot,mean,0.06675609059150077,0.10596101263201793,3.9204922040517163,PASS_SUBCRITICAL +tp4_rho0p01,tpot,p50,0.08477302587551214,0.09703754188844527,1.2264516012933129,PASS_SUBCRITICAL +tp4_rho0p01,tpot,p90,0.0379183273767328,0.08369800201750718,4.577967464077439,PASS_SUBCRITICAL +tp4_rho0p01,tpot,p99,-0.010663954751357074,0.06667813160571406,5.601417685435699,PASS_SUBCRITICAL +tp4_rho0p01,e2e,mean,0.07212904317306096,0.09777288053702092,2.564383736395996,PASS_SUBCRITICAL +tp4_rho0p01,e2e,p50,0.12069215463307655,0.1388377217196832,1.8145567086606653,PASS_SUBCRITICAL +tp4_rho0p01,e2e,p90,0.03317293927357903,0.058060760567474216,2.4887821293895183,PASS_SUBCRITICAL +tp4_rho0p01,e2e,p99,0.0338378271163308,0.06380410696893425,2.996627985260345,PASS_SUBCRITICAL diff --git a/runs/frontier-attn-structured-v0/results/trace-verdict.json b/runs/frontier-attn-structured-v0/results/trace-verdict.json new file mode 100644 index 0000000..07ed0fc --- /dev/null +++ b/runs/frontier-attn-structured-v0/results/trace-verdict.json @@ -0,0 +1,2219 @@ +{ + "schema": "frontier-attn-structured-trial-aware-verdict-v1", + "metric_note": "Primary values are per-real-trial distribution biases with request alignment by index. legacy_pooled reproduces the old milestone quantile convention only for direct comparison.", + "cells": { + "tp1_rho0p00125": { + "old": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": -0.13148990670086547, + "p50": -0.18680026171265385, + "p90": -0.2281339774144484, + "p99": -0.14956660473341468 + }, + "tpot": { + "mean": 0.1393089430926731, + "p50": 0.17688566049293195, + "p90": 0.021918816469380976, + "p99": -0.06745527151165458 + }, + "e2e": { + "mean": 0.0728867501966229, + "p50": 0.11334615386654034, + "p90": 0.11968616309773676, + "p99": -0.04061861992039894 + } + }, + { + "ttft": { + "mean": -0.13772594464937732, + "p50": -0.19220957147090617, + "p90": -0.2342597777065415, + "p99": -0.15374940938906825 + }, + "tpot": { + "mean": 0.12195501379179413, + "p50": 0.1610213245142966, + "p90": 0.005473527596398301, + "p99": -0.08127351500612069 + }, + "e2e": { + "mean": 0.05800039446459018, + "p50": 0.09970261087805131, + "p90": 0.10532164717109044, + "p99": -0.046957770855300444 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": -0.1346079256751214, + "min": -0.13772594464937732, + "max": -0.13148990670086547 + }, + "p50": { + "mean": -0.18950491659178, + "min": -0.19220957147090617, + "max": -0.18680026171265385 + }, + "p90": { + "mean": -0.23119687756049495, + "min": -0.2342597777065415, + "max": -0.2281339774144484 + }, + "p99": { + "mean": -0.15165800706124147, + "min": -0.15374940938906825, + "max": -0.14956660473341468 + } + }, + "tpot": { + "mean": { + "mean": 0.13063197844223362, + "min": 0.12195501379179413, + "max": 0.1393089430926731 + }, + "p50": { + "mean": 0.16895349250361427, + "min": 0.1610213245142966, + "max": 0.17688566049293195 + }, + "p90": { + "mean": 0.013696172032889638, + "min": 0.005473527596398301, + "max": 0.021918816469380976 + }, + "p99": { + "mean": -0.07436439325888763, + "min": -0.08127351500612069, + "max": -0.06745527151165458 + } + }, + "e2e": { + "mean": { + "mean": 0.06544357233060655, + "min": 0.05800039446459018, + "max": 0.0728867501966229 + }, + "p50": { + "mean": 0.10652438237229583, + "min": 0.09970261087805131, + "max": 0.11334615386654034 + }, + "p90": { + "mean": 0.1125039051344136, + "min": 0.10532164717109044, + "max": 0.11968616309773676 + }, + "p99": { + "mean": -0.04378819538784969, + "min": -0.046957770855300444, + "max": -0.04061861992039894 + } + } + }, + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": -0.13461915993830945, + "p50": -0.1888032883165517, + "p90": -0.23223319000679374, + "p99": -0.16824856840168442 + }, + "tpot": { + "mean": 0.1305653876178269, + "p50": 0.16837991778063455, + "p90": 0.012855564422932954, + "p99": -0.08151080694091946 + }, + "e2e": { + "mean": 0.06539157436639341, + "p50": 0.10867934171755954, + "p90": 0.10946179160934073, + "p99": -0.0657146472433028 + } + } + }, + "new": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": -0.05989664674711635, + "p50": -0.10957660686565794, + "p90": -0.0836059887499959, + "p99": -0.0653231144095356 + }, + "tpot": { + "mean": 0.1518790062372861, + "p50": 0.17738738332864315, + "p90": 0.032091888121311134, + "p99": -0.05049862983288826 + }, + "e2e": { + "mean": 0.09143423903966252, + "p50": 0.11610023742963112, + "p90": 0.12903581929912153, + "p99": -0.020226996786041312 + } + }, + { + "ttft": { + "mean": -0.06664673546990688, + "p50": -0.11549959936391153, + "p90": -0.09087881400402557, + "p99": -0.0699202655213567 + }, + "tpot": { + "mean": 0.1343336100051224, + "p50": 0.16151628416144917, + "p90": 0.015482888492302317, + "p99": -0.0645681331290643 + }, + "e2e": { + "mean": 0.07629053599972169, + "p50": 0.10242294441887236, + "p90": 0.11455135611418107, + "p99": -0.02670088618839733 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": -0.06327169110851161, + "min": -0.06664673546990688, + "max": -0.05989664674711635 + }, + "p50": { + "mean": -0.11253810311478474, + "min": -0.11549959936391153, + "max": -0.10957660686565794 + }, + "p90": { + "mean": -0.08724240137701074, + "min": -0.09087881400402557, + "max": -0.0836059887499959 + }, + "p99": { + "mean": -0.06762168996544615, + "min": -0.0699202655213567, + "max": -0.0653231144095356 + } + }, + "tpot": { + "mean": { + "mean": 0.14310630812120423, + "min": 0.1343336100051224, + "max": 0.1518790062372861 + }, + "p50": { + "mean": 0.16945183374504616, + "min": 0.16151628416144917, + "max": 0.17738738332864315 + }, + "p90": { + "mean": 0.023787388306806725, + "min": 0.015482888492302317, + "max": 0.032091888121311134 + }, + "p99": { + "mean": -0.05753338148097628, + "min": -0.0645681331290643, + "max": -0.05049862983288826 + } + }, + "e2e": { + "mean": { + "mean": 0.0838623875196921, + "min": 0.07629053599972169, + "max": 0.09143423903966252 + }, + "p50": { + "mean": 0.10926159092425174, + "min": 0.10242294441887236, + "max": 0.11610023742963112 + }, + "p90": { + "mean": 0.1217935877066513, + "min": 0.11455135611418107, + "max": 0.12903581929912153 + }, + "p99": { + "mean": -0.023463941487219322, + "min": -0.02670088618839733, + "max": -0.020226996786041312 + } + } + }, + "paired_relative_error": [ + { + "ttft": { + "mean": 0.3691695743143199, + "p50": -0.03099707280100344, + "p90": 1.7891828562343537, + "p99": 5.429960617790179 + }, + "tpot": { + "mean": 0.19492722441928262, + "p50": 0.15055142720578293, + "p90": 0.2670903700878269, + "p99": 0.6745762204781113 + }, + "e2e": { + "mean": 0.1844268367659844, + "p50": 0.1424666323844526, + "p90": 0.31842146555728224, + "p99": 0.8437286984487435 + } + }, + { + "ttft": { + "mean": 0.3669522667824523, + "p50": -0.03476798007471665, + "p90": 1.827734692042949, + "p99": 5.512577837970861 + }, + "tpot": { + "mean": 0.1783718677969176, + "p50": 0.13458290148059987, + "p90": 0.24952928603370794, + "p99": 0.6582817886518392 + }, + "e2e": { + "mean": 0.16858938086274747, + "p50": 0.12880754276426226, + "p90": 0.29822488192148, + "p99": 0.7895217968914289 + } + } + ], + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": -0.06328385143753473, + "p50": -0.1117698463137391, + "p90": -0.0884727580604037, + "p99": -0.08585570369326061 + }, + "tpot": { + "mean": 0.14303898259628217, + "p50": 0.1688780144991309, + "p90": 0.02293841255516459, + "p99": -0.06480973978913974 + }, + "e2e": { + "mean": 0.0838094906426525, + "p50": 0.11142188099086506, + "p90": 0.11872607178176105, + "p99": -0.04585643943465822 + } + }, + "waiting_p99_ms": 1337.0359972076221, + "validity": "GATE_FAIL_DIAGNOSTIC" + } + }, + "tp1_rho0p0025": { + "old": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": -0.178961187154444, + "p50": 0.004736647098298533, + "p90": -0.26795354312466707, + "p99": -0.22168753840926173 + }, + "tpot": { + "mean": 0.01036398277932227, + "p50": 0.13356090363025383, + "p90": -0.05541984044976886, + "p99": -0.26072314328609314 + }, + "e2e": { + "mean": -0.024645268935464575, + "p50": 0.0817280529155681, + "p90": 0.04955125280528564, + "p99": -0.1871016978849053 + } + }, + { + "ttft": { + "mean": -0.17456884440992257, + "p50": 0.011041256441842635, + "p90": -0.25172604669420906, + "p99": -0.21630340228356143 + }, + "tpot": { + "mean": 0.01842954963685723, + "p50": 0.13346201996043702, + "p90": -0.05215064358043298, + "p99": -0.15675916344669943 + }, + "e2e": { + "mean": -0.015641746084433396, + "p50": 0.07195840871873896, + "p90": 0.052264109329363505, + "p99": -0.17396512464775202 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": -0.1767650157821833, + "min": -0.178961187154444, + "max": -0.17456884440992257 + }, + "p50": { + "mean": 0.007888951770070584, + "min": 0.004736647098298533, + "max": 0.011041256441842635 + }, + "p90": { + "mean": -0.2598397949094381, + "min": -0.26795354312466707, + "max": -0.25172604669420906 + }, + "p99": { + "mean": -0.2189954703464116, + "min": -0.22168753840926173, + "max": -0.21630340228356143 + } + }, + "tpot": { + "mean": { + "mean": 0.01439676620808975, + "min": 0.01036398277932227, + "max": 0.01842954963685723 + }, + "p50": { + "mean": 0.13351146179534543, + "min": 0.13346201996043702, + "max": 0.13356090363025383 + }, + "p90": { + "mean": -0.05378524201510092, + "min": -0.05541984044976886, + "max": -0.05215064358043298 + }, + "p99": { + "mean": -0.2087411533663963, + "min": -0.26072314328609314, + "max": -0.15675916344669943 + } + }, + "e2e": { + "mean": { + "mean": -0.020143507509948984, + "min": -0.024645268935464575, + "max": -0.015641746084433396 + }, + "p50": { + "mean": 0.07684323081715352, + "min": 0.07195840871873896, + "max": 0.0817280529155681 + }, + "p90": { + "mean": 0.05090768106732457, + "min": 0.04955125280528564, + "max": 0.052264109329363505 + }, + "p99": { + "mean": -0.18053341126632866, + "min": -0.1871016978849053, + "max": -0.17396512464775202 + } + } + }, + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": -0.17677087458166194, + "p50": 0.008311436147272566, + "p90": -0.2608553178054708, + "p99": -0.22116442296103195 + }, + "tpot": { + "mean": 0.014380733682430142, + "p50": 0.13351145963877706, + "p90": -0.05456950130438105, + "p99": -0.23635406197836167 + }, + "e2e": { + "mean": -0.020164189983441452, + "p50": 0.07632815851795742, + "p90": 0.049809009042946335, + "p99": -0.18303183751478602 + } + } + }, + "new": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": -0.09728726150141453, + "p50": 0.011908330017980539, + "p90": -0.1382218037906623, + "p99": -0.14467638598472385 + }, + "tpot": { + "mean": 0.04813594501063553, + "p50": 0.1424913754734047, + "p90": 0.01715387922135076, + "p99": -0.2052193535460428 + }, + "e2e": { + "mean": 0.010994374852847301, + "p50": 0.11060355438719509, + "p90": 0.08139049920593726, + "p99": -0.15755189584478857 + } + }, + { + "ttft": { + "mean": -0.09245798463253013, + "p50": 0.018257940864430843, + "p90": -0.11911850444197748, + "p99": -0.13875950941310633 + }, + "tpot": { + "mean": 0.05650303913152255, + "p50": 0.14239171277369528, + "p90": 0.020674254113796777, + "p99": -0.0934499151356964 + }, + "e2e": { + "mean": 0.02032688810811885, + "p50": 0.10057312063737019, + "p90": 0.0841856530995708, + "p99": -0.1439377928383722 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": -0.09487262306697233, + "min": -0.09728726150141453, + "max": -0.09245798463253013 + }, + "p50": { + "mean": 0.015083135441205691, + "min": 0.011908330017980539, + "max": 0.018257940864430843 + }, + "p90": { + "mean": -0.12867015411631988, + "min": -0.1382218037906623, + "max": -0.11911850444197748 + }, + "p99": { + "mean": -0.1417179476989151, + "min": -0.14467638598472385, + "max": -0.13875950941310633 + } + }, + "tpot": { + "mean": { + "mean": 0.05231949207107904, + "min": 0.04813594501063553, + "max": 0.05650303913152255 + }, + "p50": { + "mean": 0.14244154412354998, + "min": 0.14239171277369528, + "max": 0.1424913754734047 + }, + "p90": { + "mean": 0.018914066667573767, + "min": 0.01715387922135076, + "max": 0.020674254113796777 + }, + "p99": { + "mean": -0.14933463434086958, + "min": -0.2052193535460428, + "max": -0.0934499151356964 + } + }, + "e2e": { + "mean": { + "mean": 0.015660631480483075, + "min": 0.010994374852847301, + "max": 0.02032688810811885 + }, + "p50": { + "mean": 0.10558833751228264, + "min": 0.10057312063737019, + "max": 0.11060355438719509 + }, + "p90": { + "mean": 0.08278807615275402, + "min": 0.08139049920593726, + "max": 0.0841856530995708 + }, + "p99": { + "mean": -0.1507448443415804, + "min": -0.15755189584478857, + "max": -0.1439377928383722 + } + } + }, + "paired_relative_error": [ + { + "ttft": { + "mean": 0.39789106943830893, + "p50": -0.037298357008305895, + "p90": 1.9730972451814688, + "p99": 6.018295451307029 + }, + "tpot": { + "mean": 0.1553028513608087, + "p50": 0.12385990892916658, + "p90": 0.2584968822537069, + "p99": 0.9658213367115374 + }, + "e2e": { + "mean": 0.1471398515879261, + "p50": 0.10601983073958936, + "p90": 0.32930377247615056, + "p99": 1.075635524765205 + } + }, + { + "ttft": { + "mean": 0.4002572158008619, + "p50": -0.031486174528958244, + "p90": 1.8666003636442456, + "p99": 5.991354563261557 + }, + "tpot": { + "mean": 0.1566771540679317, + "p50": 0.12379976526368523, + "p90": 0.25274583367589193, + "p99": 0.958693609453715 + }, + "e2e": { + "mean": 0.14911603622614725, + "p50": 0.10650308347135491, + "p90": 0.3195494670044913, + "p99": 1.1185032164459736 + } + } + ], + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": -0.09487906467830536, + "p50": 0.015508635458377175, + "p90": -0.129865645852306, + "p99": -0.14410151018893788 + }, + "tpot": { + "mean": 0.052302860177306544, + "p50": 0.14244154194999165, + "p90": 0.018069551387063856, + "p99": -0.1790206784227079 + }, + "e2e": { + "mean": 0.01563919326555167, + "p50": 0.10505951594320918, + "p90": 0.08165607475145953, + "p99": -0.15333409130015813 + } + }, + "waiting_p99_ms": 1892.1769691703946, + "validity": "GATE_FAIL_DIAGNOSTIC" + } + }, + "tp2_rho0p0025": { + "old": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": -0.11106335941203382, + "p50": -0.1855202869158692, + "p90": -0.21828233796243518, + "p99": -0.1688334681304367 + }, + "tpot": { + "mean": 0.1303208667385706, + "p50": 0.16909257797747365, + "p90": 0.05206789287995917, + "p99": 0.14191216788372232 + }, + "e2e": { + "mean": 0.1031266658038281, + "p50": 0.13020268054402595, + "p90": 0.09830535586040931, + "p99": -0.0581273596953024 + } + }, + { + "ttft": { + "mean": -0.11215904539344854, + "p50": -0.18289416715548218, + "p90": -0.21914116973979564, + "p99": -0.17209787391954623 + }, + "tpot": { + "mean": 0.14399128493700397, + "p50": 0.1866331158045056, + "p90": 0.06642456306402275, + "p99": 0.15612928840488144 + }, + "e2e": { + "mean": 0.11544885011206724, + "p50": 0.14502095370348014, + "p90": 0.10849722315778151, + "p99": -0.045303035491763635 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": -0.11161120240274118, + "min": -0.11215904539344854, + "max": -0.11106335941203382 + }, + "p50": { + "mean": -0.18420722703567569, + "min": -0.1855202869158692, + "max": -0.18289416715548218 + }, + "p90": { + "mean": -0.2187117538511154, + "min": -0.21914116973979564, + "max": -0.21828233796243518 + }, + "p99": { + "mean": -0.17046567102499147, + "min": -0.17209787391954623, + "max": -0.1688334681304367 + } + }, + "tpot": { + "mean": { + "mean": 0.13715607583778727, + "min": 0.1303208667385706, + "max": 0.14399128493700397 + }, + "p50": { + "mean": 0.17786284689098963, + "min": 0.16909257797747365, + "max": 0.1866331158045056 + }, + "p90": { + "mean": 0.05924622797199096, + "min": 0.05206789287995917, + "max": 0.06642456306402275 + }, + "p99": { + "mean": 0.1490207281443019, + "min": 0.14191216788372232, + "max": 0.15612928840488144 + } + }, + "e2e": { + "mean": { + "mean": 0.10928775795794766, + "min": 0.1031266658038281, + "max": 0.11544885011206724 + }, + "p50": { + "mean": 0.13761181712375303, + "min": 0.13020268054402595, + "max": 0.14502095370348014 + }, + "p90": { + "mean": 0.10340128950909541, + "min": 0.09830535586040931, + "max": 0.10849722315778151 + }, + "p99": { + "mean": -0.05171519759353302, + "min": -0.0581273596953024, + "max": -0.045303035491763635 + } + } + }, + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": -0.11161154024124531, + "p50": -0.18420934047220774, + "p90": -0.21836264490202395, + "p99": -0.17283197594971011 + }, + "tpot": { + "mean": 0.13711499081487563, + "p50": 0.17555321305308488, + "p90": 0.0591579975137338, + "p99": 0.1300483675091633 + }, + "e2e": { + "mean": 0.10925353865257875, + "p50": 0.1339165600755408, + "p90": 0.10666828724664539, + "p99": -0.07437942975605877 + } + } + }, + "new": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": -0.044844475267623835, + "p50": -0.08906525404741798, + "p90": -0.11412907883561804, + "p99": -0.07227786537885415 + }, + "tpot": { + "mean": 0.14386158526258666, + "p50": 0.1787588067010671, + "p90": 0.071419448886029, + "p99": 0.17844287277146695 + }, + "e2e": { + "mean": 0.12073678040141796, + "p50": 0.14420262806746403, + "p90": 0.11915070375608593, + "p99": -0.022788557992646938 + } + }, + { + "ttft": { + "mean": -0.046021781355297144, + "p50": -0.08612813517467069, + "p90": -0.11510233828040589, + "p99": -0.07592149320892366 + }, + "tpot": { + "mean": 0.15769576871598226, + "p50": 0.196444372264794, + "p90": 0.08604019319402298, + "p99": 0.19311481061458743 + }, + "e2e": { + "mean": 0.13325567383154438, + "p50": 0.15920445684060733, + "p90": 0.12953600816854335, + "p99": -0.009483068681846096 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": -0.045433128311460486, + "min": -0.046021781355297144, + "max": -0.044844475267623835 + }, + "p50": { + "mean": -0.08759669461104433, + "min": -0.08906525404741798, + "max": -0.08612813517467069 + }, + "p90": { + "mean": -0.11461570855801197, + "min": -0.11510233828040589, + "max": -0.11412907883561804 + }, + "p99": { + "mean": -0.07409967929388891, + "min": -0.07592149320892366, + "max": -0.07227786537885415 + } + }, + "tpot": { + "mean": { + "mean": 0.15077867698928446, + "min": 0.14386158526258666, + "max": 0.15769576871598226 + }, + "p50": { + "mean": 0.18760158948293054, + "min": 0.1787588067010671, + "max": 0.196444372264794 + }, + "p90": { + "mean": 0.07872982104002599, + "min": 0.071419448886029, + "max": 0.08604019319402298 + }, + "p99": { + "mean": 0.1857788416930272, + "min": 0.17844287277146695, + "max": 0.19311481061458743 + } + }, + "e2e": { + "mean": { + "mean": 0.12699622711648118, + "min": 0.12073678040141796, + "max": 0.13325567383154438 + }, + "p50": { + "mean": 0.15170354245403567, + "min": 0.14420262806746403, + "max": 0.15920445684060733 + }, + "p90": { + "mean": 0.12434335596231463, + "min": 0.11915070375608593, + "max": 0.12953600816854335 + }, + "p99": { + "mean": -0.016135813337246518, + "min": -0.022788557992646938, + "max": -0.009483068681846096 + } + } + }, + "paired_relative_error": [ + { + "ttft": { + "mean": 0.28317282752846445, + "p50": 0.0015652470674283874, + "p90": 0.7651897154483899, + "p99": 4.595783098224935 + }, + "tpot": { + "mean": 0.1752842205127379, + "p50": 0.17053139492828456, + "p90": 0.24211599227572878, + "p99": 0.7603540706787565 + }, + "e2e": { + "mean": 0.16397230912194014, + "p50": 0.14764836952252053, + "p90": 0.26575788904393577, + "p99": 0.6911771461107379 + } + }, + { + "ttft": { + "mean": 0.27885319671790226, + "p50": 0.0008797795879422852, + "p90": 0.7521533350007815, + "p99": 4.589795377891504 + }, + "tpot": { + "mean": 0.18903026397070546, + "p50": 0.18546796656470424, + "p90": 0.25643004508741135, + "p99": 0.7281958169280102 + }, + "e2e": { + "mean": 0.17634926693766606, + "p50": 0.15964733210774384, + "p90": 0.2801020190882037, + "p99": 0.7024304395510836 + } + } + ], + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": -0.045433491316312524, + "p50": -0.0875990583320057, + "p90": -0.1142200855839675, + "p99": -0.07674087497700505 + }, + "tpot": { + "mean": 0.15073709978689778, + "p50": 0.18527285925405948, + "p90": 0.07863996768588354, + "p99": 0.1661995397125918 + }, + "e2e": { + "mean": 0.12696146154030977, + "p50": 0.14796251184179712, + "p90": 0.12767235986604622, + "p99": -0.03965040123988098 + } + }, + "waiting_p99_ms": 777.5911879793948, + "validity": "PASS_SUBCRITICAL" + } + }, + "tp2_rho0p005": { + "old": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": -0.141581861927082, + "p50": -0.2051822104110931, + "p90": -0.1895782660551383, + "p99": -0.19346662831715172 + }, + "tpot": { + "mean": 0.025318500347963065, + "p50": 0.07182548360524435, + "p90": -0.04507556167777899, + "p99": -0.10054590154610009 + }, + "e2e": { + "mean": 0.01846906077214167, + "p50": 0.07205509683921338, + "p90": -0.037157045612690995, + "p99": -0.051216767638022506 + } + }, + { + "ttft": { + "mean": -0.1410153906245514, + "p50": -0.20356790487770302, + "p90": -0.18910363664562158, + "p99": -0.192398015096606 + }, + "tpot": { + "mean": 0.03709549859113398, + "p50": 0.08312549432043648, + "p90": -0.029759535186826266, + "p99": -0.09947054412002211 + }, + "e2e": { + "mean": 0.029324095335773688, + "p50": 0.07587484709261458, + "p90": -0.027431862141837213, + "p99": -0.04314368195514718 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": -0.14129862627581669, + "min": -0.141581861927082, + "max": -0.1410153906245514 + }, + "p50": { + "mean": -0.20437505764439806, + "min": -0.2051822104110931, + "max": -0.20356790487770302 + }, + "p90": { + "mean": -0.18934095135037993, + "min": -0.1895782660551383, + "max": -0.18910363664562158 + }, + "p99": { + "mean": -0.19293232170687885, + "min": -0.19346662831715172, + "max": -0.192398015096606 + } + }, + "tpot": { + "mean": { + "mean": 0.031206999469548524, + "min": 0.025318500347963065, + "max": 0.03709549859113398 + }, + "p50": { + "mean": 0.07747548896284041, + "min": 0.07182548360524435, + "max": 0.08312549432043648 + }, + "p90": { + "mean": -0.03741754843230263, + "min": -0.04507556167777899, + "max": -0.029759535186826266 + }, + "p99": { + "mean": -0.1000082228330611, + "min": -0.10054590154610009, + "max": -0.09947054412002211 + } + }, + "e2e": { + "mean": { + "mean": 0.02389657805395768, + "min": 0.01846906077214167, + "max": 0.029324095335773688 + }, + "p50": { + "mean": 0.07396497196591398, + "min": 0.07205509683921338, + "max": 0.07587484709261458 + }, + "p90": { + "mean": -0.0322944538772641, + "min": -0.037157045612690995, + "max": -0.027431862141837213 + }, + "p99": { + "mean": -0.047180224796584846, + "min": -0.051216767638022506, + "max": -0.04314368195514718 + } + } + }, + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": -0.14129871969878843, + "p50": -0.2042045530944649, + "p90": -0.19003454588767263, + "p99": -0.19373351009539902 + }, + "tpot": { + "mean": 0.03117337438562004, + "p50": 0.07668249597302182, + "p90": -0.03724827658429621, + "p99": -0.10050436157089844 + }, + "e2e": { + "mean": 0.023867807624916495, + "p50": 0.07536668131278851, + "p90": -0.03100470462321266, + "p99": -0.04909581632382212 + } + } + }, + "new": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": -0.07113138559444453, + "p50": -0.1294796632798084, + "p90": -0.11063584772248519, + "p99": -0.08982438463776078 + }, + "tpot": { + "mean": 0.048585629713612626, + "p50": 0.08234652581183928, + "p90": 0.003101343693636994, + "p99": -0.05952673172238401 + }, + "e2e": { + "mean": 0.04462683161260099, + "p50": 0.0878889042422101, + "p90": -0.006843953727060337, + "p99": -0.0036641600308731643 + } + }, + { + "ttft": { + "mean": -0.07051842392629604, + "p50": -0.12771160295843229, + "p90": -0.11011498510890981, + "p99": -0.08861845103384182 + }, + "tpot": { + "mean": 0.060629878515092876, + "p50": 0.09375745746657616, + "p90": 0.019190079238111446, + "p99": -0.05840233314016777 + }, + "e2e": { + "mean": 0.05576066061144686, + "p50": 0.09176507061640418, + "p90": 0.003187406757202862, + "p99": 0.004813545234813467 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": -0.07082490476037029, + "min": -0.07113138559444453, + "max": -0.07051842392629604 + }, + "p50": { + "mean": -0.12859563311912034, + "min": -0.1294796632798084, + "max": -0.12771160295843229 + }, + "p90": { + "mean": -0.11037541641569751, + "min": -0.11063584772248519, + "max": -0.11011498510890981 + }, + "p99": { + "mean": -0.0892214178358013, + "min": -0.08982438463776078, + "max": -0.08861845103384182 + } + }, + "tpot": { + "mean": { + "mean": 0.05460775411435275, + "min": 0.048585629713612626, + "max": 0.060629878515092876 + }, + "p50": { + "mean": 0.08805199163920771, + "min": 0.08234652581183928, + "max": 0.09375745746657616 + }, + "p90": { + "mean": 0.01114571146587422, + "min": 0.003101343693636994, + "max": 0.019190079238111446 + }, + "p99": { + "mean": -0.05896453243127589, + "min": -0.05952673172238401, + "max": -0.05840233314016777 + } + }, + "e2e": { + "mean": { + "mean": 0.05019374611202393, + "min": 0.04462683161260099, + "max": 0.05576066061144686 + }, + "p50": { + "mean": 0.08982698742930714, + "min": 0.0878889042422101, + "max": 0.09176507061640418 + }, + "p90": { + "mean": -0.0018282734849287376, + "min": -0.006843953727060337, + "max": 0.003187406757202862 + }, + "p99": { + "mean": 0.0005746926019701514, + "min": -0.0036641600308731643, + "max": 0.004813545234813467 + } + } + }, + "paired_relative_error": [ + { + "ttft": { + "mean": 0.3165020807873401, + "p50": 0.008761867404954687, + "p90": 1.027515859973651, + "p99": 4.618320663100516 + }, + "tpot": { + "mean": 0.09697936002481403, + "p50": 0.08127529560627057, + "p90": 0.21395969552811614, + "p99": 0.6138944058729238 + }, + "e2e": { + "mean": 0.09434768590630697, + "p50": 0.07464475818134242, + "p90": 0.22980866153598797, + "p99": 0.6060590884554267 + } + }, + { + "ttft": { + "mean": 0.3247210835745076, + "p50": 0.00808317178222498, + "p90": 1.0875948711956118, + "p99": 4.791245921444296 + }, + "tpot": { + "mean": 0.10993925696112139, + "p50": 0.09402822239992542, + "p90": 0.22283625990015465, + "p99": 0.6083519770881531 + }, + "e2e": { + "mean": 0.10658386724241943, + "p50": 0.08490549835786211, + "p90": 0.2445084767702898, + "p99": 0.6267174054324888 + } + } + ], + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": -0.07082500585057615, + "p50": -0.1284088888361358, + "p90": -0.111136573344055, + "p99": -0.09012556161973535 + }, + "tpot": { + "mean": 0.05457336599027876, + "p50": 0.0872512146277895, + "p90": 0.01132352325010614, + "p99": -0.05948329733667248 + }, + "e2e": { + "mean": 0.050164236759908075, + "p50": 0.0912493994395978, + "p90": -0.0004979191794830456, + "p99": -0.0014369075488634014 + } + }, + "waiting_p99_ms": 1166.1196883368564, + "validity": "GATE_FAIL_DIAGNOSTIC" + } + }, + "tp4_rho0p0025": { + "old": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": 0.025153752214840692, + "p50": -0.04039475356731359, + "p90": -0.05617389473222491, + "p99": -0.07293111999637672 + }, + "tpot": { + "mean": 0.21469140550335514, + "p50": 0.22290333666146916, + "p90": 0.1721777151300294, + "p99": 0.1622573651896084 + }, + "e2e": { + "mean": 0.18156895570089593, + "p50": 0.20381782987468136, + "p90": 0.18582117345248075, + "p99": 0.15166386730143064 + } + }, + { + "ttft": { + "mean": 0.024891430828052088, + "p50": -0.03968933093624011, + "p90": -0.05623634198811063, + "p99": -0.0733553458724018 + }, + "tpot": { + "mean": 0.21936408831731027, + "p50": 0.2282296491940284, + "p90": 0.17345944256977727, + "p99": 0.16387998107689886 + }, + "e2e": { + "mean": 0.18499760077470503, + "p50": 0.2077274237083677, + "p90": 0.18741568436689596, + "p99": 0.15614012298902055 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": 0.02502259152144639, + "min": 0.024891430828052088, + "max": 0.025153752214840692 + }, + "p50": { + "mean": -0.04004204225177685, + "min": -0.04039475356731359, + "max": -0.03968933093624011 + }, + "p90": { + "mean": -0.05620511836016777, + "min": -0.05623634198811063, + "max": -0.05617389473222491 + }, + "p99": { + "mean": -0.07314323293438926, + "min": -0.0733553458724018, + "max": -0.07293111999637672 + } + }, + "tpot": { + "mean": { + "mean": 0.21702774691033272, + "min": 0.21469140550335514, + "max": 0.21936408831731027 + }, + "p50": { + "mean": 0.22556649292774877, + "min": 0.22290333666146916, + "max": 0.2282296491940284 + }, + "p90": { + "mean": 0.17281857884990334, + "min": 0.1721777151300294, + "max": 0.17345944256977727 + }, + "p99": { + "mean": 0.16306867313325363, + "min": 0.1622573651896084, + "max": 0.16387998107689886 + } + }, + "e2e": { + "mean": { + "mean": 0.1832832782378005, + "min": 0.18156895570089593, + "max": 0.18499760077470503 + }, + "p50": { + "mean": 0.20577262679152453, + "min": 0.20381782987468136, + "max": 0.2077274237083677 + }, + "p90": { + "mean": 0.18661842890968836, + "min": 0.18582117345248075, + "max": 0.18741568436689596 + }, + "p99": { + "mean": 0.1539019951452256, + "min": 0.15166386730143064, + "max": 0.15614012298902055 + } + } + }, + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": 0.025022574738277282, + "p50": -0.040042171846277425, + "p90": -0.056230097634382616, + "p99": -0.07535478089127973 + }, + "tpot": { + "mean": 0.2170232618103144, + "p50": 0.22406751004936917, + "p90": 0.1725668492041552, + "p99": 0.1603764334794579 + }, + "e2e": { + "mean": 0.18328079455378776, + "p50": 0.2057109615696404, + "p90": 0.1869703879211648, + "p99": 0.14838304065885655 + } + } + }, + "new": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": 0.07780438024003325, + "p50": 0.028775382574128122, + "p90": 0.025284043303025253, + "p99": 0.02322998991779303 + }, + "tpot": { + "mean": 0.2271612069590991, + "p50": 0.22290523091840317, + "p90": 0.17920508135226615, + "p99": 0.25623246663646587 + }, + "e2e": { + "mean": 0.19018228704673856, + "p50": 0.21062858068127338, + "p90": 0.1916437019129637, + "p99": 0.15489591747842305 + } + }, + { + "ttft": { + "mean": 0.07752858635145746, + "p50": 0.029531653384293997, + "p90": 0.025216206468834785, + "p99": 0.022761760805360378 + }, + "tpot": { + "mean": 0.2318818586865509, + "p50": 0.22823155170133197, + "p90": 0.1804944929238653, + "p99": 0.2579862802233231 + }, + "e2e": { + "mean": 0.19363592605420304, + "p50": 0.21456029353388054, + "p90": 0.1932460420729579, + "p99": 0.15938473541043005 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": 0.07766648329574535, + "min": 0.07752858635145746, + "max": 0.07780438024003325 + }, + "p50": { + "mean": 0.02915351797921106, + "min": 0.028775382574128122, + "max": 0.029531653384293997 + }, + "p90": { + "mean": 0.02525012488593002, + "min": 0.025216206468834785, + "max": 0.025284043303025253 + }, + "p99": { + "mean": 0.022995875361576705, + "min": 0.022761760805360378, + "max": 0.02322998991779303 + } + }, + "tpot": { + "mean": { + "mean": 0.229521532822825, + "min": 0.2271612069590991, + "max": 0.2318818586865509 + }, + "p50": { + "mean": 0.22556839130986756, + "min": 0.22290523091840317, + "max": 0.22823155170133197 + }, + "p90": { + "mean": 0.1798497871380657, + "min": 0.17920508135226615, + "max": 0.1804944929238653 + }, + "p99": { + "mean": 0.2571093734298945, + "min": 0.25623246663646587, + "max": 0.2579862802233231 + } + }, + "e2e": { + "mean": { + "mean": 0.1919091065504708, + "min": 0.19018228704673856, + "max": 0.19363592605420304 + }, + "p50": { + "mean": 0.21259443710757697, + "min": 0.21062858068127338, + "max": 0.21456029353388054 + }, + "p90": { + "mean": 0.1924448719929608, + "min": 0.1916437019129637, + "max": 0.1932460420729579 + }, + "p99": { + "mean": 0.15714032644442655, + "min": 0.15489591747842305, + "max": 0.15938473541043005 + } + } + }, + "paired_relative_error": [ + { + "ttft": { + "mean": 0.28957904842187726, + "p50": 0.11967766737369243, + "p90": 0.5365251338257485, + "p99": 3.680446479340459 + }, + "tpot": { + "mean": 0.2360056579049085, + "p50": 0.21333501659187393, + "p90": 0.27188619704554073, + "p99": 0.8481115630045084 + }, + "e2e": { + "mean": 0.22033571422861917, + "p50": 0.20586761334949796, + "p90": 0.282293891126381, + "p99": 0.6891849392053429 + } + }, + { + "ttft": { + "mean": 0.28322292774037994, + "p50": 0.12089296869324834, + "p90": 0.5087862765467016, + "p99": 3.551749946511281 + }, + "tpot": { + "mean": 0.24183792864428846, + "p50": 0.21903748389459432, + "p90": 0.2783964492106047, + "p99": 0.973107877920707 + }, + "e2e": { + "mean": 0.2239979384097061, + "p50": 0.21064946414808636, + "p90": 0.28840198884356866, + "p99": 0.6983274345617249 + } + } + ], + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": 0.07766646565061346, + "p50": 0.029153379043297147, + "p90": 0.025222989748299444, + "p99": 0.0205549324689376 + }, + "tpot": { + "mean": 0.2295170016795841, + "p50": 0.22406940610958842, + "p90": 0.1795965483385546, + "p99": 0.254199450927048 + }, + "e2e": { + "mean": 0.19190660476104554, + "p50": 0.21253242300694286, + "p90": 0.19279855916666536, + "p99": 0.15160588346499027 + } + }, + "waiting_p99_ms": 302.3540478261558, + "validity": "PASS_SUBCRITICAL" + } + }, + "tp4_rho0p005": { + "old": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": 0.02817612266853226, + "p50": 0.029949406527602123, + "p90": -0.05512408243570239, + "p99": -0.04532941341343785 + }, + "tpot": { + "mean": 0.17688090609346543, + "p50": 0.1740795026677315, + "p90": 0.11018831255907795, + "p99": 0.3192029274754835 + }, + "e2e": { + "mean": 0.1569382150647865, + "p50": 0.1678105991098637, + "p90": 0.12797847572660626, + "p99": 0.16781804606311032 + } + }, + { + "ttft": { + "mean": 0.02912207227616738, + "p50": 0.027190990310942655, + "p90": -0.056337427761921965, + "p99": -0.04121672646120952 + }, + "tpot": { + "mean": 0.16462206191460754, + "p50": 0.16195538436894344, + "p90": 0.09988703619210754, + "p99": 0.31352142471539024 + }, + "e2e": { + "mean": 0.1453260018192159, + "p50": 0.15247501518771847, + "p90": 0.12213267160696055, + "p99": 0.15897391264426655 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": 0.028649097472349817, + "min": 0.02817612266853226, + "max": 0.02912207227616738 + }, + "p50": { + "mean": 0.02857019841927239, + "min": 0.027190990310942655, + "max": 0.029949406527602123 + }, + "p90": { + "mean": -0.055730755098812174, + "min": -0.056337427761921965, + "max": -0.05512408243570239 + }, + "p99": { + "mean": -0.04327306993732369, + "min": -0.04532941341343785, + "max": -0.04121672646120952 + } + }, + "tpot": { + "mean": { + "mean": 0.1707514840040365, + "min": 0.16462206191460754, + "max": 0.17688090609346543 + }, + "p50": { + "mean": 0.16801744351833747, + "min": 0.16195538436894344, + "max": 0.1740795026677315 + }, + "p90": { + "mean": 0.10503767437559275, + "min": 0.09988703619210754, + "max": 0.11018831255907795 + }, + "p99": { + "mean": 0.3163621760954369, + "min": 0.31352142471539024, + "max": 0.3192029274754835 + } + }, + "e2e": { + "mean": { + "mean": 0.1511321084420012, + "min": 0.1453260018192159, + "max": 0.1569382150647865 + }, + "p50": { + "mean": 0.1601428071487911, + "min": 0.15247501518771847, + "max": 0.1678105991098637 + }, + "p90": { + "mean": 0.1250555736667834, + "min": 0.12213267160696055, + "max": 0.12797847572660626 + }, + "p99": { + "mean": 0.16339597935368844, + "min": 0.15897391264426655, + "max": 0.16781804606311032 + } + } + }, + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": 0.028648879997638963, + "p50": 0.028237979190582876, + "p90": -0.055961238578361966, + "p99": -0.045491187615110146 + }, + "tpot": { + "mean": 0.1707193936623511, + "p50": 0.16851374859025317, + "p90": 0.10492621353626864, + "p99": 0.31231888769471766 + }, + "e2e": { + "mean": 0.151102823467785, + "p50": 0.1594213531394918, + "p90": 0.1266352410718406, + "p99": 0.15576537699445703 + } + } + }, + "new": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": 0.0830717367573255, + "p50": 0.09202782803564245, + "p90": 0.00028495427033840576, + "p99": 0.052715475291315185 + }, + "tpot": { + "mean": 0.1875681469549417, + "p50": 0.17992592310879849, + "p90": 0.12349339185025797, + "p99": 0.37339354462095886 + }, + "e2e": { + "mean": 0.16897544744343032, + "p50": 0.18209714770541094, + "p90": 0.13787606868147936, + "p99": 0.18904626199769245 + } + }, + { + "ttft": { + "mean": 0.08406819180217444, + "p50": 0.08910315304597226, + "p90": -0.0009995435685739617, + "p99": 0.05725053613897321 + }, + "tpot": { + "mean": 0.1751979803646619, + "p50": 0.1677414318174738, + "p90": 0.11306865958187659, + "p99": 0.3674786553708338 + }, + "e2e": { + "mean": 0.1572424162428063, + "p50": 0.1665739541099918, + "p90": 0.1319789697978685, + "p99": 0.18004136280321917 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": 0.08356996427974997, + "min": 0.0830717367573255, + "max": 0.08406819180217444 + }, + "p50": { + "mean": 0.09056549054080736, + "min": 0.08910315304597226, + "max": 0.09202782803564245 + }, + "p90": { + "mean": -0.00035729464911777795, + "min": -0.0009995435685739617, + "max": 0.00028495427033840576 + }, + "p99": { + "mean": 0.054983005715144195, + "min": 0.052715475291315185, + "max": 0.05725053613897321 + } + }, + "tpot": { + "mean": { + "mean": 0.18138306365980178, + "min": 0.1751979803646619, + "max": 0.1875681469549417 + }, + "p50": { + "mean": 0.17383367746313616, + "min": 0.1677414318174738, + "max": 0.17992592310879849 + }, + "p90": { + "mean": 0.11828102571606727, + "min": 0.11306865958187659, + "max": 0.12349339185025797 + }, + "p99": { + "mean": 0.37043609999589633, + "min": 0.3674786553708338, + "max": 0.37339354462095886 + } + }, + "e2e": { + "mean": { + "mean": 0.1631089318431183, + "min": 0.1572424162428063, + "max": 0.16897544744343032 + }, + "p50": { + "mean": 0.17433555090770136, + "min": 0.1665739541099918, + "max": 0.18209714770541094 + }, + "p90": { + "mean": 0.13492751923967394, + "min": 0.1319789697978685, + "max": 0.13787606868147936 + }, + "p99": { + "mean": 0.1845438124004558, + "min": 0.18004136280321917, + "max": 0.18904626199769245 + } + } + }, + "paired_relative_error": [ + { + "ttft": { + "mean": 0.3546551986110786, + "p50": 0.13992436338556363, + "p90": 0.6114206167819235, + "p99": 3.950640311967927 + }, + "tpot": { + "mean": 0.20750924221545067, + "p50": 0.19665829732786516, + "p90": 0.3036816181470401, + "p99": 0.6918780517787506 + }, + "e2e": { + "mean": 0.1987877190488419, + "p50": 0.1867264266897267, + "p90": 0.3097631367891218, + "p99": 0.6627422276019761 + } + }, + { + "ttft": { + "mean": 0.35485558010616797, + "p50": 0.1421694148145008, + "p90": 0.6220489522097341, + "p99": 3.6653171696017184 + }, + "tpot": { + "mean": 0.19287611849010033, + "p50": 0.17961331208149112, + "p90": 0.2699038364626021, + "p99": 0.6456553921719683 + }, + "e2e": { + "mean": 0.18689663758628208, + "p50": 0.17071369418138402, + "p90": 0.2832272665867874, + "p99": 0.6154236743406891 + } + } + ], + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": 0.08356973519379125, + "p50": 0.09021324737193111, + "p90": -0.0006012940489499138, + "p99": 0.05253708684194205 + }, + "tpot": { + "mean": 0.1813506819061229, + "p50": 0.1743324539248605, + "p90": 0.11816822907155744, + "p99": 0.3662267201704473 + }, + "e2e": { + "mean": 0.1630793421767109, + "p50": 0.17360527090575292, + "p90": 0.13652104764058856, + "p99": 0.17677450343779522 + } + }, + "waiting_p99_ms": 499.9578367015957, + "validity": "PASS_SUBCRITICAL" + } + }, + "tp4_rho0p01": { + "old": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": 0.001069282130605019, + "p50": 0.23624600468603504, + "p90": -0.06243912870195954, + "p99": -0.09510776821394695 + }, + "tpot": { + "mean": 0.06803177229402764, + "p50": 0.08606085683396018, + "p90": 0.04169520062620168, + "p99": -0.007371970069137671 + }, + "e2e": { + "mean": 0.07347872011642306, + "p50": 0.12052386744228415, + "p90": 0.03440294047660334, + "p99": 0.038874086941855734 + } + }, + { + "ttft": { + "mean": -0.003986909712741579, + "p50": 0.22473642702400629, + "p90": -0.06765340173498505, + "p99": -0.09602231432410636 + }, + "tpot": { + "mean": 0.06548345266052943, + "p50": 0.08325866738979519, + "p90": 0.03667309817451709, + "p99": -0.012893094435078642 + }, + "e2e": { + "mean": 0.07078275584585807, + "p50": 0.1217320014916679, + "p90": 0.03247676382487049, + "p99": 0.03246095420030834 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": -0.0014588137910682799, + "min": -0.003986909712741579, + "max": 0.001069282130605019 + }, + "p50": { + "mean": 0.23049121585502066, + "min": 0.22473642702400629, + "max": 0.23624600468603504 + }, + "p90": { + "mean": -0.06504626521847229, + "min": -0.06765340173498505, + "max": -0.06243912870195954 + }, + "p99": { + "mean": -0.09556504126902665, + "min": -0.09602231432410636, + "max": -0.09510776821394695 + } + }, + "tpot": { + "mean": { + "mean": 0.06675761247727854, + "min": 0.06548345266052943, + "max": 0.06803177229402764 + }, + "p50": { + "mean": 0.08465976211187769, + "min": 0.08325866738979519, + "max": 0.08606085683396018 + }, + "p90": { + "mean": 0.039184149400359386, + "min": 0.03667309817451709, + "max": 0.04169520062620168 + }, + "p99": { + "mean": -0.010132532252108157, + "min": -0.012893094435078642, + "max": -0.007371970069137671 + } + }, + "e2e": { + "mean": { + "mean": 0.07213073798114056, + "min": 0.07078275584585807, + "max": 0.07347872011642306 + }, + "p50": { + "mean": 0.12112793446697602, + "min": 0.12052386744228415, + "max": 0.1217320014916679 + }, + "p90": { + "mean": 0.033439852150736915, + "min": 0.03247676382487049, + "max": 0.03440294047660334 + }, + "p99": { + "mean": 0.03566752057108204, + "min": 0.03246095420030834, + "max": 0.038874086941855734 + } + } + }, + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": -0.0014652143973501086, + "p50": 0.22936601881498542, + "p90": -0.0646093465409875, + "p99": -0.09621678853313553 + }, + "tpot": { + "mean": 0.06675609059150077, + "p50": 0.08477302587551214, + "p90": 0.0379183273767328, + "p99": -0.010663954751357074 + }, + "e2e": { + "mean": 0.07212904317306096, + "p50": 0.12069215463307655, + "p90": 0.03317293927357903, + "p99": 0.0338378271163308 + } + } + }, + "new": { + "trialwise_distribution_bias": [ + { + "ttft": { + "mean": 0.06234024172127873, + "p50": 0.24853946560541346, + "p90": -0.009255094378797821, + "p99": -0.014267710892388628 + }, + "tpot": { + "mean": 0.10728357759318019, + "p50": 0.09833993314940369, + "p90": 0.08764146258310636, + "p99": 0.07022746965618598 + }, + "e2e": { + "mean": 0.09915483987796365, + "p50": 0.13866670972485098, + "p90": 0.05932039093418303, + "p99": 0.068986345174611 + } + }, + { + "ttft": { + "mean": 0.056974583059155086, + "p50": 0.23691543455574957, + "p90": -0.014765152021066699, + "p99": -0.01526395950520138 + }, + "tpot": { + "mean": 0.10464160330568303, + "p50": 0.09550606196492119, + "p90": 0.08239784923775459, + "p99": 0.06427472725757602 + }, + "e2e": { + "mean": 0.09639439188714723, + "p50": 0.13989440514740228, + "p90": 0.05734781513815735, + "p99": 0.06238732473827772 + } + } + ], + "trialwise_distribution_bias_summary": { + "ttft": { + "mean": { + "mean": 0.05965741239021691, + "min": 0.056974583059155086, + "max": 0.06234024172127873 + }, + "p50": { + "mean": 0.2427274500805815, + "min": 0.23691543455574957, + "max": 0.24853946560541346 + }, + "p90": { + "mean": -0.01201012319993226, + "min": -0.014765152021066699, + "max": -0.009255094378797821 + }, + "p99": { + "mean": -0.014765835198795003, + "min": -0.01526395950520138, + "max": -0.014267710892388628 + } + }, + "tpot": { + "mean": { + "mean": 0.10596259044943161, + "min": 0.10464160330568303, + "max": 0.10728357759318019 + }, + "p50": { + "mean": 0.09692299755716244, + "min": 0.09550606196492119, + "max": 0.09833993314940369 + }, + "p90": { + "mean": 0.08501965591043048, + "min": 0.08239784923775459, + "max": 0.08764146258310636 + }, + "p99": { + "mean": 0.067251098456881, + "min": 0.06427472725757602, + "max": 0.07022746965618598 + } + }, + "e2e": { + "mean": { + "mean": 0.09777461588255544, + "min": 0.09639439188714723, + "max": 0.09915483987796365 + }, + "p50": { + "mean": 0.13928055743612663, + "min": 0.13866670972485098, + "max": 0.13989440514740228 + }, + "p90": { + "mean": 0.05833410303617019, + "min": 0.05734781513815735, + "max": 0.05932039093418303 + }, + "p99": { + "mean": 0.06568683495644435, + "min": 0.06238732473827772, + "max": 0.068986345174611 + } + } + }, + "paired_relative_error": [ + { + "ttft": { + "mean": 0.31553688775988453, + "p50": 0.11838592821378686, + "p90": 0.6506248643406535, + "p99": 3.727741795520502 + }, + "tpot": { + "mean": 0.1255843946058368, + "p50": 0.09013696629948227, + "p90": 0.2591848042876693, + "p99": 0.6394926577997524 + }, + "e2e": { + "mean": 0.12720160501967023, + "p50": 0.09589002059334667, + "p90": 0.26593105160104036, + "p99": 0.6329767056751748 + } + }, + { + "ttft": { + "mean": 0.3055207717400012, + "p50": 0.11067975650739464, + "p90": 0.6346487333489169, + "p99": 3.723279708208583 + }, + "tpot": { + "mean": 0.12290433684113768, + "p50": 0.08881300531692908, + "p90": 0.2544711123916822, + "p99": 0.635068592349857 + }, + "e2e": { + "mean": 0.1238739253923782, + "p50": 0.0938720089907427, + "p90": 0.2617045417392297, + "p99": 0.6208489208233263 + } + } + ], + "legacy_pooled_distribution_bias": { + "ttft": { + "mean": 0.059650620031540064, + "p50": 0.24159106387124998, + "p90": -0.011548419893895705, + "p99": -0.015475807392170575 + }, + "tpot": { + "mean": 0.10596101263201793, + "p50": 0.09703754188844527, + "p90": 0.08369800201750718, + "p99": 0.06667813160571406 + }, + "e2e": { + "mean": 0.09777288053702092, + "p50": 0.1388377217196832, + "p90": 0.058060760567474216, + "p99": 0.06380410696893425 + } + }, + "waiting_p99_ms": 743.1356935161051, + "validity": "PASS_SUBCRITICAL" + } + } + }, + "checks": { + "tp1_ttft_mean_p99_improve_ge_5pp": true, + "tp2_tp4_ttft_e2e_no_abs_regression_gt_5pp": false, + "regressions": [ + { + "cell": "tp4_rho0p0025", + "metric": "ttft", + "quantile": "mean", + "old_bias": 0.025022574738277282, + "new_bias": 0.07766646565061346, + "abs_bias_delta_pp": 5.2643890912336175, + "validity": "PASS_SUBCRITICAL" + }, + { + "cell": "tp4_rho0p005", + "metric": "ttft", + "quantile": "mean", + "old_bias": 0.028648879997638963, + "new_bias": 0.08356973519379125, + "abs_bias_delta_pp": 5.492085519615229, + "validity": "PASS_SUBCRITICAL" + }, + { + "cell": "tp4_rho0p005", + "metric": "ttft", + "quantile": "p50", + "old_bias": 0.028237979190582876, + "new_bias": 0.09021324737193111, + "abs_bias_delta_pp": 6.197526818134823, + "validity": "PASS_SUBCRITICAL" + }, + { + "cell": "tp4_rho0p01", + "metric": "ttft", + "quantile": "mean", + "old_bias": -0.0014652143973501086, + "new_bias": 0.059650620031540064, + "abs_bias_delta_pp": 5.8185405634189955, + "validity": "PASS_SUBCRITICAL" + } + ], + "trace_gate": false + } +} \ No newline at end of file diff --git a/runs/frontier-attn-structured-v0/run_replay.py b/runs/frontier-attn-structured-v0/run_replay.py new file mode 100644 index 0000000..054452e --- /dev/null +++ b/runs/frontier-attn-structured-v0/run_replay.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Replay one real-trace cell with the structured-attention experiment commit.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] +S3_REAL = REPO / "runs/frontier-s3-real-v0" +BASE_REFERENCE = ( + REPO + / "runs/frontier-collective-joint-v0/counterfactual/joint-r2/manifest.json" +) +BASE_COMMIT = "deadc4a321f0baaa534c6ebd17f974123733cdc2" +EXPERIMENT_COMMIT = "1f8900a4ac64e45754b03d0aa7c1dddab65785cf" +PATCH = ROOT / "0001-Experiment-with-structured-attention-prefill-predict.patch" + + +def load_s3_module(): + spec = importlib.util.spec_from_file_location( + "s3_prefix_replay", S3_REAL / "run_frontier_prefix_replay.py" + ) + module = importlib.util.module_from_spec(spec) + sys.path.insert(0, str(S3_REAL)) + spec.loader.exec_module(module) + return module + + +def git(checkout: Path, *args: str) -> str: + return subprocess.check_output( + ["git", "-C", str(checkout), *args], text=True + ).strip() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--trace", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument( + "--config", + choices=("tp4_mns16", "tp2_mns16", "tp1_mns16"), + required=True, + ) + parser.add_argument("--label", required=True) + parser.add_argument("--max-tokens", type=int, required=True) + parser.add_argument("--duration-s", type=float) + parser.add_argument("--cache-root", type=Path, required=True) + parser.add_argument( + "--frontier-checkout", + type=Path, + default=Path("/tmp/frontier-attn-structured-v0"), + ) + parser.add_argument( + "--attention-profile", + type=Path, + default=REPO + / "runs/frontier-prefill-kvgrowth-fix-v0/profiles/" + "profile-v5-kvgrowth/attention.csv", + ) + args = parser.parse_args() + + frontier = args.frontier_checkout.resolve() + profile = args.attention_profile.resolve() + if git(frontier, "rev-parse", "HEAD") != EXPERIMENT_COMMIT: + raise SystemExit(f"unexpected experiment checkout HEAD: {frontier}") + if git(frontier, "rev-parse", "HEAD^") != BASE_COMMIT: + raise SystemExit("experiment commit is not directly based on frozen Frontier") + if git(frontier, "status", "--porcelain"): + raise SystemExit("experiment Frontier checkout must be clean") + if not profile.is_file(): + raise SystemExit(f"attention profile missing: {profile}") + + reference = json.loads(BASE_REFERENCE.read_text()) + reference["frontier_checkout"] = str(frontier) + reference["frontier_commit"] = EXPERIMENT_COMMIT + generated_reference = ROOT / "frontier-reference.json" + generated_reference.write_text(json.dumps(reference, indent=2)) + + module = load_s3_module() + module.REFERENCE = generated_reference + module.EXPECTED_FRONTIER_COMMIT = EXPERIMENT_COMMIT + original_replace = module.replace_flag + + def replace_and_override(argv: list[str], flag: str, value: str) -> None: + original_replace(argv, flag, value) + if flag.endswith("trace_file"): + atten_flag = ( + "--random_forrest_execution_time_predictor_config_atten_input_file" + ) + original_replace(argv, atten_flag, str(profile)) + no_cache = ( + "--random_forrest_execution_time_predictor_config_no_cache" + ) + if no_cache in argv: + argv.remove(no_cache) + + module.replace_flag = replace_and_override + module.parse_args = lambda: args + module.main() + + manifest_path = args.output_root / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest.update( + { + "schema": "frontier-attn-structured-replay-v1", + "frontier_base_commit": BASE_COMMIT, + "frontier_experiment_commit": EXPERIMENT_COMMIT, + "frontier_patch": str(PATCH.resolve()), + "frontier_patch_sha256": module.sha256(PATCH), + "attention_profile_override": str(profile), + "attention_profile_sha256": module.sha256(profile), + "model_cache_enabled": True, + } + ) + manifest_path.write_text(json.dumps(manifest, indent=2)) + print(f"structured replay done: {args.output_root}") + + +if __name__ == "__main__": + main()